def run(self, state: State) -> None: """Add release information to justification for selected packages.""" justification_addition = [] for resolved_package_tuple in state.resolved_dependencies.values(): conf_matched = self._conf_map.get(resolved_package_tuple[0]) if not conf_matched: continue version = conf_matched["package_version"].get("version") if version is not None and resolved_package_tuple[ 1] not in SpecifierSet(version): continue index_url = conf_matched["package_version"].get("index_url") if index_url is not None and resolved_package_tuple[2] != index_url: continue justification_addition.append({ "type": "INFO", "message": f"Release notes for package {resolved_package_tuple[0]!r}", "link": self._construct_release_notes_url( organization=conf_matched["organization"], repository=conf_matched["repository"], tag_version_prefix=conf_matched.get("tag_version_prefix"), locked_version=resolved_package_tuple[1], ), }) state.add_justification(justification_addition)
def test_add_state_order_single(self) -> None: """Test adding states to beam and order during addition when score is same - iteration is relevant.""" beam = Beam(width=1) state01 = State( score=0.0, latest_version_offset=0, iteration=1, iteration_states_added=0, ) state01.add_justification([{"state": "01"}]) beam.add_state(state01) state02 = State( score=0.0, latest_version_offset=0, iteration=0, iteration_states_added=0, ) state02.add_justification([{"state": "02"}]) beam.add_state(state02) assert list(beam.iter_states()) == [state01] assert list(beam.iter_states_sorted(reverse=True)) == [state01] assert list(beam.iter_states_sorted(reverse=False)) == [state01]
def run(self, state: State) -> None: """Add release information to justification for selected packages.""" justification_addition = [] for resolved_package_tuple in state.resolved_dependencies.values(): conf_package_version = self.configuration["package_version"] if not conf_package_version or resolved_package_tuple[ 0] != conf_package_version["name"]: continue version = conf_package_version.get("version") if version is not None and resolved_package_tuple[ 1] not in SpecifierSet(version): continue develop = conf_package_version.get("develop") if develop is not None: package_version = self.context.get_package_version( resolved_package_tuple) if not package_version: # This is a programming error as the give dependency has to be registered in the context. _LOGGER.error( "No matching package version for %r registered in the context", resolved_package_tuple) continue if package_version.develop != develop: continue index_url = conf_package_version.get("index_url") if not self._index_url_check(index_url, resolved_package_tuple[2]): continue if self._configuration["prescription"]["run"]: # Can happen if prescription states criteria that match multiple times. We add # justification only once in such cases. continue self._configuration["prescription"]["run"] = True release_notes_conf = self.configuration["release_notes"] justification_addition.append({ "type": "INFO", "message": f"Release notes for package {resolved_package_tuple[0]!r}", "link": self._construct_release_notes_url( organization=release_notes_conf["organization"], repository=release_notes_conf["repository"], tag_version_prefix=release_notes_conf.get( "tag_version_prefix"), locked_version=resolved_package_tuple[1], ), "package_name": resolved_package_tuple[0], }) state.add_justification(justification_addition)
def test_justification(self, final_state: State) -> None: """Test manipulation with state justification.""" assert final_state.justification == [{"foo": "bar"}] final_state.add_justification([{"hello": "thoth"}]) assert final_state.justification == [{ "foo": "bar" }, { "hello": "thoth" }]
def final_state() -> State: """A fixture for a final state.""" state = State( score=0.5, resolved_dependencies=OrderedDict( {"daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple")}), unresolved_dependencies=OrderedDict(), advised_runtime_environment=None, ) state.add_justification([{"foo": "bar"}]) return state
def final_state() -> State: # noqa: D401 """A fixture for a final state.""" state = State( score=0.5, resolved_dependencies={ "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple") }, unresolved_dependencies={}, advised_runtime_environment=None, ) state.add_justification(AdviserTestCase.JUSTIFICATION_SAMPLE_1) return state
def test_run_justification_noop(self) -> None: """Test no operation when justification is present.""" state = State() assert not state.justification state.add_justification([{"type": "INFO", "message": "Foo bar"}]) unit = NoObservationWrap() unit.run(state) assert len(state.justification) == 1 assert set(state.justification[0].keys()) == {"type", "message"}
def run(self, state: State) -> None: """Run main entry-point for wrap units to filter and score packages.""" if not self._run_state(state): return None justification = self.run_prescription.get("justification") if justification: state.add_justification(justification) advised_manifest_changes = self.run_prescription.get( "advised_manifest_changes") if advised_manifest_changes: state.advised_manifest_changes.append(advised_manifest_changes) self._run_base()
def state() -> State: """A fixture for a non-final state.""" state = State( score=0.1, unresolved_dependencies=OrderedDict( {"flask": ("flask", "1.1.1", "https://pypi.org/simple")} ), resolved_dependencies=OrderedDict( {"hexsticker": ("hexsticker", "1.0.0", "https://pypi.org/simple")} ), advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"} ), ) state.add_justification([{"foo": "bar"}, {"bar": "baz"}]) return state
def _get_state() -> State: """Create a fixture for a non-final state.""" unresolved_dependency = ("flask", "1.1.1", "https://pypi.org/simple") unresolved_dependencies = { id(unresolved_dependency): unresolved_dependency } state = State( score=0.1, unresolved_dependencies={"flask": unresolved_dependencies}, resolved_dependencies={ "hexsticker": ("hexsticker", "1.0.0", "https://pypi.org/simple") }, advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"}), ) state.add_justification(AdviserTestCase.JUSTIFICATION_SAMPLE_1) return state
def test_add_state_order_single(self) -> None: """Test adding states to beam and order during addition when score is same - iteration is relevant.""" beam = Beam(width=1) state01 = State( score=0.0, iteration=1, ) state01.add_justification(self.JUSTIFICATION_SAMPLE_1) beam.add_state(state01) state02 = State( score=0.0, iteration=0, ) state02.add_justification(self.JUSTIFICATION_SAMPLE_2) beam.add_state(state02) assert list(beam.iter_states()) == [state01] assert list(beam.iter_states_sorted(reverse=True)) == [state01] assert list(beam.iter_states_sorted(reverse=False)) == [state01]
def run(self, state: State) -> None: """Run main entry-point for wrap units to filter and score packages.""" if not self._run_state(state): return None prescription_conf = self._configuration["prescription"] if not prescription_conf["run"]: justification = self.run_prescription.get("justification") if justification: state.add_justification(justification) advised_manifest_changes = self.run_prescription.get( "advised_manifest_changes") if advised_manifest_changes: state.advised_manifest_changes.append(advised_manifest_changes) try: self._run_base() finally: prescription_conf["run"] = True
def test_add_state_order_multi(self) -> None: """Test adding states to beam and order during addition when score is same.""" beam = Beam(width=2) state01 = State(score=0.0, iteration=0, iteration_states_added=0) state01.add_justification([{"state": "01"}]) beam.add_state(state01) state02 = State(score=0.0, iteration=0, iteration_states_added=1) state02.add_justification([{"state": "02"}]) beam.add_state(state02) state03 = State(score=0.0, iteration=1, iteration_states_added=0) state03.add_justification([{"state": "03"}]) beam.add_state(state03) state04 = State(score=0.0, iteration=1, iteration_states_added=1) state04.add_justification([{"state": "04"}]) beam.add_state(state04) assert list(beam.iter_states_sorted()) == [state03, state04] assert list(beam.iter_states_sorted(reverse=True)) == [state03, state04] assert list(beam.iter_states_sorted(reverse=False)) == [state04, state03]
def test_add_state_order_multi(self) -> None: """Test adding states to beam and order during addition when score is same.""" beam = Beam(width=2) state01 = State(score=0.0) state01.add_justification(self.JUSTIFICATION_SAMPLE_1) beam.add_state(state01) state02 = State(score=0.0) state02.add_justification(self.JUSTIFICATION_SAMPLE_2) beam.add_state(state02) state03 = State(score=0.0) state03.add_justification(self.JUSTIFICATION_SAMPLE_3) beam.add_state(state03) assert list(beam.iter_states_sorted()) == [state01, state02] assert list(beam.iter_states_sorted(reverse=True)) == [state01, state02] assert list(beam.iter_states_sorted(reverse=False)) == [state01, state02]
def test_justification(self, final_state: State) -> None: """Test manipulation with state justification.""" assert final_state.justification == self.JUSTIFICATION_SAMPLE_1 final_state.add_justification(self.JUSTIFICATION_SAMPLE_2) assert final_state.justification == list( chain(self.JUSTIFICATION_SAMPLE_1, self.JUSTIFICATION_SAMPLE_2))
def test_from_final_state(self, context: Context) -> None: """Test instantiating product from a final state.""" state = State( score=0.5, resolved_dependencies=OrderedDict( { "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.17.4", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), } ), unresolved_dependencies=OrderedDict(), advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"} ), ) state.add_justification([{"foo": "bar"}]) pypi = Source("https://pypi.org/simple") pv_daiquiri_locked = PackageVersion( name="daiquiri", version="==1.6.0", index=pypi, develop=False ) pv_numpy_locked = PackageVersion( name="numpy", version="==1.17.4", index=pypi, develop=False ) pv_tensorflow_locked = PackageVersion( name="tensorflow", version="==2.0.0", index=pypi, develop=False ) context.should_receive("get_package_version").with_args( ("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_daiquiri_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "daiquiri", "1.6.0", "https://pypi.org/simple" ).and_return(["000"]).ordered() context.should_receive("get_package_version").with_args( ("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.17.4", "https://pypi.org/simple" ).and_return(["111"]).ordered() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["222"]).ordered() pv_daiquiri = PackageVersion( name="daiquiri", version="*", index=pypi, develop=False ) pv_tensorflow = PackageVersion( name="tensorflow", version=">=2.0.0", index=pypi, develop=False ) project = flexmock() project.should_receive("iter_dependencies").with_args( with_devel=True ).and_return([pv_daiquiri, pv_tensorflow]).once() context.project = project context.dependencies = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): set() }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): { ("numpy", "1.17.4", "https://pypi.org/simple") } }, } context.dependents = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): { (("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7") } }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): set() }, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.17.4", os_name="fedora", os_version="31", python_version="3.7", ).and_return("python_version >= '3.7'").once() product = Product.from_final_state(state=state, context=context) assert product.score == state.score assert product.justification == state.justification assert product.advised_runtime_environment == state.advised_runtime_environment assert product.project.to_dict() == { "requirements": { "packages": { "daiquiri": {"index": "pypi-org", "version": "*"}, "tensorflow": {"index": "pypi-org", "version": ">=2.0.0"}, }, "dev-packages": {}, "source": [ { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", } ], }, "requirements_locked": { "_meta": { "sources": [ { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", } ], "requires": {}, "hash": { "sha256": "c3a2f42932b6e5cd30f5664b11eda605f5fbd672f1b88729561d0d3edd10b5d9" }, "pipfile-spec": 6, }, "default": { "daiquiri": { "version": "==1.6.0", "hashes": ["sha256:000"], "index": "pypi-org", }, "numpy": { "version": "==1.17.4", "hashes": ["sha256:111"], "index": "pypi-org", "markers": "python_version >= '3.7'", }, "tensorflow": { "version": "==2.0.0", "hashes": ["sha256:222"], "index": "pypi-org", }, }, "develop": {}, }, "runtime_environment": { "hardware": {"cpu_family": None, "cpu_model": None}, "operating_system": {"name": None, "version": None}, "python_version": None, "cuda_version": None, "name": None, }, }
def test_from_final_state(self, context: Context) -> None: """Test instantiating product from a final state.""" state = State( score=0.5, resolved_dependencies={ "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.17.4", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, advised_runtime_environment=RuntimeEnvironment.from_dict({"python_version": "3.6"}), ) state.add_justification(self.JUSTIFICATION_SAMPLE_1) pypi = Source("https://pypi.org/simple") pv_daiquiri_locked = PackageVersion(name="daiquiri", version="==1.6.0", index=pypi, develop=False) pv_numpy_locked = PackageVersion(name="numpy", version="==1.17.4", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_daiquiri_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "daiquiri", "1.6.0", "https://pypi.org/simple" ).and_return(["000"]).ordered() context.should_receive("get_package_version").with_args( ("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.17.4", "https://pypi.org/simple" ).and_return(["111"]).ordered() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["222"]).ordered() pv_daiquiri = PackageVersion(name="daiquiri", version="*", index=pypi, develop=False) pv_tensorflow = PackageVersion(name="tensorflow", version=">=2.0.0", index=pypi, develop=False) project = flexmock( pipfile=Pipfile.from_string(self._PIPFILE), runtime_environment=RuntimeEnvironment.from_dict({"operating_system": {"name": "rhel"}}), ) project.should_receive("iter_dependencies").with_args(with_devel=True).and_return( [pv_daiquiri, pv_tensorflow] ).once() context.project = project context.dependencies = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": {("numpy", "1.17.4", "https://pypi.org/simple"): set()}, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): {("numpy", "1.17.4", "https://pypi.org/simple")} }, } context.dependents = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): { ( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ) } }, "tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()}, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.17.4", os_name="fedora", os_version="31", python_version="3.7", marker_evaluation_result=True, ).and_return("python_version >= '3.7'").once() assert "THOTH_ADVISER_METADATA" not in os.environ metadata_justification = {"thoth.adviser": {"justification": [{"bar": "baz"}]}} os.environ["THOTH_ADVISER_METADATA"] = json.dumps(metadata_justification) try: product = Product.from_final_state(state=state, context=context) finally: os.environ.pop("THOTH_ADVISER_METADATA") assert product.score == state.score assert product.justification == list( chain(metadata_justification["thoth.adviser"]["justification"], state.justification) ) assert product.advised_runtime_environment == state.advised_runtime_environment assert product.project.to_dict() == { "constraints": [], "requirements": { "packages": { "daiquiri": {"index": "pypi-org-simple", "version": "*"}, "tensorflow": {"index": "pypi-org-simple", "version": ">=2.0.0"}, }, "dev-packages": {}, "requires": {"python_version": "3.7"}, "source": [ { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", }, { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org-simple", }, ], "thoth": { "allow_prereleases": {"black": True}, }, }, "requirements_locked": { "_meta": { "sources": [ {"name": "pypi-org", "url": "https://pypi.org/simple", "verify_ssl": True}, { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org-simple", }, ], "requires": {"python_version": "3.7"}, "hash": {"sha256": "6cc8365e799b949fb6cc564cea2d8e0e8a782ab676a006e65abbe14621b93381"}, "pipfile-spec": 6, }, "default": { "daiquiri": { "version": "==1.6.0", "hashes": ["sha256:000"], "index": "pypi-org-simple", }, "numpy": { "version": "==1.17.4", "hashes": ["sha256:111"], "index": "pypi-org-simple", "markers": "python_version >= '3.7'", }, "tensorflow": { "version": "==2.0.0", "hashes": ["sha256:222"], "index": "pypi-org-simple", }, }, "develop": {}, }, "runtime_environment": { "hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None}, "operating_system": {"name": "rhel", "version": None}, "python_version": None, "cuda_version": None, "labels": None, "cudnn_version": None, "name": None, "platform": None, "base_image": None, "mkl_version": None, "openblas_version": None, "openmpi_version": None, "recommendation_type": None, }, }
def test_from_final_state(self, context: Context) -> None: """Test instantiating product from a final state.""" state = State( score=0.5, resolved_dependencies={ "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.17.4", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"}), ) state.add_justification(self.JUSTIFICATION_SAMPLE_1) pypi = Source("https://pypi.org/simple") pv_daiquiri_locked = PackageVersion(name="daiquiri", version="==1.6.0", index=pypi, develop=False) pv_numpy_locked = PackageVersion(name="numpy", version="==1.17.4", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False).and_return(pv_daiquiri_locked).ordered() context.graph.should_receive( "get_python_package_hashes_sha256").with_args( "daiquiri", "1.6.0", "https://pypi.org/simple").and_return(["000"]).ordered() context.should_receive("get_package_version").with_args( ("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False).and_return(pv_numpy_locked).ordered() context.graph.should_receive( "get_python_package_hashes_sha256").with_args( "numpy", "1.17.4", "https://pypi.org/simple").and_return(["111"]).ordered() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False).and_return(pv_tensorflow_locked).ordered() context.graph.should_receive( "get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple").and_return(["222"]).ordered() pv_daiquiri = PackageVersion(name="daiquiri", version="*", index=pypi, develop=False) pv_tensorflow = PackageVersion(name="tensorflow", version=">=2.0.0", index=pypi, develop=False) project = flexmock( pipfile=Pipfile.from_string(self._PIPFILE), runtime_environment=RuntimeEnvironment.from_dict( {"operating_system": { "name": "rhel" }}), ) project.should_receive("iter_dependencies").with_args( with_devel=True).and_return([pv_daiquiri, pv_tensorflow]).once() context.project = project context.dependencies = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): set() }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): {("numpy", "1.17.4", "https://pypi.org/simple")} }, } context.dependents = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): {( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", )} }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): set() }, } context.graph.should_receive( "get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.17.4", os_name="fedora", os_version="31", python_version="3.7", ).and_return("python_version >= '3.7'").once() product = Product.from_final_state(state=state, context=context) assert product.score == state.score assert product.justification == state.justification assert product.advised_runtime_environment == state.advised_runtime_environment assert product.project.to_dict() == { "requirements": { "packages": { "daiquiri": { "index": "pypi-org", "version": "*" }, "tensorflow": { "index": "pypi-org", "version": ">=2.0.0" }, }, "dev-packages": {}, "requires": { "python_version": "3.7" }, "source": [{ "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", }], }, "requirements_locked": { "_meta": { "sources": [{ "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", }], "requires": { "python_version": "3.7" }, "hash": { "sha256": "f08689732b596fd705a45bbf9ec44c3995b17a1aa6392c46500aeb736c4d4e88" }, "pipfile-spec": 6, }, "default": { "daiquiri": { "version": "==1.6.0", "hashes": ["sha256:000"], "index": "pypi-org", }, "numpy": { "version": "==1.17.4", "hashes": ["sha256:111"], "index": "pypi-org", "markers": "python_version >= '3.7'", }, "tensorflow": { "version": "==2.0.0", "hashes": ["sha256:222"], "index": "pypi-org", }, }, "develop": {}, }, "runtime_environment": { "hardware": { "cpu_family": None, "cpu_model": None }, "operating_system": { "name": "rhel", "version": None }, "python_version": None, "cuda_version": None, "name": None, "platform": None, }, }