def test_to_dict(self, pipeline_config: PipelineConfig) -> None: """Test conversion to a dict.""" report = Report(count=3, pipeline=pipeline_config) project = flexmock() project_dict = {"aresto momentum": "avada kedavra"} project.should_receive("to_dict").with_args( keep_thoth_section=True).and_return( project_dict).twice() # In test and in the report. product = Product( project=project, score=0.666, justification=[{ "gryffindor": "le gladium leviosa" }], advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"}), ) report.add_product(product) assert report.product_count() == 1 assert list(report.iter_products()) == [product] assert report.to_dict() == { "pipeline": pipeline_config.to_dict(), "products": [product.to_dict()], "stack_info": [], "resolver_iterations": 0, "accepted_final_states_count": 0, "discarded_final_states_count": 0, }
def test_add_product(self, pipeline_config: PipelineConfig) -> None: """Test adding a product to a report.""" report = Report(count=2, pipeline=pipeline_config) product1 = Product(project=None, score=0.42, justification=[], advised_runtime_environment=None) report.add_product(product1) assert report.product_count() == 1 assert list(report.iter_products()) == [product1] assert list(report.iter_products_sorted()) == [product1] product2 = Product(project=None, score=0.0, justification=[], advised_runtime_environment=None) report.add_product(product2) assert report.product_count() == 2 assert set(report.iter_products()) == {product1, product2} assert list(report.iter_products_sorted()) == [product1, product2] assert list( report.iter_products_sorted(reverse=True)) == [product1, product2] assert list(report.iter_products_sorted(reverse=False)) == [ product2, product1 ] product3 = Product(project=None, score=0.98, justification=[], advised_runtime_environment=None) report.add_product(product3) assert report.product_count() == 2 assert set(report.iter_products()) == {product3, product1} assert list(report.iter_products_sorted()) == [product3, product1] assert list( report.iter_products_sorted(reverse=True)) == [product3, product1] assert list(report.iter_products_sorted(reverse=False)) == [ product1, product3 ] product4 = Product( project=None, score=0.666, justification=[], advised_runtime_environment=None, ) report.add_product(product4) assert report.product_count() == 2 assert set(report.iter_products()) == {product3, product4} assert list(report.iter_products_sorted()) == [product3, product4] assert list( report.iter_products_sorted(reverse=True)) == [product3, product4] assert list(report.iter_products_sorted(reverse=False)) == [ product4, product3 ]
def test_to_dict(self) -> None: """Test conversion of this product into a dictionary representation.""" project = flexmock() project.should_receive("to_dict").with_args().and_return({ "baz": "bar" }).once() advised_runtime_environment = flexmock() advised_runtime_environment.should_receive( "to_dict").with_args().and_return({ "hello": "thoth" }).once() advised_manifest_changes = [[{ "apiVersion": "apps.openshift.io/v1", "kind": "DeploymentConfig", "patch": { "op": "add", "path": "spec.template.spec.containers[0].env", "value": { "name": "OMP_NUM_THREADS", "value": "1" }, }, }]] product = Product( advised_manifest_changes=advised_manifest_changes, advised_runtime_environment=advised_runtime_environment, justification=[{ "foo": "bar" }], project=project, score=0.999, ) assert product.to_dict() == { "score": 0.999, "project": { "baz": "bar" }, "justification": [{ "foo": "bar" }], "advised_runtime_environment": { "hello": "thoth" }, "advised_manifest_changes": advised_manifest_changes, }
def test_to_dict_metadata(self, pipeline_config: PipelineConfig) -> None: """Test conversion to a dict with passed metadata.""" report = Report(count=3, pipeline=pipeline_config) project = flexmock() project_dict = {"aresto momentum": "avada kedavra"} project.should_receive("to_dict").with_args( keep_thoth_section=True).and_return(project_dict) product = Product( project=project, score=0.666, justification=[{ "gryffindor": "le gladium leviosa" }], advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"}), ) report.add_product(product) stack_info = [{"type": "WARNING", "message": "Hello, metadata"}] stack_info_metadata = { "thoth.adviser": { "stack_info": stack_info, } } report.set_stack_info([{"foo": "bar"}]) assert "THOTH_ADVISER_METADATA" not in os.environ os.environ["THOTH_ADVISER_METADATA"] = json.dumps(stack_info_metadata) try: assert report.product_count() == 1 assert list(report.iter_products()) == [product] assert report.to_dict() == { "pipeline": pipeline_config.to_dict(), "products": [product.to_dict()], "stack_info": list(chain(stack_info, report.stack_info)), "resolver_iterations": 0, "accepted_final_states_count": 0, "discarded_final_states_count": 0, } except Exception: os.environ.pop("THOTH_ADVISER_METADATA") raise
def test_to_dict(self) -> None: """Test conversion of this product into a dictionary representation.""" project = flexmock() project.should_receive("to_dict").with_args().and_return({"baz": "bar"}).once() advised_runtime_environment = flexmock() advised_runtime_environment.should_receive("to_dict").with_args().and_return( {"hello": "thoth"} ).once() product = Product( score=0.999, project=project, justification=[{"foo": "bar"}], advised_runtime_environment=advised_runtime_environment, ) assert product.to_dict() == { "score": 0.999, "project": {"baz": "bar"}, "justification": [{"foo": "bar"}], "advised_runtime_environment": {"hello": "thoth"}, }
def test_construct_dependency_graph_cycle(self, context: Context) -> None: """Test constructing dependency graph information with cycles.""" context = flexmock(dependencies=self._DEPENDENCIES_CYCLE) dependency_graph = Product._construct_dependency_graph(context, self._DEPENDENCIES_CYCLE_PIPFILE_LOCK) assert dependency_graph["nodes"] == ["a", "b", "c"] assert set(tuple(i) for i in dependency_graph["edges"]) == {(0, 1), (2, 0), (2, 1)}
def test_construct_dependency_graph_basic(self) -> None: """Test constructing dependency graph.""" context = flexmock(dependencies=self._DEPENDENCIES_NO_CYCLE) dependency_graph = Product._construct_dependency_graph(context, self._DEPENDENCIES_NO_CYCLE_PIPFILE_LOCK) assert dependency_graph["nodes"] == ["absl-py", "astor", "six", "tensorflow"] assert set(tuple(i) for i in dependency_graph["edges"]) == {(0, 2), (1, 2), (3, 1), (3, 0)}
def test_no_observation(self, context: Context) -> None: """Test adding information about no justification added.""" state = State( score=0.0, resolved_dependencies={ "flask": ("flask", "0.12", "https://pypi.org/simple"), }, unresolved_dependencies={}, ) context.project.pipfile.packages.packages.pop("tensorflow") context.graph.should_receive("get_python_package_hashes_sha256").with_args( "flask", "0.12", "https://pypi.org/simple" ).and_return(["222"]).once() pypi = Source("https://pypi.org/simple") pv_pandas_locked = PackageVersion(name="flask", version="==0.12", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("flask", "0.12", "https://pypi.org/simple"), graceful=False ).and_return(pv_pandas_locked).once() context.dependents = { "flask": {("flask", "0.12", "https://pypi.org/simple"): set()}, } product = Product.from_final_state(context=context, state=state) expected = { "advised_manifest_changes": [], "advised_runtime_environment": None, "dependency_graph": {"edges": [], "nodes": ["flask"]}, "justification": [ { "type": "INFO", "message": "No issues spotted for this stack based on Thoth's database", "link": jl("no_observations"), } ], "project": { "constraints": [], "requirements": { "dev-packages": {}, "packages": {"flask": "*"}, "requires": {"python_version": "3.6"}, "source": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, {"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True}, ], "thoth": { "allow_prereleases": {}, "disable_index_adjustment": False, }, }, "requirements_locked": { "_meta": { "hash": {"sha256": "2e49395dfa87159358e581bd22e656c27c0dab04894d1b137a14f85bb387ea51"}, "pipfile-spec": 6, "requires": {"python_version": "3.6"}, "sources": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, {"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True}, ], }, "default": { "flask": {"hashes": ["sha256:222"], "index": "pypi-org-simple", "version": "==0.12"}, }, "develop": {}, }, "runtime_environment": { "base_image": None, "cuda_version": None, "cudnn_version": None, "hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None}, "name": None, "operating_system": {"name": None, "version": None}, "labels": None, "platform": None, "mkl_version": None, "openmpi_version": None, "openblas_version": None, "python_version": None, "recommendation_type": None, }, }, "score": 0.0, } assert product.to_dict() == expected
def test_environment_markers_shared(self, context: Context) -> None: """Test handling of environment markers when multiple dependencies share one.""" state = State( score=0.0, resolved_dependencies={ "pandas": ("pandas", "1.0.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.0.0", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, justification=[{"type": "INFO", "message": "Foo bar", "link": "https://thoth-station.ninja"}], ) # Make sure tested packages are not direct dependencies. In such cases the behaviour is # different, see other tests. context.project.pipfile.packages.packages.pop("pandas", None) context.project.pipfile.packages.packages.pop("numpy", None) context.project.pipfile.packages.packages.pop("tensorflow", None) context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.0.0", "https://pypi.org/simple" ).and_return(["000"]).once() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["111"]).once() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "pandas", "1.0.0", "https://pypi.org/simple" ).and_return(["222"]).once() pypi = Source("https://pypi.org/simple") pv_numpy_locked = PackageVersion(name="numpy", version="==1.0.0", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) pv_pandas_locked = PackageVersion(name="pandas", version="==1.0.0", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).once() context.should_receive("get_package_version").with_args( ("pandas", "1.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_pandas_locked).once() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).once() context.dependents = { "numpy": { ("numpy", "1.0.0", "https://pypi.org/simple"): [ # set to list for reproducible runs. ( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ), ( ("pandas", "1.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ), ] }, "tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()}, "pandas": {("pandas", "1.0.0", "https://pypi.org/simple"): set()}, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.0.0", os_name="fedora", os_version="31", python_version="3.7", marker_evaluation_result=True, ).and_return("python_version >= '3.8'").once() context.graph.should_receive("get_python_environment_marker").with_args( "pandas", "1.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.0.0", os_name="fedora", os_version="31", python_version="3.7", marker_evaluation_result=True, ).and_return(None).once() product = Product.from_final_state(context=context, state=state) expected = { "advised_manifest_changes": [], "advised_runtime_environment": None, "dependency_graph": {"edges": [], "nodes": ["pandas", "numpy", "tensorflow"]}, "justification": [ { "link": "https://thoth-station.ninja", "message": "Foo bar", "type": "INFO", } ], "project": { "constraints": [], "requirements": { "dev-packages": {}, "packages": {"flask": "*"}, "requires": {"python_version": "3.6"}, "source": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, {"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True}, ], "thoth": { "allow_prereleases": {}, "disable_index_adjustment": False, }, }, "requirements_locked": { "_meta": { "hash": {"sha256": "2e49395dfa87159358e581bd22e656c27c0dab04894d1b137a14f85bb387ea51"}, "pipfile-spec": 6, "requires": {"python_version": "3.6"}, "sources": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, {"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True}, ], }, "default": { "numpy": {"hashes": ["sha256:000"], "index": "pypi-org-simple", "version": "==1.0.0"}, "pandas": {"hashes": ["sha256:222"], "index": "pypi-org-simple", "version": "==1.0.0"}, "tensorflow": {"hashes": ["sha256:111"], "index": "pypi-org-simple", "version": "==2.0.0"}, }, "develop": {}, }, "runtime_environment": { "base_image": None, "cuda_version": None, "cudnn_version": None, "hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None}, "name": None, "operating_system": {"name": None, "version": None}, "labels": None, "platform": None, "mkl_version": None, "openmpi_version": None, "openblas_version": None, "python_version": None, "recommendation_type": None, }, }, "score": 0.0, } assert product.to_dict() == expected
def test_environment_markers_direct_dependency(self, context: Context) -> None: """Test handling environment markers for direct dependencies.""" state = State( score=0.0, resolved_dependencies={ "numpy": ("numpy", "1.0.0", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, justification=[{"type": "INFO", "message": "Foo bar", "link": "https://thoth-station.ninja"}], ) pypi = Source("https://pypi.org/simple") # Let's assume tensorflow is our direct dependency with an environment marker set. It sets also an environment # marker for numpy. context.project.pipfile.packages.packages.pop("tensorflow", None) tf_package_version_direct = PackageVersion( name="tensorflow", version=">1.0.0", index=pypi, develop=False, markers="python_version >= '3.6'" ) context.project.pipfile.add_package_version(tf_package_version_direct) # Just to make sure numpy is not in the direct dependency listing. context.project.pipfile.packages.packages.pop("numpy", None) pv_numpy_locked = PackageVersion(name="numpy", version="==1.0.0", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) context.graph.should_receive("get_python_package_hashes_sha256").with_args( *pv_numpy_locked.to_tuple() ).and_return(["000"]).once() context.graph.should_receive("get_python_package_hashes_sha256").with_args( *pv_tensorflow_locked.to_tuple() ).and_return(["111"]).once() context.should_receive("get_package_version").with_args( ("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).once() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).once() context.dependents = { "numpy": { ("numpy", "1.0.0", "https://pypi.org/simple"): { ( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ) } }, "tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()}, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.0.0", os_name="fedora", os_version="31", python_version="3.7", marker_evaluation_result=True, ).and_return("python_version >= '3.7'").once() product = Product.from_final_state(context=context, state=state) expected = { "advised_manifest_changes": [], "advised_runtime_environment": None, "justification": [ { "link": "https://thoth-station.ninja", "message": "Foo bar", "type": "INFO", } ], "dependency_graph": {"edges": [], "nodes": ["numpy", "tensorflow"]}, "project": { "constraints": [], "requirements": { "dev-packages": {}, "packages": { "flask": "*", "tensorflow": { "index": "pypi-org-simple", "markers": "python_version >= '3.6'", "version": ">1.0.0", }, }, "requires": {"python_version": "3.6"}, "source": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True, }, { "name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True, }, ], "thoth": { "allow_prereleases": {}, "disable_index_adjustment": False, }, }, "requirements_locked": { "_meta": { "hash": {"sha256": "382cd84046246d08cf670a07c61628958dec76e05d2473bf936866fc54619f9a"}, "pipfile-spec": 6, "requires": {"python_version": "3.6"}, "sources": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, { "name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True, }, ], }, "default": { "numpy": { "hashes": ["sha256:000"], "index": "pypi-org-simple", "markers": "python_version >= '3.7'", "version": "==1.0.0", }, "tensorflow": { "hashes": ["sha256:111"], "index": "pypi-org-simple", "markers": "python_version >= '3.6'", "version": "==2.0.0", }, }, "develop": {}, }, "runtime_environment": { "base_image": None, "cuda_version": None, "cudnn_version": None, "hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None}, "name": None, "operating_system": {"name": None, "version": None}, "openblas_version": None, "labels": None, "openmpi_version": None, "mkl_version": None, "python_version": None, "recommendation_type": None, "platform": None, }, }, "score": 0.0, } assert product.to_dict() == expected
def test_to_dict(self, context: Context) -> None: """Test conversion of this product into a dictionary representation.""" project = context.project project.should_receive("to_dict").with_args(keep_thoth_section=True).and_return({"baz": "bar"}).once() advised_runtime_environment = flexmock() advised_runtime_environment.should_receive("to_dict").with_args().and_return({"hello": "thoth"}).once() advised_manifest_changes = [ [ { "apiVersion": "apps.openshift.io/v1", "kind": "DeploymentConfig", "patch": { "op": "add", "path": "spec.template.spec.containers[0].env", "value": {"name": "OMP_NUM_THREADS", "value": "1"}, }, } ] ] product = Product( advised_manifest_changes=advised_manifest_changes, advised_runtime_environment=advised_runtime_environment, justification=[{"foo": "bar"}], project=project, score=0.999, context=context, ) assert product.to_dict() == { "score": 0.999, "project": {"baz": "bar"}, "justification": [{"foo": "bar"}], "advised_runtime_environment": {"hello": "thoth"}, "advised_manifest_changes": advised_manifest_changes, "dependency_graph": { "edges": [], "nodes": [ "absl-py", "astor", "click", "flask", "gast", "grpcio", "itsdangerous", "jinja2", "markdown", "markupsafe", "numpy", "protobuf", "six", "tensorboard", "tensorflow", "termcolor", "werkzeug", "wheel", ], }, }
def test_from_final_state(self, context: Context) -> None: """Test instantiating product from a final state.""" state = State( score=0.5, resolved_dependencies={ "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.17.4", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, advised_runtime_environment=RuntimeEnvironment.from_dict({"python_version": "3.6"}), ) state.add_justification(self.JUSTIFICATION_SAMPLE_1) pypi = Source("https://pypi.org/simple") pv_daiquiri_locked = PackageVersion(name="daiquiri", version="==1.6.0", index=pypi, develop=False) pv_numpy_locked = PackageVersion(name="numpy", version="==1.17.4", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_daiquiri_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "daiquiri", "1.6.0", "https://pypi.org/simple" ).and_return(["000"]).ordered() context.should_receive("get_package_version").with_args( ("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.17.4", "https://pypi.org/simple" ).and_return(["111"]).ordered() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["222"]).ordered() pv_daiquiri = PackageVersion(name="daiquiri", version="*", index=pypi, develop=False) pv_tensorflow = PackageVersion(name="tensorflow", version=">=2.0.0", index=pypi, develop=False) project = flexmock( pipfile=Pipfile.from_string(self._PIPFILE), runtime_environment=RuntimeEnvironment.from_dict({"operating_system": {"name": "rhel"}}), ) project.should_receive("iter_dependencies").with_args(with_devel=True).and_return( [pv_daiquiri, pv_tensorflow] ).once() context.project = project context.dependencies = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": {("numpy", "1.17.4", "https://pypi.org/simple"): set()}, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): {("numpy", "1.17.4", "https://pypi.org/simple")} }, } context.dependents = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): { ( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ) } }, "tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()}, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.17.4", os_name="fedora", os_version="31", python_version="3.7", marker_evaluation_result=True, ).and_return("python_version >= '3.7'").once() assert "THOTH_ADVISER_METADATA" not in os.environ metadata_justification = {"thoth.adviser": {"justification": [{"bar": "baz"}]}} os.environ["THOTH_ADVISER_METADATA"] = json.dumps(metadata_justification) try: product = Product.from_final_state(state=state, context=context) finally: os.environ.pop("THOTH_ADVISER_METADATA") assert product.score == state.score assert product.justification == list( chain(metadata_justification["thoth.adviser"]["justification"], state.justification) ) assert product.advised_runtime_environment == state.advised_runtime_environment assert product.project.to_dict() == { "constraints": [], "requirements": { "packages": { "daiquiri": {"index": "pypi-org-simple", "version": "*"}, "tensorflow": {"index": "pypi-org-simple", "version": ">=2.0.0"}, }, "dev-packages": {}, "requires": {"python_version": "3.7"}, "source": [ { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", }, { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org-simple", }, ], "thoth": { "allow_prereleases": {"black": True}, }, }, "requirements_locked": { "_meta": { "sources": [ {"name": "pypi-org", "url": "https://pypi.org/simple", "verify_ssl": True}, { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org-simple", }, ], "requires": {"python_version": "3.7"}, "hash": {"sha256": "6cc8365e799b949fb6cc564cea2d8e0e8a782ab676a006e65abbe14621b93381"}, "pipfile-spec": 6, }, "default": { "daiquiri": { "version": "==1.6.0", "hashes": ["sha256:000"], "index": "pypi-org-simple", }, "numpy": { "version": "==1.17.4", "hashes": ["sha256:111"], "index": "pypi-org-simple", "markers": "python_version >= '3.7'", }, "tensorflow": { "version": "==2.0.0", "hashes": ["sha256:222"], "index": "pypi-org-simple", }, }, "develop": {}, }, "runtime_environment": { "hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None}, "operating_system": {"name": "rhel", "version": None}, "python_version": None, "cuda_version": None, "labels": None, "cudnn_version": None, "name": None, "platform": None, "base_image": None, "mkl_version": None, "openblas_version": None, "openmpi_version": None, "recommendation_type": None, }, }
def test_from_final_state(self, context: Context) -> None: """Test instantiating product from a final state.""" state = State( score=0.5, resolved_dependencies=OrderedDict( { "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.17.4", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), } ), unresolved_dependencies=OrderedDict(), advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"} ), ) state.add_justification([{"foo": "bar"}]) pypi = Source("https://pypi.org/simple") pv_daiquiri_locked = PackageVersion( name="daiquiri", version="==1.6.0", index=pypi, develop=False ) pv_numpy_locked = PackageVersion( name="numpy", version="==1.17.4", index=pypi, develop=False ) pv_tensorflow_locked = PackageVersion( name="tensorflow", version="==2.0.0", index=pypi, develop=False ) context.should_receive("get_package_version").with_args( ("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_daiquiri_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "daiquiri", "1.6.0", "https://pypi.org/simple" ).and_return(["000"]).ordered() context.should_receive("get_package_version").with_args( ("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.17.4", "https://pypi.org/simple" ).and_return(["111"]).ordered() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).ordered() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["222"]).ordered() pv_daiquiri = PackageVersion( name="daiquiri", version="*", index=pypi, develop=False ) pv_tensorflow = PackageVersion( name="tensorflow", version=">=2.0.0", index=pypi, develop=False ) project = flexmock() project.should_receive("iter_dependencies").with_args( with_devel=True ).and_return([pv_daiquiri, pv_tensorflow]).once() context.project = project context.dependencies = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): set() }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): { ("numpy", "1.17.4", "https://pypi.org/simple") } }, } context.dependents = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): { (("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7") } }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): set() }, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.17.4", os_name="fedora", os_version="31", python_version="3.7", ).and_return("python_version >= '3.7'").once() product = Product.from_final_state(state=state, context=context) assert product.score == state.score assert product.justification == state.justification assert product.advised_runtime_environment == state.advised_runtime_environment assert product.project.to_dict() == { "requirements": { "packages": { "daiquiri": {"index": "pypi-org", "version": "*"}, "tensorflow": {"index": "pypi-org", "version": ">=2.0.0"}, }, "dev-packages": {}, "source": [ { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", } ], }, "requirements_locked": { "_meta": { "sources": [ { "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", } ], "requires": {}, "hash": { "sha256": "c3a2f42932b6e5cd30f5664b11eda605f5fbd672f1b88729561d0d3edd10b5d9" }, "pipfile-spec": 6, }, "default": { "daiquiri": { "version": "==1.6.0", "hashes": ["sha256:000"], "index": "pypi-org", }, "numpy": { "version": "==1.17.4", "hashes": ["sha256:111"], "index": "pypi-org", "markers": "python_version >= '3.7'", }, "tensorflow": { "version": "==2.0.0", "hashes": ["sha256:222"], "index": "pypi-org", }, }, "develop": {}, }, "runtime_environment": { "hardware": {"cpu_family": None, "cpu_model": None}, "operating_system": {"name": None, "version": None}, "python_version": None, "cuda_version": None, "name": None, }, }
def test_environment_markers(self, context: Context) -> None: """Test handling of environment markers across multiple runs.""" state = State( score=0.0, resolved_dependencies=OrderedDict( { "numpy": ("numpy", "1.0.0", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), } ), unresolved_dependencies=OrderedDict(), ) context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.0.0", "https://pypi.org/simple" ).and_return(["000"]).once() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["111"]).once() pypi = Source("https://pypi.org/simple") pv_numpy_locked = PackageVersion( name="numpy", version="==1.0.0", index=pypi, develop=False ) pv_tensorflow_locked = PackageVersion( name="tensorflow", version="==2.0.0", index=pypi, develop=False ) context.should_receive("get_package_version").with_args( ("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).twice() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).twice() context.dependents = { "numpy": { ("numpy", "1.0.0", "https://pypi.org/simple"): { ( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ) } }, "tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()}, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.0.0", os_name="fedora", os_version="31", python_version="3.7", ).and_return("python_version >= '3.7'").and_return( "python_version >= '3' or 1" ).twice() product = Product.from_final_state(context=context, state=state) expected = { "advised_runtime_environment": None, "justification": [], "project": { "requirements": { "dev-packages": {}, "packages": {"flask": "*", "tensorflow": "==1.9.0"}, "requires": {"python_version": "3.6"}, "source": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True, }, { "name": "pypi-org", "url": "https://pypi.org/simple", "verify_ssl": True, }, ], }, "requirements_locked": { "_meta": { "hash": { "sha256": "e55b6bbaba9467f1629c34e7a4180a6a2d82df37e02e762866e7aac27ced0f99" }, "pipfile-spec": 6, "requires": {"python_version": "3.6"}, "sources": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True, }, { "name": "pypi-org", "url": "https://pypi.org/simple", "verify_ssl": True, }, ], }, "default": { "numpy": { "hashes": ["sha256:000"], "index": "pypi-org", "markers": "python_version >= '3.7'", "version": "==1.0.0", }, "tensorflow": { "hashes": ["sha256:111"], "index": "pypi-org", "version": "==2.0.0", }, }, "develop": {}, }, "runtime_environment": { "cuda_version": None, "hardware": {"cpu_family": None, "cpu_model": None}, "name": None, "operating_system": {"name": None, "version": None}, "python_version": None, "platform": None, }, }, "score": 0.0, } assert product.to_dict() == expected # Markers should not intersect. product = Product.from_final_state(context=context, state=state) expected["project"]["requirements_locked"]["default"]["numpy"][ "markers" ] = "python_version >= '3' or 1" assert product.to_dict() == expected
def test_from_final_state(self, context: Context) -> None: """Test instantiating product from a final state.""" state = State( score=0.5, resolved_dependencies={ "daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.17.4", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, advised_runtime_environment=RuntimeEnvironment.from_dict( {"python_version": "3.6"}), ) state.add_justification(self.JUSTIFICATION_SAMPLE_1) pypi = Source("https://pypi.org/simple") pv_daiquiri_locked = PackageVersion(name="daiquiri", version="==1.6.0", index=pypi, develop=False) pv_numpy_locked = PackageVersion(name="numpy", version="==1.17.4", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False).and_return(pv_daiquiri_locked).ordered() context.graph.should_receive( "get_python_package_hashes_sha256").with_args( "daiquiri", "1.6.0", "https://pypi.org/simple").and_return(["000"]).ordered() context.should_receive("get_package_version").with_args( ("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False).and_return(pv_numpy_locked).ordered() context.graph.should_receive( "get_python_package_hashes_sha256").with_args( "numpy", "1.17.4", "https://pypi.org/simple").and_return(["111"]).ordered() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False).and_return(pv_tensorflow_locked).ordered() context.graph.should_receive( "get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple").and_return(["222"]).ordered() pv_daiquiri = PackageVersion(name="daiquiri", version="*", index=pypi, develop=False) pv_tensorflow = PackageVersion(name="tensorflow", version=">=2.0.0", index=pypi, develop=False) project = flexmock( pipfile=Pipfile.from_string(self._PIPFILE), runtime_environment=RuntimeEnvironment.from_dict( {"operating_system": { "name": "rhel" }}), ) project.should_receive("iter_dependencies").with_args( with_devel=True).and_return([pv_daiquiri, pv_tensorflow]).once() context.project = project context.dependencies = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): set() }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): {("numpy", "1.17.4", "https://pypi.org/simple")} }, } context.dependents = { "daiquiri": { ("daiquiri", "1.6.0", "https://pypi.org/simple"): set(), }, "numpy": { ("numpy", "1.17.4", "https://pypi.org/simple"): {( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", )} }, "tensorflow": { ("tensorflow", "2.0.0", "https://pypi.org/simple"): set() }, } context.graph.should_receive( "get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.17.4", os_name="fedora", os_version="31", python_version="3.7", ).and_return("python_version >= '3.7'").once() product = Product.from_final_state(state=state, context=context) assert product.score == state.score assert product.justification == state.justification assert product.advised_runtime_environment == state.advised_runtime_environment assert product.project.to_dict() == { "requirements": { "packages": { "daiquiri": { "index": "pypi-org", "version": "*" }, "tensorflow": { "index": "pypi-org", "version": ">=2.0.0" }, }, "dev-packages": {}, "requires": { "python_version": "3.7" }, "source": [{ "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", }], }, "requirements_locked": { "_meta": { "sources": [{ "url": "https://pypi.org/simple", "verify_ssl": True, "name": "pypi-org", }], "requires": { "python_version": "3.7" }, "hash": { "sha256": "f08689732b596fd705a45bbf9ec44c3995b17a1aa6392c46500aeb736c4d4e88" }, "pipfile-spec": 6, }, "default": { "daiquiri": { "version": "==1.6.0", "hashes": ["sha256:000"], "index": "pypi-org", }, "numpy": { "version": "==1.17.4", "hashes": ["sha256:111"], "index": "pypi-org", "markers": "python_version >= '3.7'", }, "tensorflow": { "version": "==2.0.0", "hashes": ["sha256:222"], "index": "pypi-org", }, }, "develop": {}, }, "runtime_environment": { "hardware": { "cpu_family": None, "cpu_model": None }, "operating_system": { "name": "rhel", "version": None }, "python_version": None, "cuda_version": None, "name": None, "platform": None, }, }
def test_environment_markers_shared(self, context: Context) -> None: """Test handling of environment markers when multiple dependencies share one.""" state = State( score=0.0, resolved_dependencies={ "pandas": ("pandas", "1.0.0", "https://pypi.org/simple"), "numpy": ("numpy", "1.0.0", "https://pypi.org/simple"), "tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"), }, unresolved_dependencies={}, ) context.graph.should_receive("get_python_package_hashes_sha256").with_args( "numpy", "1.0.0", "https://pypi.org/simple" ).and_return(["000"]).once() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple" ).and_return(["111"]).once() context.graph.should_receive("get_python_package_hashes_sha256").with_args( "pandas", "1.0.0", "https://pypi.org/simple" ).and_return(["222"]).once() pypi = Source("https://pypi.org/simple") pv_numpy_locked = PackageVersion(name="numpy", version="==1.0.0", index=pypi, develop=False) pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False) pv_pandas_locked = PackageVersion(name="pandas", version="==1.0.0", index=pypi, develop=False) context.should_receive("get_package_version").with_args( ("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_numpy_locked).once() context.should_receive("get_package_version").with_args( ("pandas", "1.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_pandas_locked).once() context.should_receive("get_package_version").with_args( ("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False ).and_return(pv_tensorflow_locked).once() context.dependents = { "numpy": { ("numpy", "1.0.0", "https://pypi.org/simple"): [ # set to list for reproducible runs. ( ("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ), ( ("pandas", "1.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7", ), ] }, "tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()}, "pandas": {("pandas", "1.0.0", "https://pypi.org/simple"): set()}, } context.graph.should_receive("get_python_environment_marker").with_args( "tensorflow", "2.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.0.0", os_name="fedora", os_version="31", python_version="3.7", ).and_return("python_version >= '3.8'").once() context.graph.should_receive("get_python_environment_marker").with_args( "pandas", "1.0.0", "https://pypi.org/simple", dependency_name="numpy", dependency_version="1.0.0", os_name="fedora", os_version="31", python_version="3.7", ).and_return(None).once() product = Product.from_final_state(context=context, state=state) expected = { "advised_manifest_changes": [], "advised_runtime_environment": None, "justification": [], "project": { "constraints": [], "requirements": { "dev-packages": {}, "packages": {"flask": "*", "tensorflow": "==1.9.0"}, "requires": {"python_version": "3.6"}, "source": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, {"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True}, ], "thoth": { "allow_prereleases": {}, "disable_index_adjustment": False, }, }, "requirements_locked": { "_meta": { "hash": {"sha256": "4628b328465fa6946ca9abf9c3576fb502436d0a40300d798058677de0f6128a"}, "pipfile-spec": 6, "requires": {"python_version": "3.6"}, "sources": [ {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}, {"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True}, ], }, "default": { "numpy": {"hashes": ["sha256:000"], "index": "pypi-org-simple", "version": "==1.0.0"}, "pandas": {"hashes": ["sha256:222"], "index": "pypi-org-simple", "version": "==1.0.0"}, "tensorflow": {"hashes": ["sha256:111"], "index": "pypi-org-simple", "version": "==2.0.0"}, }, "develop": {}, }, "runtime_environment": { "base_image": None, "cuda_version": None, "cudnn_version": None, "hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None}, "name": None, "operating_system": {"name": None, "version": None}, "platform": None, "mkl_version": None, "openmpi_version": None, "openblas_version": None, "python_version": None, "recommendation_type": None, }, }, "score": 0.0, } assert product.to_dict() == expected