def test_run(self) -> None:
"""Test running this stride to filter same stacks."""
context = flexmock()
state = State()
state.add_resolved_dependency(
("tensorflow", "2.2.0", "https://pypi.org/simple"))
unit = UniqueStackStride()
with unit.assigned_context(context):
assert unit.run(state) is None
# Running the stride on the same stack should cause rejecting it.
with pytest.raises(NotAcceptable):
unit.run(state)
# A stack with another package should be included.
state.add_resolved_dependency(
("numpy", "1.19.1", "https://pypi.org/simple"))
assert unit.run(state) is None
def test_run_package_version_from_with_resolved(
self,
context: Context,
state: State,
package_version_from_version: str,
package_version_from_index: str,
package_version_from_develop: str,
resolved_version: str,
resolved_index: str,
resolved_develop: str,
add_resolved: bool,
pipeline_run: bool,
) -> None:
"""Test running the prescription based on the dependency introduced."""
prescription_str = f"""
name: SkipPackageStep
type: step.SkipPackage
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
state:
package_version_from:
- name: scikit-learn
version: "{package_version_from_version}"
develop: {package_version_from_develop}
index_url: {package_version_from_index}
resolved_dependencies:
- name: click
version: "{resolved_version}"
develop: {resolved_develop}
index_url: {resolved_index}
run:
stack_info:
- type: WARNING
message: A warning message
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_SKIP_PACKAGE_STEP_SCHEMA(prescription)
SkipPackageStepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
package_version_resolved = PackageVersion(
name="click",
version="==8.0.0",
index=pypi,
develop=False,
)
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
if add_resolved:
state.add_resolved_dependency(package_version_resolved.to_tuple())
context.register_package_version(package_version_resolved)
runtime_env = context.project.runtime_environment
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
context.stack_info.clear()
unit = SkipPackageStepPrescription()
unit.pre_run()
with unit.assigned_context(context):
if pipeline_run:
with pytest.raises(SkipPackage):
unit.run(state, package_version)
# Run one more time to verify the stack info is added just once.
with pytest.raises(SkipPackage):
unit.run(state, package_version)
else:
assert unit.run(state, package_version) is None
if pipeline_run:
assert self.verify_justification_schema(context.stack_info)
assert len(context.stack_info) == 1
assert context.stack_info == [{
"type": "WARNING",
"link": "https://thoth-station.ninja",
"message": "A warning message"
}]
else:
assert not context.stack_info
def run(self, state: State, package_version: PackageVersion) -> None:
"""Run main entry-point for steps to skip packages."""
if not self._index_url_check(self._index_url, package_version.index.url):
return None
if self._specifier and package_version.locked_version not in self._specifier:
return None
if self._develop is not None and package_version.develop != self._develop:
return None
if not self._run_state_with_initiator(state, package_version):
return None
add_package_version = self.run_prescription["package_version"]
add_package_version_name = add_package_version["name"]
add_package_version_version = add_package_version["locked_version"][2:]
add_package_version_index_url = add_package_version["index_url"]
add_package_version_develop = add_package_version["develop"]
add_package_version_tuple = (
add_package_version_name,
add_package_version_version,
add_package_version_index_url,
)
resolved = state.resolved_dependencies.get(add_package_version_name)
if resolved:
if resolved == add_package_version_tuple:
_LOGGER.debug(
"%s: Not adding package %r as it is already in the resolved listing",
self.get_unit_name(),
add_package_version_tuple,
)
else:
_LOGGER.debug(
"%s: Not adding package %r as another package %r is already present in the resolved listing",
self.get_unit_name(),
add_package_version_tuple,
resolved,
)
return None
runtime_env = self.context.project.runtime_environment
py_ver = runtime_env.python_version.replace(".", "")
# XXX: this could be moved to thoth-common
solver_name = f"solver-{runtime_env.operating_system.name}-{runtime_env.operating_system.version}-py{py_ver}"
if not self.context.graph.python_package_version_exists(
add_package_version_name,
add_package_version_version,
add_package_version_index_url,
solver_name=solver_name,
):
_LOGGER.debug(
"%s: Not adding package %r as the given package was not solved by %r",
self.get_unit_name(),
add_package_version_tuple,
solver_name,
)
return None
try:
if not self.context.graph.is_python_package_index_enabled(add_package_version_index_url):
_LOGGER.debug(
"%s: Not adding package %r as index %r is not enabled",
self.get_unit_name(),
add_package_version_tuple,
add_package_version_index_url,
)
return None
except NotFoundError:
_LOGGER.debug(
"%s: Not adding package %r as index %r is not known to the resolver",
self.get_unit_name(),
add_package_version_tuple,
add_package_version_index_url,
)
return None
try:
self._run_base()
finally:
self._configuration["prescription"]["run"] = True
pv = PackageVersion(
name=add_package_version_name,
version=add_package_version["locked_version"],
index=Source(add_package_version_index_url),
develop=add_package_version_develop,
)
self.context.register_package_version(pv)
state.add_unresolved_dependency(add_package_version_tuple)
def test_environment_markers_shared(self, context: Context) -> None:
"""Test handling of environment markers when multiple dependencies share one."""
state = State(
score=0.0,
resolved_dependencies={
"pandas": ("pandas", "1.0.0", "https://pypi.org/simple"),
"numpy": ("numpy", "1.0.0", "https://pypi.org/simple"),
"tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"),
},
unresolved_dependencies={},
justification=[{"type": "INFO", "message": "Foo bar", "link": "https://thoth-station.ninja"}],
)
# Make sure tested packages are not direct dependencies. In such cases the behaviour is
# different, see other tests.
context.project.pipfile.packages.packages.pop("pandas", None)
context.project.pipfile.packages.packages.pop("numpy", None)
context.project.pipfile.packages.packages.pop("tensorflow", None)
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"numpy", "1.0.0", "https://pypi.org/simple"
).and_return(["000"]).once()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"tensorflow", "2.0.0", "https://pypi.org/simple"
).and_return(["111"]).once()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"pandas", "1.0.0", "https://pypi.org/simple"
).and_return(["222"]).once()
pypi = Source("https://pypi.org/simple")
pv_numpy_locked = PackageVersion(name="numpy", version="==1.0.0", index=pypi, develop=False)
pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False)
pv_pandas_locked = PackageVersion(name="pandas", version="==1.0.0", index=pypi, develop=False)
context.should_receive("get_package_version").with_args(
("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_numpy_locked).once()
context.should_receive("get_package_version").with_args(
("pandas", "1.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_pandas_locked).once()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_tensorflow_locked).once()
context.dependents = {
"numpy": {
("numpy", "1.0.0", "https://pypi.org/simple"): [ # set to list for reproducible runs.
(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
),
(
("pandas", "1.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
),
]
},
"tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()},
"pandas": {("pandas", "1.0.0", "https://pypi.org/simple"): set()},
}
context.graph.should_receive("get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.0.0",
os_name="fedora",
os_version="31",
python_version="3.7",
marker_evaluation_result=True,
).and_return("python_version >= '3.8'").once()
context.graph.should_receive("get_python_environment_marker").with_args(
"pandas",
"1.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.0.0",
os_name="fedora",
os_version="31",
python_version="3.7",
marker_evaluation_result=True,
).and_return(None).once()
product = Product.from_final_state(context=context, state=state)
expected = {
"advised_manifest_changes": [],
"advised_runtime_environment": None,
"dependency_graph": {"edges": [], "nodes": ["pandas", "numpy", "tensorflow"]},
"justification": [
{
"link": "https://thoth-station.ninja",
"message": "Foo bar",
"type": "INFO",
}
],
"project": {
"constraints": [],
"requirements": {
"dev-packages": {},
"packages": {"flask": "*"},
"requires": {"python_version": "3.6"},
"source": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True},
],
"thoth": {
"allow_prereleases": {},
"disable_index_adjustment": False,
},
},
"requirements_locked": {
"_meta": {
"hash": {"sha256": "2e49395dfa87159358e581bd22e656c27c0dab04894d1b137a14f85bb387ea51"},
"pipfile-spec": 6,
"requires": {"python_version": "3.6"},
"sources": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True},
],
},
"default": {
"numpy": {"hashes": ["sha256:000"], "index": "pypi-org-simple", "version": "==1.0.0"},
"pandas": {"hashes": ["sha256:222"], "index": "pypi-org-simple", "version": "==1.0.0"},
"tensorflow": {"hashes": ["sha256:111"], "index": "pypi-org-simple", "version": "==2.0.0"},
},
"develop": {},
},
"runtime_environment": {
"base_image": None,
"cuda_version": None,
"cudnn_version": None,
"hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None},
"name": None,
"operating_system": {"name": None, "version": None},
"labels": None,
"platform": None,
"mkl_version": None,
"openmpi_version": None,
"openblas_version": None,
"python_version": None,
"recommendation_type": None,
},
},
"score": 0.0,
}
assert product.to_dict() == expected
def test_from_final_state(self, context: Context) -> None:
"""Test instantiating product from a final state."""
state = State(
score=0.5,
resolved_dependencies={
"daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"),
"numpy": ("numpy", "1.17.4", "https://pypi.org/simple"),
"tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"),
},
unresolved_dependencies={},
advised_runtime_environment=RuntimeEnvironment.from_dict({"python_version": "3.6"}),
)
state.add_justification(self.JUSTIFICATION_SAMPLE_1)
pypi = Source("https://pypi.org/simple")
pv_daiquiri_locked = PackageVersion(name="daiquiri", version="==1.6.0", index=pypi, develop=False)
pv_numpy_locked = PackageVersion(name="numpy", version="==1.17.4", index=pypi, develop=False)
pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False)
context.should_receive("get_package_version").with_args(
("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_daiquiri_locked).ordered()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"daiquiri", "1.6.0", "https://pypi.org/simple"
).and_return(["000"]).ordered()
context.should_receive("get_package_version").with_args(
("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False
).and_return(pv_numpy_locked).ordered()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"numpy", "1.17.4", "https://pypi.org/simple"
).and_return(["111"]).ordered()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_tensorflow_locked).ordered()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"tensorflow", "2.0.0", "https://pypi.org/simple"
).and_return(["222"]).ordered()
pv_daiquiri = PackageVersion(name="daiquiri", version="*", index=pypi, develop=False)
pv_tensorflow = PackageVersion(name="tensorflow", version=">=2.0.0", index=pypi, develop=False)
project = flexmock(
pipfile=Pipfile.from_string(self._PIPFILE),
runtime_environment=RuntimeEnvironment.from_dict({"operating_system": {"name": "rhel"}}),
)
project.should_receive("iter_dependencies").with_args(with_devel=True).and_return(
[pv_daiquiri, pv_tensorflow]
).once()
context.project = project
context.dependencies = {
"daiquiri": {
("daiquiri", "1.6.0", "https://pypi.org/simple"): set(),
},
"numpy": {("numpy", "1.17.4", "https://pypi.org/simple"): set()},
"tensorflow": {
("tensorflow", "2.0.0", "https://pypi.org/simple"): {("numpy", "1.17.4", "https://pypi.org/simple")}
},
}
context.dependents = {
"daiquiri": {
("daiquiri", "1.6.0", "https://pypi.org/simple"): set(),
},
"numpy": {
("numpy", "1.17.4", "https://pypi.org/simple"): {
(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
)
}
},
"tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()},
}
context.graph.should_receive("get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.17.4",
os_name="fedora",
os_version="31",
python_version="3.7",
marker_evaluation_result=True,
).and_return("python_version >= '3.7'").once()
assert "THOTH_ADVISER_METADATA" not in os.environ
metadata_justification = {"thoth.adviser": {"justification": [{"bar": "baz"}]}}
os.environ["THOTH_ADVISER_METADATA"] = json.dumps(metadata_justification)
try:
product = Product.from_final_state(state=state, context=context)
finally:
os.environ.pop("THOTH_ADVISER_METADATA")
assert product.score == state.score
assert product.justification == list(
chain(metadata_justification["thoth.adviser"]["justification"], state.justification)
)
assert product.advised_runtime_environment == state.advised_runtime_environment
assert product.project.to_dict() == {
"constraints": [],
"requirements": {
"packages": {
"daiquiri": {"index": "pypi-org-simple", "version": "*"},
"tensorflow": {"index": "pypi-org-simple", "version": ">=2.0.0"},
},
"dev-packages": {},
"requires": {"python_version": "3.7"},
"source": [
{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org",
},
{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org-simple",
},
],
"thoth": {
"allow_prereleases": {"black": True},
},
},
"requirements_locked": {
"_meta": {
"sources": [
{"name": "pypi-org", "url": "https://pypi.org/simple", "verify_ssl": True},
{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org-simple",
},
],
"requires": {"python_version": "3.7"},
"hash": {"sha256": "6cc8365e799b949fb6cc564cea2d8e0e8a782ab676a006e65abbe14621b93381"},
"pipfile-spec": 6,
},
"default": {
"daiquiri": {
"version": "==1.6.0",
"hashes": ["sha256:000"],
"index": "pypi-org-simple",
},
"numpy": {
"version": "==1.17.4",
"hashes": ["sha256:111"],
"index": "pypi-org-simple",
"markers": "python_version >= '3.7'",
},
"tensorflow": {
"version": "==2.0.0",
"hashes": ["sha256:222"],
"index": "pypi-org-simple",
},
},
"develop": {},
},
"runtime_environment": {
"hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None},
"operating_system": {"name": "rhel", "version": None},
"python_version": None,
"cuda_version": None,
"labels": None,
"cudnn_version": None,
"name": None,
"platform": None,
"base_image": None,
"mkl_version": None,
"openblas_version": None,
"openmpi_version": None,
"recommendation_type": None,
},
}
def test_run_no_only_once(self) -> None:
"""Test running this stride with only-once pipeline flag set."""
unit = OneVersionStride()
unit.update_configuration({"package_name": "tensorflow", "only_once": False})
unit.pre_run()
state1 = State(score=1.0)
state1.add_resolved_dependency(("tensorflow", "2.0.0", "https://pypi.org/simple"))
assert unit.run(state1) is None
state2 = State(score=1.0)
state2.add_resolved_dependency(("tensorflow", "2.0.0", "https://pypi.org/simple"))
assert unit.run(state2) is None
state3 = State(score=1.0)
state3.add_resolved_dependency(("tensorflow", "2.1.0", "https://pypi.org/simple"))
with pytest.raises(NotAcceptable):
unit.run(state3)
def test_run_add_justification(self) -> None:
"""Test adding information Intel's MKL environment variable."""
state = State()
state.add_resolved_dependency(
("pytorch", "1.4.0", "https://pypi.org/simple"))
assert len(state.advised_manifest_changes) == 0
assert not state.justification
unit = MKLThreadsWrap()
unit.run(state)
assert len(state.justification) == 1
assert set(
state.justification[0].keys()) == {"type", "message", "link"}
assert state.justification[0]["type"] == "WARNING"
assert state.justification[0][
"link"], "Empty link to justification document provided"
assert len(state.advised_manifest_changes) == 1
assert state.advised_manifest_changes[0] == [{
"apiVersion": "apps.openshift.io/v1",
"kind": "DeploymentConfig",
"patch": {
"op": "add",
"path": "/spec/template/spec/containers/0/env/0",
"value": {
"name": "OMP_NUM_THREADS",
"value": "1"
},
},
}]
patch = jsonpatch.JsonPatch(
obj["patch"] for obj in state.advised_manifest_changes[0])
deployment_config = yaml.safe_load(self._DEPLOYMENT_CONFIG)
assert jsonpatch.apply_patch(deployment_config, patch) == {
"apiVersion": "apps.openshift.io/v1",
"kind": "DeploymentConfig",
"metadata": {
"name": "foo",
"namespace": "some-namespace"
},
"spec": {
"replicas": 1,
"selector": {
"service": "foo"
},
"template": {
"spec": {
"containers": [{
"env": [
{
"name": "OMP_NUM_THREADS",
"value": "1"
},
{
"name": "APP_MODULE",
"value": "foo"
},
],
"image":
"foo:latest",
}]
}
},
},
}
def test_run_package_version_from_with_resolved(
self,
context: Context,
state: State,
package_version_from_version: str,
package_version_from_index: str,
package_version_from_develop: str,
resolved_version: str,
resolved_index: str,
resolved_develop: str,
add_resolved: bool,
pipeline_run: bool,
) -> None:
"""Test running the prescription based on the dependency introduced."""
prescription_str = f"""
name: StepUnit
type: step
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
state:
package_version_from:
- name: scikit-learn
version: "{package_version_from_version}"
develop: {package_version_from_develop}
index_url: {package_version_from_index}
resolved_dependencies:
- name: click
version: "{resolved_version}"
develop: {resolved_develop}
index_url: {resolved_index}
run:
score: 0.5
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STEP_SCHEMA(prescription)
StepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
package_version_resolved = PackageVersion(
name="click",
version="==8.0.0",
index=pypi,
develop=False,
)
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
if add_resolved:
state.add_resolved_dependency(package_version_resolved.to_tuple())
context.register_package_version(package_version_resolved)
runtime_env = context.project.runtime_environment
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
unit = StepPrescription()
unit.pre_run()
with unit.assigned_context(context):
result = unit.run(state, package_version)
if pipeline_run:
assert isinstance(result, tuple)
assert result[0] == 0.5
assert result[1] is None
else:
assert result is None
def test_add_state_order_multi(self) -> None:
"""Test adding states to beam and order during addition when score is same."""
beam = Beam(width=2)
state01 = State(score=0.0, iteration=0, iteration_states_added=0)
state01.add_justification([{"state": "01"}])
beam.add_state(state01)
state02 = State(score=0.0, iteration=0, iteration_states_added=1)
state02.add_justification([{"state": "02"}])
beam.add_state(state02)
state03 = State(score=0.0, iteration=1, iteration_states_added=0)
state03.add_justification([{"state": "03"}])
beam.add_state(state03)
state04 = State(score=0.0, iteration=1, iteration_states_added=1)
state04.add_justification([{"state": "04"}])
beam.add_state(state04)
assert list(beam.iter_states_sorted()) == [state03, state04]
assert list(beam.iter_states_sorted(reverse=True)) == [state03, state04]
assert list(beam.iter_states_sorted(reverse=False)) == [state04, state03]
def test_run_package_version_from_with_resolved(
self,
context: Context,
state: State,
package_version_from_version: str,
package_version_from_index: str,
package_version_from_develop: str,
resolved_version: str,
resolved_index: str,
resolved_develop: str,
add_resolved: bool,
pipeline_run: bool,
) -> None:
"""Test running the add package unit."""
prescription_str = f"""
name: AddPackageStep
type: step.AddPackage
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
state:
package_version_from:
- name: scikit-learn
version: "{package_version_from_version}"
develop: {package_version_from_develop}
index_url: {package_version_from_index}
resolved_dependencies:
- name: click
version: "{resolved_version}"
develop: {resolved_develop}
index_url: {resolved_index}
run:
package_version:
name: daiquiri
locked_version: ==2.0.0
index_url: https://pypi.org/simple
develop: true
stack_info:
- type: INFO
message: "Hello, Thoth!"
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_ADD_PACKAGE_STEP_SCHEMA(prescription)
AddPackageStepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
package_version_resolved = PackageVersion(
name="click",
version="==8.0.0",
index=pypi,
develop=False,
)
assert "dauiqiri" not in state.resolved_dependencies
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
if add_resolved:
state.add_resolved_dependency(package_version_resolved.to_tuple())
context.register_package_version(package_version_resolved)
runtime_env = context.project.runtime_environment
runtime_env.operating_system.name = "rhel"
runtime_env.operating_system.version = "8"
runtime_env.python_version = "3.8"
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
pv_tuple = ("daiquiri", "2.0.0", "https://pypi.org/simple")
if pipeline_run:
context.graph.should_receive(
"python_package_version_exists").with_args(
*pv_tuple,
solver_name="solver-rhel-8-py38").and_return(True).once()
context.graph.should_receive(
"is_python_package_index_enabled").with_args(
"https://pypi.org/simple").and_return(True).once()
unit = AddPackageStepPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state, package_version) is None
if pipeline_run:
assert "daiquiri" in state.unresolved_dependencies
assert set(
state.unresolved_dependencies.get("daiquiri").values()) == {
pv_tuple
}
pv = context.get_package_version(pv_tuple, graceful=True)
assert pv is not None
assert pv.to_tuple() == pv_tuple
assert pv.develop is True
else:
assert "daiquiri" not in state.unresolved_dependencies
assert state.unresolved_dependencies.get("daiquiri") is None
def test_run_package_version_already_resolved_same_name(
self, context: Context, state: State) -> None:
"""Test running the prescription when the given package is already in the resolved state (same name)."""
prescription_str = """
name: AddPackageStep
type: step.AddPackage
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
version: "~=1.19.0"
state:
package_version_from:
- name: scikit-learn
version: "<=0.25.0"
develop: false
index_url: "https://pypi.org/simple"
resolved_dependencies:
- name: click
version: "==8.0.0"
develop: false
index_url: https://pypi.org/simple
run:
package_version:
name: daiquiri
locked_version: ==2.0.0
index_url: https://pypi.org/simple
develop: true
stack_info:
- type: INFO
message: "Hello, Thoth!"
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_ADD_PACKAGE_STEP_SCHEMA(prescription)
AddPackageStepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
package_version_resolved = PackageVersion(
name="click",
version="==8.0.0",
index=pypi,
develop=False,
)
pv_daiquiri = PackageVersion(
name="daiquiri",
version="==2.5.0",
index=pypi,
develop=False,
)
pv_tuple = ("daiquiri", "2.5.0", "https://pypi.org/simple")
state.add_resolved_dependency(pv_tuple)
context.register_package_version(pv_daiquiri)
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
state.add_resolved_dependency(package_version_resolved.to_tuple())
context.register_package_version(package_version_resolved)
runtime_env = context.project.runtime_environment
runtime_env.operating_system.name = "rhel"
runtime_env.operating_system.version = "8"
runtime_env.python_version = "3.8"
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
unit = AddPackageStepPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state, package_version) is None
assert "daiquiri" not in state.unresolved_dependencies
assert "daiquiri" in state.resolved_dependencies
assert set(state.resolved_dependencies["daiquiri"]) == set(pv_tuple)
def test_run_package_version_index_url_not_known(self, context: Context,
state: State) -> None:
"""Test running the prescription based on the dependency introduced when index_url is not known."""
prescription_str = """
name: AddPackageStep
type: step.AddPackage
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
version: "~=1.19.0"
state:
package_version_from:
- name: scikit-learn
version: "<=0.25.0"
develop: false
index_url: "https://pypi.org/simple"
resolved_dependencies:
- name: click
version: "==8.0.0"
develop: false
index_url: https://pypi.org/simple
run:
package_version:
name: daiquiri
locked_version: ==2.0.0
index_url: https://pypi.org/simple
develop: true
stack_info:
- type: INFO
message: "Hello, Thoth!"
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_ADD_PACKAGE_STEP_SCHEMA(prescription)
AddPackageStepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
package_version_resolved = PackageVersion(
name="click",
version="==8.0.0",
index=pypi,
develop=False,
)
assert "dauiqiri" not in state.resolved_dependencies
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
state.add_resolved_dependency(package_version_resolved.to_tuple())
context.register_package_version(package_version_resolved)
runtime_env = context.project.runtime_environment
runtime_env.operating_system.name = "rhel"
runtime_env.operating_system.version = "8"
runtime_env.python_version = "3.8"
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
pv_tuple = ("daiquiri", "2.0.0", "https://pypi.org/simple")
context.graph.should_receive(
"python_package_version_exists").with_args(
*pv_tuple,
solver_name="solver-rhel-8-py38").and_return(True).once()
context.graph.should_receive(
"is_python_package_index_enabled").with_args(
"https://pypi.org/simple").and_raise(NotFoundError).once()
unit = AddPackageStepPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state, package_version) is None
assert "daiquiri" not in state.unresolved_dependencies
assert state.unresolved_dependencies.get("daiquiri") is None
def test_environment_markers_shared(self, context: Context) -> None:
"""Test handling of environment markers when multiple dependencies share one."""
state = State(
score=0.0,
resolved_dependencies={
"pandas": ("pandas", "1.0.0", "https://pypi.org/simple"),
"numpy": ("numpy", "1.0.0", "https://pypi.org/simple"),
"tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"),
},
unresolved_dependencies={},
)
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"numpy", "1.0.0", "https://pypi.org/simple"
).and_return(["000"]).once()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"tensorflow", "2.0.0", "https://pypi.org/simple"
).and_return(["111"]).once()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"pandas", "1.0.0", "https://pypi.org/simple"
).and_return(["222"]).once()
pypi = Source("https://pypi.org/simple")
pv_numpy_locked = PackageVersion(name="numpy", version="==1.0.0", index=pypi, develop=False)
pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False)
pv_pandas_locked = PackageVersion(name="pandas", version="==1.0.0", index=pypi, develop=False)
context.should_receive("get_package_version").with_args(
("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_numpy_locked).once()
context.should_receive("get_package_version").with_args(
("pandas", "1.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_pandas_locked).once()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_tensorflow_locked).once()
context.dependents = {
"numpy": {
("numpy", "1.0.0", "https://pypi.org/simple"): [ # set to list for reproducible runs.
(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
),
(
("pandas", "1.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
),
]
},
"tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()},
"pandas": {("pandas", "1.0.0", "https://pypi.org/simple"): set()},
}
context.graph.should_receive("get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.0.0",
os_name="fedora",
os_version="31",
python_version="3.7",
).and_return("python_version >= '3.8'").once()
context.graph.should_receive("get_python_environment_marker").with_args(
"pandas",
"1.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.0.0",
os_name="fedora",
os_version="31",
python_version="3.7",
).and_return(None).once()
product = Product.from_final_state(context=context, state=state)
expected = {
"advised_manifest_changes": [],
"advised_runtime_environment": None,
"justification": [],
"project": {
"constraints": [],
"requirements": {
"dev-packages": {},
"packages": {"flask": "*", "tensorflow": "==1.9.0"},
"requires": {"python_version": "3.6"},
"source": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True},
],
"thoth": {
"allow_prereleases": {},
"disable_index_adjustment": False,
},
},
"requirements_locked": {
"_meta": {
"hash": {"sha256": "4628b328465fa6946ca9abf9c3576fb502436d0a40300d798058677de0f6128a"},
"pipfile-spec": 6,
"requires": {"python_version": "3.6"},
"sources": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True},
],
},
"default": {
"numpy": {"hashes": ["sha256:000"], "index": "pypi-org-simple", "version": "==1.0.0"},
"pandas": {"hashes": ["sha256:222"], "index": "pypi-org-simple", "version": "==1.0.0"},
"tensorflow": {"hashes": ["sha256:111"], "index": "pypi-org-simple", "version": "==2.0.0"},
},
"develop": {},
},
"runtime_environment": {
"base_image": None,
"cuda_version": None,
"cudnn_version": None,
"hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None},
"name": None,
"operating_system": {"name": None, "version": None},
"platform": None,
"mkl_version": None,
"openmpi_version": None,
"openblas_version": None,
"python_version": None,
"recommendation_type": None,
},
},
"score": 0.0,
}
assert product.to_dict() == expected
def test_run_package_version_from(
self,
context: Context,
state: State,
package_version_from_version: str,
package_version_from_index: str,
package_version_from_develop: str,
register_dependency: bool,
pipeline_run: bool,
) -> None:
"""Test running the prescription based on the dependency introduced."""
prescription_str = f"""
name: SkipPackageStep
type: step.SkipPackage
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
state:
package_version_from:
- name: scikit-learn
version: "{package_version_from_version}"
develop: {package_version_from_develop}
index_url: {package_version_from_index}
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_SKIP_PACKAGE_STEP_SCHEMA(prescription)
SkipPackageStepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
if register_dependency:
runtime_env = context.project.runtime_environment
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
unit = SkipPackageStepPrescription()
unit.pre_run()
with unit.assigned_context(context):
if pipeline_run:
with pytest.raises(SkipPackage):
unit.run(state, package_version)
else:
assert unit.run(state, package_version) is None
def test_add_state_order_multi(self) -> None:
"""Test adding states to beam and order during addition when score is same."""
beam = Beam(width=2)
state01 = State(score=0.0)
state01.add_justification(self.JUSTIFICATION_SAMPLE_1)
beam.add_state(state01)
state02 = State(score=0.0)
state02.add_justification(self.JUSTIFICATION_SAMPLE_2)
beam.add_state(state02)
state03 = State(score=0.0)
state03.add_justification(self.JUSTIFICATION_SAMPLE_3)
beam.add_state(state03)
assert list(beam.iter_states_sorted()) == [state01, state02]
assert list(beam.iter_states_sorted(reverse=True)) == [state01, state02]
assert list(beam.iter_states_sorted(reverse=False)) == [state01, state02]
def test_justification(self, final_state: State) -> None:
"""Test manipulation with state justification."""
assert final_state.justification == self.JUSTIFICATION_SAMPLE_1
final_state.add_justification(self.JUSTIFICATION_SAMPLE_2)
assert final_state.justification == list(
chain(self.JUSTIFICATION_SAMPLE_1, self.JUSTIFICATION_SAMPLE_2))
def test_run_state_not_index_url(self, context: Context,
state: State) -> None:
"""Test running the prescription if state matches."""
prescription_str = """
name: StepUnit
type: step
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
version: '==1.19.1'
index_url:
not: 'https://pypi.org/simple'
state:
resolved_dependencies:
- name: tensorflow
version: '~=2.4.0'
index_url:
not: 'https://pypi.org/simple'
run:
score: 0.5
stack_info:
- type: WARNING
message: Hello, Thoth!
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STEP_SCHEMA(prescription)
StepPrescription.set_prescription(prescription)
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=Source("https://thoth-station.ninja/simple"),
develop=False,
)
state.add_resolved_dependency(
("tensorflow", "2.4.0", "https://thoth-station.ninja/simple"))
unit = StepPrescription()
unit.pre_run()
with unit.assigned_context(context):
result = unit.run(state, package_version)
assert isinstance(result, tuple)
assert result[0] == 0.5
assert result[1] is None
self.verify_justification_schema(context.stack_info)
assert context.stack_info == [{
"type": "WARNING",
"message": "Hello, Thoth!",
"link": "https://thoth-station.ninja"
}]
# Run one more time to make sure stack info is added just once.
result = unit.run(state, package_version)
assert isinstance(result, tuple)
assert result[0] == 0.5
assert result[1] is None
assert context.stack_info == [{
"type": "WARNING",
"message": "Hello, Thoth!",
"link": "https://thoth-station.ninja"
}]
def test_get_random_first_unresolved_dependency(self) -> None:
"""Test getting random first unresolved dependency."""
state = State(score=1.0)
package_tuple1 = ("tensorflow", "2.1.0", "https://pypi.org/simple")
package_tuple2 = ("selinon", "1.0.0", "https://pypi.org/simple")
state.add_unresolved_dependency(package_tuple1)
assert state.get_random_first_unresolved_dependency() is package_tuple1
state.add_unresolved_dependency(package_tuple2)
random_state = random.getstate()
try:
random.seed(42)
assert state.get_random_first_unresolved_dependency(
) is package_tuple1
assert state.get_random_first_unresolved_dependency(
) is package_tuple1
assert state.get_random_first_unresolved_dependency(
) is package_tuple2
assert state.get_random_first_unresolved_dependency(
) is package_tuple1
finally:
random.setstate(random_state)
def test_run_package_version_from_with_other(self, context: Context,
state: State,
allow_other: bool) -> None:
"""Test running the prescription based on the dependency introduced without with considering other packages."""
prescription_str = f"""
name: StepUnit
type: step
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
state:
package_version_from_allow_other: {'true' if allow_other else 'false'}
package_version_from:
- name: scikit-learn
version: "<1.0.0"
develop: false
index_url: https://pypi.org/simple
run:
score: 0.5
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STEP_SCHEMA(prescription)
StepPrescription.set_prescription(prescription)
pypi = Source("https://pypi.org/simple")
package_version = PackageVersion(
name="numpy",
version="==1.19.4",
index=pypi,
develop=False,
)
package_version_other = PackageVersion(
name="tensorflow",
version="==2.6.0",
index=pypi,
develop=False,
)
package_version_from = PackageVersion(
name="scikit-learn",
version="==0.24.2",
index=pypi,
develop=False,
)
state.add_resolved_dependency(package_version_from.to_tuple())
context.register_package_version(package_version_from)
state.add_resolved_dependency(package_version_other.to_tuple())
context.register_package_version(package_version_other)
runtime_env = context.project.runtime_environment
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_from.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
context.register_package_tuple(
package_version.to_tuple(),
dependent_tuple=package_version_other.to_tuple(),
develop=False,
extras=None,
os_name=runtime_env.operating_system.name,
os_version=runtime_env.operating_system.version,
python_version=runtime_env.python_version,
)
unit = StepPrescription()
unit.pre_run()
with unit.assigned_context(context):
result = unit.run(state, package_version)
if allow_other:
assert isinstance(result, tuple)
assert result[0] == 0.5
assert result[1] is None
else:
assert result is None
def test_get_random_first_unresolved_dependency_error(self) -> None:
"""Test raising an error on random first unresolved dependency if no dependency is available."""
state = State(score=1.0)
with pytest.raises(IndexError):
state.get_random_first_unresolved_dependency()
def test_environment_markers(self, context: Context) -> None:
"""Test handling of environment markers across multiple runs."""
state = State(
score=0.0,
resolved_dependencies=OrderedDict(
{
"numpy": ("numpy", "1.0.0", "https://pypi.org/simple"),
"tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"),
}
),
unresolved_dependencies=OrderedDict(),
)
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"numpy", "1.0.0", "https://pypi.org/simple"
).and_return(["000"]).once()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"tensorflow", "2.0.0", "https://pypi.org/simple"
).and_return(["111"]).once()
pypi = Source("https://pypi.org/simple")
pv_numpy_locked = PackageVersion(
name="numpy", version="==1.0.0", index=pypi, develop=False
)
pv_tensorflow_locked = PackageVersion(
name="tensorflow", version="==2.0.0", index=pypi, develop=False
)
context.should_receive("get_package_version").with_args(
("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_numpy_locked).twice()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_tensorflow_locked).twice()
context.dependents = {
"numpy": {
("numpy", "1.0.0", "https://pypi.org/simple"): {
(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
)
}
},
"tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()},
}
context.graph.should_receive("get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.0.0",
os_name="fedora",
os_version="31",
python_version="3.7",
).and_return("python_version >= '3.7'").and_return(
"python_version >= '3' or 1"
).twice()
product = Product.from_final_state(context=context, state=state)
expected = {
"advised_runtime_environment": None,
"justification": [],
"project": {
"requirements": {
"dev-packages": {},
"packages": {"flask": "*", "tensorflow": "==1.9.0"},
"requires": {"python_version": "3.6"},
"source": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
{
"name": "pypi-org",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
],
},
"requirements_locked": {
"_meta": {
"hash": {
"sha256": "e55b6bbaba9467f1629c34e7a4180a6a2d82df37e02e762866e7aac27ced0f99"
},
"pipfile-spec": 6,
"requires": {"python_version": "3.6"},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
{
"name": "pypi-org",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
],
},
"default": {
"numpy": {
"hashes": ["sha256:000"],
"index": "pypi-org",
"markers": "python_version >= '3.7'",
"version": "==1.0.0",
},
"tensorflow": {
"hashes": ["sha256:111"],
"index": "pypi-org",
"version": "==2.0.0",
},
},
"develop": {},
},
"runtime_environment": {
"cuda_version": None,
"hardware": {"cpu_family": None, "cpu_model": None},
"name": None,
"operating_system": {"name": None, "version": None},
"python_version": None,
"platform": None,
},
},
"score": 0.0,
}
assert product.to_dict() == expected
# Markers should not intersect.
product = Product.from_final_state(context=context, state=state)
expected["project"]["requirements_locked"]["default"]["numpy"][
"markers"
] = "python_version >= '3' or 1"
assert product.to_dict() == expected
def test_from_final_state(self, context: Context) -> None:
"""Test instantiating product from a final state."""
state = State(
score=0.5,
resolved_dependencies={
"daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"),
"numpy": ("numpy", "1.17.4", "https://pypi.org/simple"),
"tensorflow":
("tensorflow", "2.0.0", "https://pypi.org/simple"),
},
unresolved_dependencies={},
advised_runtime_environment=RuntimeEnvironment.from_dict(
{"python_version": "3.6"}),
)
state.add_justification(self.JUSTIFICATION_SAMPLE_1)
pypi = Source("https://pypi.org/simple")
pv_daiquiri_locked = PackageVersion(name="daiquiri",
version="==1.6.0",
index=pypi,
develop=False)
pv_numpy_locked = PackageVersion(name="numpy",
version="==1.17.4",
index=pypi,
develop=False)
pv_tensorflow_locked = PackageVersion(name="tensorflow",
version="==2.0.0",
index=pypi,
develop=False)
context.should_receive("get_package_version").with_args(
("daiquiri", "1.6.0", "https://pypi.org/simple"),
graceful=False).and_return(pv_daiquiri_locked).ordered()
context.graph.should_receive(
"get_python_package_hashes_sha256").with_args(
"daiquiri", "1.6.0",
"https://pypi.org/simple").and_return(["000"]).ordered()
context.should_receive("get_package_version").with_args(
("numpy", "1.17.4", "https://pypi.org/simple"),
graceful=False).and_return(pv_numpy_locked).ordered()
context.graph.should_receive(
"get_python_package_hashes_sha256").with_args(
"numpy", "1.17.4",
"https://pypi.org/simple").and_return(["111"]).ordered()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
graceful=False).and_return(pv_tensorflow_locked).ordered()
context.graph.should_receive(
"get_python_package_hashes_sha256").with_args(
"tensorflow", "2.0.0",
"https://pypi.org/simple").and_return(["222"]).ordered()
pv_daiquiri = PackageVersion(name="daiquiri",
version="*",
index=pypi,
develop=False)
pv_tensorflow = PackageVersion(name="tensorflow",
version=">=2.0.0",
index=pypi,
develop=False)
project = flexmock(
pipfile=Pipfile.from_string(self._PIPFILE),
runtime_environment=RuntimeEnvironment.from_dict(
{"operating_system": {
"name": "rhel"
}}),
)
project.should_receive("iter_dependencies").with_args(
with_devel=True).and_return([pv_daiquiri, pv_tensorflow]).once()
context.project = project
context.dependencies = {
"daiquiri": {
("daiquiri", "1.6.0", "https://pypi.org/simple"): set(),
},
"numpy": {
("numpy", "1.17.4", "https://pypi.org/simple"): set()
},
"tensorflow": {
("tensorflow", "2.0.0", "https://pypi.org/simple"):
{("numpy", "1.17.4", "https://pypi.org/simple")}
},
}
context.dependents = {
"daiquiri": {
("daiquiri", "1.6.0", "https://pypi.org/simple"): set(),
},
"numpy": {
("numpy", "1.17.4", "https://pypi.org/simple"): {(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
)}
},
"tensorflow": {
("tensorflow", "2.0.0", "https://pypi.org/simple"): set()
},
}
context.graph.should_receive(
"get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.17.4",
os_name="fedora",
os_version="31",
python_version="3.7",
).and_return("python_version >= '3.7'").once()
product = Product.from_final_state(state=state, context=context)
assert product.score == state.score
assert product.justification == state.justification
assert product.advised_runtime_environment == state.advised_runtime_environment
assert product.project.to_dict() == {
"requirements": {
"packages": {
"daiquiri": {
"index": "pypi-org",
"version": "*"
},
"tensorflow": {
"index": "pypi-org",
"version": ">=2.0.0"
},
},
"dev-packages": {},
"requires": {
"python_version": "3.7"
},
"source": [{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org",
}],
},
"requirements_locked": {
"_meta": {
"sources": [{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org",
}],
"requires": {
"python_version": "3.7"
},
"hash": {
"sha256":
"f08689732b596fd705a45bbf9ec44c3995b17a1aa6392c46500aeb736c4d4e88"
},
"pipfile-spec":
6,
},
"default": {
"daiquiri": {
"version": "==1.6.0",
"hashes": ["sha256:000"],
"index": "pypi-org",
},
"numpy": {
"version": "==1.17.4",
"hashes": ["sha256:111"],
"index": "pypi-org",
"markers": "python_version >= '3.7'",
},
"tensorflow": {
"version": "==2.0.0",
"hashes": ["sha256:222"],
"index": "pypi-org",
},
},
"develop": {},
},
"runtime_environment": {
"hardware": {
"cpu_family": None,
"cpu_model": None
},
"operating_system": {
"name": "rhel",
"version": None
},
"python_version": None,
"cuda_version": None,
"name": None,
"platform": None,
},
}
def test_from_final_state(self, context: Context) -> None:
"""Test instantiating product from a final state."""
state = State(
score=0.5,
resolved_dependencies=OrderedDict(
{
"daiquiri": ("daiquiri", "1.6.0", "https://pypi.org/simple"),
"numpy": ("numpy", "1.17.4", "https://pypi.org/simple"),
"tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"),
}
),
unresolved_dependencies=OrderedDict(),
advised_runtime_environment=RuntimeEnvironment.from_dict(
{"python_version": "3.6"}
),
)
state.add_justification([{"foo": "bar"}])
pypi = Source("https://pypi.org/simple")
pv_daiquiri_locked = PackageVersion(
name="daiquiri", version="==1.6.0", index=pypi, develop=False
)
pv_numpy_locked = PackageVersion(
name="numpy", version="==1.17.4", index=pypi, develop=False
)
pv_tensorflow_locked = PackageVersion(
name="tensorflow", version="==2.0.0", index=pypi, develop=False
)
context.should_receive("get_package_version").with_args(
("daiquiri", "1.6.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_daiquiri_locked).ordered()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"daiquiri", "1.6.0", "https://pypi.org/simple"
).and_return(["000"]).ordered()
context.should_receive("get_package_version").with_args(
("numpy", "1.17.4", "https://pypi.org/simple"), graceful=False
).and_return(pv_numpy_locked).ordered()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"numpy", "1.17.4", "https://pypi.org/simple"
).and_return(["111"]).ordered()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_tensorflow_locked).ordered()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"tensorflow", "2.0.0", "https://pypi.org/simple"
).and_return(["222"]).ordered()
pv_daiquiri = PackageVersion(
name="daiquiri", version="*", index=pypi, develop=False
)
pv_tensorflow = PackageVersion(
name="tensorflow", version=">=2.0.0", index=pypi, develop=False
)
project = flexmock()
project.should_receive("iter_dependencies").with_args(
with_devel=True
).and_return([pv_daiquiri, pv_tensorflow]).once()
context.project = project
context.dependencies = {
"daiquiri": {
("daiquiri", "1.6.0", "https://pypi.org/simple"): set(),
},
"numpy": {
("numpy", "1.17.4", "https://pypi.org/simple"): set()
},
"tensorflow": {
("tensorflow", "2.0.0", "https://pypi.org/simple"): {
("numpy", "1.17.4", "https://pypi.org/simple")
}
},
}
context.dependents = {
"daiquiri": {
("daiquiri", "1.6.0", "https://pypi.org/simple"): set(),
},
"numpy": {
("numpy", "1.17.4", "https://pypi.org/simple"): {
(("tensorflow", "2.0.0", "https://pypi.org/simple"), "fedora", "31", "3.7")
}
},
"tensorflow": {
("tensorflow", "2.0.0", "https://pypi.org/simple"): set()
},
}
context.graph.should_receive("get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.17.4",
os_name="fedora",
os_version="31",
python_version="3.7",
).and_return("python_version >= '3.7'").once()
product = Product.from_final_state(state=state, context=context)
assert product.score == state.score
assert product.justification == state.justification
assert product.advised_runtime_environment == state.advised_runtime_environment
assert product.project.to_dict() == {
"requirements": {
"packages": {
"daiquiri": {"index": "pypi-org", "version": "*"},
"tensorflow": {"index": "pypi-org", "version": ">=2.0.0"},
},
"dev-packages": {},
"source": [
{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org",
}
],
},
"requirements_locked": {
"_meta": {
"sources": [
{
"url": "https://pypi.org/simple",
"verify_ssl": True,
"name": "pypi-org",
}
],
"requires": {},
"hash": {
"sha256": "c3a2f42932b6e5cd30f5664b11eda605f5fbd672f1b88729561d0d3edd10b5d9"
},
"pipfile-spec": 6,
},
"default": {
"daiquiri": {
"version": "==1.6.0",
"hashes": ["sha256:000"],
"index": "pypi-org",
},
"numpy": {
"version": "==1.17.4",
"hashes": ["sha256:111"],
"index": "pypi-org",
"markers": "python_version >= '3.7'",
},
"tensorflow": {
"version": "==2.0.0",
"hashes": ["sha256:222"],
"index": "pypi-org",
},
},
"develop": {},
},
"runtime_environment": {
"hardware": {"cpu_family": None, "cpu_model": None},
"operating_system": {"name": None, "version": None},
"python_version": None,
"cuda_version": None,
"name": None,
},
}
def test_justification(self, final_state: State) -> None:
"""Test manipulation with state justification."""
assert final_state.justification == [{"foo": "bar"}]
final_state.add_justification([{"hello": "thoth"}])
assert final_state.justification == [{"foo": "bar"}, {"hello": "thoth"}]
def test_environment_markers_direct_dependency(self, context: Context) -> None:
"""Test handling environment markers for direct dependencies."""
state = State(
score=0.0,
resolved_dependencies={
"numpy": ("numpy", "1.0.0", "https://pypi.org/simple"),
"tensorflow": ("tensorflow", "2.0.0", "https://pypi.org/simple"),
},
unresolved_dependencies={},
justification=[{"type": "INFO", "message": "Foo bar", "link": "https://thoth-station.ninja"}],
)
pypi = Source("https://pypi.org/simple")
# Let's assume tensorflow is our direct dependency with an environment marker set. It sets also an environment
# marker for numpy.
context.project.pipfile.packages.packages.pop("tensorflow", None)
tf_package_version_direct = PackageVersion(
name="tensorflow", version=">1.0.0", index=pypi, develop=False, markers="python_version >= '3.6'"
)
context.project.pipfile.add_package_version(tf_package_version_direct)
# Just to make sure numpy is not in the direct dependency listing.
context.project.pipfile.packages.packages.pop("numpy", None)
pv_numpy_locked = PackageVersion(name="numpy", version="==1.0.0", index=pypi, develop=False)
pv_tensorflow_locked = PackageVersion(name="tensorflow", version="==2.0.0", index=pypi, develop=False)
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
*pv_numpy_locked.to_tuple()
).and_return(["000"]).once()
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
*pv_tensorflow_locked.to_tuple()
).and_return(["111"]).once()
context.should_receive("get_package_version").with_args(
("numpy", "1.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_numpy_locked).once()
context.should_receive("get_package_version").with_args(
("tensorflow", "2.0.0", "https://pypi.org/simple"), graceful=False
).and_return(pv_tensorflow_locked).once()
context.dependents = {
"numpy": {
("numpy", "1.0.0", "https://pypi.org/simple"): {
(
("tensorflow", "2.0.0", "https://pypi.org/simple"),
"fedora",
"31",
"3.7",
)
}
},
"tensorflow": {("tensorflow", "2.0.0", "https://pypi.org/simple"): set()},
}
context.graph.should_receive("get_python_environment_marker").with_args(
"tensorflow",
"2.0.0",
"https://pypi.org/simple",
dependency_name="numpy",
dependency_version="1.0.0",
os_name="fedora",
os_version="31",
python_version="3.7",
marker_evaluation_result=True,
).and_return("python_version >= '3.7'").once()
product = Product.from_final_state(context=context, state=state)
expected = {
"advised_manifest_changes": [],
"advised_runtime_environment": None,
"justification": [
{
"link": "https://thoth-station.ninja",
"message": "Foo bar",
"type": "INFO",
}
],
"dependency_graph": {"edges": [], "nodes": ["numpy", "tensorflow"]},
"project": {
"constraints": [],
"requirements": {
"dev-packages": {},
"packages": {
"flask": "*",
"tensorflow": {
"index": "pypi-org-simple",
"markers": "python_version >= '3.6'",
"version": ">1.0.0",
},
},
"requires": {"python_version": "3.6"},
"source": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
{
"name": "pypi-org-simple",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
],
"thoth": {
"allow_prereleases": {},
"disable_index_adjustment": False,
},
},
"requirements_locked": {
"_meta": {
"hash": {"sha256": "382cd84046246d08cf670a07c61628958dec76e05d2473bf936866fc54619f9a"},
"pipfile-spec": 6,
"requires": {"python_version": "3.6"},
"sources": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{
"name": "pypi-org-simple",
"url": "https://pypi.org/simple",
"verify_ssl": True,
},
],
},
"default": {
"numpy": {
"hashes": ["sha256:000"],
"index": "pypi-org-simple",
"markers": "python_version >= '3.7'",
"version": "==1.0.0",
},
"tensorflow": {
"hashes": ["sha256:111"],
"index": "pypi-org-simple",
"markers": "python_version >= '3.6'",
"version": "==2.0.0",
},
},
"develop": {},
},
"runtime_environment": {
"base_image": None,
"cuda_version": None,
"cudnn_version": None,
"hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None},
"name": None,
"operating_system": {"name": None, "version": None},
"openblas_version": None,
"labels": None,
"openmpi_version": None,
"mkl_version": None,
"python_version": None,
"recommendation_type": None,
"platform": None,
},
},
"score": 0.0,
}
assert product.to_dict() == expected
def test_is_final(self, state: State, final_state: State) -> None:
"""Test checks for final states."""
assert final_state.is_final()
assert not state.is_final()
state.unresolved_dependencies.pop("flask")
assert state.is_final()
def test_no_observation(self, context: Context) -> None:
"""Test adding information about no justification added."""
state = State(
score=0.0,
resolved_dependencies={
"flask": ("flask", "0.12", "https://pypi.org/simple"),
},
unresolved_dependencies={},
)
context.project.pipfile.packages.packages.pop("tensorflow")
context.graph.should_receive("get_python_package_hashes_sha256").with_args(
"flask", "0.12", "https://pypi.org/simple"
).and_return(["222"]).once()
pypi = Source("https://pypi.org/simple")
pv_pandas_locked = PackageVersion(name="flask", version="==0.12", index=pypi, develop=False)
context.should_receive("get_package_version").with_args(
("flask", "0.12", "https://pypi.org/simple"), graceful=False
).and_return(pv_pandas_locked).once()
context.dependents = {
"flask": {("flask", "0.12", "https://pypi.org/simple"): set()},
}
product = Product.from_final_state(context=context, state=state)
expected = {
"advised_manifest_changes": [],
"advised_runtime_environment": None,
"dependency_graph": {"edges": [], "nodes": ["flask"]},
"justification": [
{
"type": "INFO",
"message": "No issues spotted for this stack based on Thoth's database",
"link": jl("no_observations"),
}
],
"project": {
"constraints": [],
"requirements": {
"dev-packages": {},
"packages": {"flask": "*"},
"requires": {"python_version": "3.6"},
"source": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True},
],
"thoth": {
"allow_prereleases": {},
"disable_index_adjustment": False,
},
},
"requirements_locked": {
"_meta": {
"hash": {"sha256": "2e49395dfa87159358e581bd22e656c27c0dab04894d1b137a14f85bb387ea51"},
"pipfile-spec": 6,
"requires": {"python_version": "3.6"},
"sources": [
{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True},
{"name": "pypi-org-simple", "url": "https://pypi.org/simple", "verify_ssl": True},
],
},
"default": {
"flask": {"hashes": ["sha256:222"], "index": "pypi-org-simple", "version": "==0.12"},
},
"develop": {},
},
"runtime_environment": {
"base_image": None,
"cuda_version": None,
"cudnn_version": None,
"hardware": {"cpu_family": None, "cpu_model": None, "gpu_model": None},
"name": None,
"operating_system": {"name": None, "version": None},
"labels": None,
"platform": None,
"mkl_version": None,
"openmpi_version": None,
"openblas_version": None,
"python_version": None,
"recommendation_type": None,
},
},
"score": 0.0,
}
assert product.to_dict() == expected
def test_run_classifiers(self, state: State, context: Context) -> None:
"""Test running and assigning all the classifiers."""
trove_classifiers = [
"Development Status:: 3 - Alpha",
"Development Status:: 7 - Inactive",
"Environment :: GPU :: NVIDIA CUDA",
"Environment :: GPU :: NVIDIA CUDA :: 11.2",
"Environment :: GPU :: NVIDIA CUDA :: 9.2",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.9",
]
package_name = "tensorflow-gpu"
package_version = "2.6.0"
index_url = "https://pypi.org/simple"
context.recommendation_type = RecommendationType.STABLE
context.project.runtime_environment.cuda_version = "10.2"
context.project.runtime_environment.python_version = "3.8"
context.graph.should_receive("get_python_package_version_trove_classifiers_all").with_args(
package_name=package_name,
package_version=package_version,
index_url=index_url,
os_name=context.project.runtime_environment.operating_system.name,
os_version=context.project.runtime_environment.operating_system.version,
python_version=context.project.runtime_environment.python_version,
).and_return(trove_classifiers).once()
state.justification.clear()
unit = self.UNIT_TESTED()
unit.pre_run()
state.resolved_dependencies.clear()
state.add_resolved_dependency((package_name, package_version, index_url))
with unit.assigned_context(context):
unit.run(state)
assert state.justification == [
{
"link": jl("trove_cuda"),
"message": "No CUDA specific trove classifier matching CUDA version used '10.2'",
"package_name": package_name,
"type": "WARNING",
},
{
"link": jl("trove_unstable"),
"message": "Development status stated in trove classifiers is 'alpha' which "
"might be not suitable to be used with recommendation type "
"'stable'",
"package_name": "tensorflow-gpu",
"type": "WARNING",
},
{
"link": jl("trove_inactive"),
"message": f"Inactive development status of {package_name!r} stated in trove " "classifiers",
"package_name": package_name,
"type": "WARNING",
},
{
"link": jl("trove_py"),
"message": "No Python specific trove classifier matching Python version used '3.8'",
"package_name": package_name,
"type": "WARNING",
},
]
def test_advised_manifest_changes(self, state: State,
context: Context) -> None:
"""Test advising changes in the manifest files."""
prescription_str = """
name: WrapUnit
type: wrap
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: intel-tensorflow
run:
advised_manifest_changes:
apiVersion: apps.openshift.io/v1
kind: DeploymentConfig
patch:
op: add
path: /spec/template/spec/containers/0/env/0
value:
name: OMP_NUM_THREADS
value: "1"
stack_info:
- type: INFO
message: Added advised changes
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_WRAP_SCHEMA(prescription)
WrapPrescription.set_prescription(prescription)
state.add_resolved_dependency(
("intel-tensorflow", "2.2.0", "https://pypi.org/simple"))
state.justification.clear()
assert state.advised_manifest_changes == []
unit = WrapPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
# Run one more time to verify stack info and advised manifest changes are added just once.
assert unit.run(state) is None
assert state.advised_manifest_changes == [{
"apiVersion": "apps.openshift.io/v1",
"kind": "DeploymentConfig",
"patch": {
"op": "add",
"path": "/spec/template/spec/containers/0/env/0",
"value": {
"name": "OMP_NUM_THREADS",
"value": "1"
},
},
}]
assert context.stack_info == [{
"type": "INFO",
"message": "Added advised changes",
"link": "https://thoth-station.ninja"
}]
def test_random_termial_error(self, n) -> None:
"""Test error out when random termial is used wih negative or zero value."""
with pytest.raises(ValueError):
State._random_termial(n)
