def test_verify_multiple_should_include(
self, builder_context: PipelineBuilderContext) -> None:
"""Verify multiple should_include calls do not loop endlessly."""
builder_context.recommendation_type = RecommendationType.LATEST
builder_context.project = Project.from_strings(
self._CASE_DISALLOWED_PIPFILE)
self.verify_multiple_should_include(builder_context)
def test_not_include_thoth_prereleases_allowed(self, builder_context: PipelineBuilderContext) -> None:
"""Test not including this pipeline unit."""
builder_context.recommendation_type = RecommendationType.LATEST
builder_context.project = Project.from_strings(self._CASE_SELECTIVE_PRERELEASES_ALLOWED_PIPFILE)
assert builder_context.is_adviser_pipeline()
assert list(CutPreReleasesSieve.should_include(builder_context)) == []
def test_no_python_config(self) -> None:
"""Test assigning Python version from Thoth's config file."""
context = flexmock(project=Project.from_strings(
self._CASE_PIPFILE_PYTHON),
stack_info=[])
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = "8"
assert context.project.runtime_environment.python_version is None
boot = PythonVersionBoot()
with PythonVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version == "8"
assert context.project.runtime_environment.python_version == "3.6"
assert context.stack_info == [
{
'Message':
'No version of Python specified in the configuration, using '
"Python version found in Pipfile: '3.6'",
'type':
'WARNING'
},
]
def post_provenance_python(application_stack: dict,
origin: str = None,
debug: bool = False,
force: bool = False):
"""Check provenance for the given application stack."""
parameters = locals()
try:
project = Project.from_strings(application_stack["requirements"],
application_stack["requirements_lock"])
except ThothPythonException as exc:
return {
"parameters": parameters,
"error": f"Invalid application stack supplied: {str(exc)}"
}, 400
except Exception as exc:
return {
"parameters": parameters,
"error": "Invalid application stack supplied"
}, 400
graph = GraphDatabase()
graph.connect()
parameters["whitelisted_sources"] = list(
graph.get_python_package_index_urls())
force = parameters.pop("force", False)
cached_document_id = _compute_digest_params(
dict(**project.to_dict(),
origin=origin,
whitelisted_sources=parameters["whitelisted_sources"]))
timestamp_now = int(time.mktime(datetime.datetime.utcnow().timetuple()))
cache = ProvenanceCacheStore()
cache.connect()
if not force:
try:
cache_record = cache.retrieve_document_record(cached_document_id)
if cache_record[
"timestamp"] + Configuration.THOTH_CACHE_EXPIRATION > timestamp_now:
return {
"analysis_id": cache_record.pop("analysis_id"),
"cached": True,
"parameters": parameters
}, 202
except CacheMiss:
pass
response, status = _do_schedule(
parameters,
_OPENSHIFT.schedule_provenance_checker,
output=Configuration.THOTH_PROVENANCE_CHECKER_OUTPUT)
if status == 202:
cache.store_document_record(cached_document_id, {
"analysis_id": response["analysis_id"],
"timestamp": timestamp_now
})
return response, status
def test_no_python_pipfile(self) -> None:
"""Test assigning Python version from Pipfile."""
context = flexmock(project=Project.from_strings(
self._CASE_PIPFILE_NO_PYTHON),
stack_info=[])
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = "8"
context.project.runtime_environment.python_version = "3.6"
boot = PythonVersionBoot()
with PythonVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version == "8"
assert context.project.runtime_environment.python_version == "3.6"
assert context.stack_info == [{
"message":
"No version of Python specified explicitly, assigning the one "
"found in Thoth's configuration: '3.6'",
"type":
"WARNING",
"link":
jl("py_version"),
}]
def test_python_version_mismatch(self) -> None:
"""Test when python version stated in Pipfile does not match with the one provided in the configuration."""
context = flexmock(project=Project.from_strings(
self._CASE_PIPFILE_PYTHON),
stack_info=[])
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = "8"
context.project.runtime_environment.python_version = "3.8"
boot = PythonVersionBoot()
with PythonVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version == "8"
assert context.project.runtime_environment.python_version == "3.8"
assert context.stack_info == [{
"message":
"Python version stated in Pipfile ('3.6') does not match with the one "
"specified in the Thoth configuration ('3.8'), using Python version from Thoth "
"configuration implicitly",
"type":
"WARNING",
"link":
jl("py_version"),
}]
def test_os_sieve(self):
"""Test removal of packages based on AICoE package source index configuration.
We keep only TensorFlow release which is from PyPI and manylinux2010 build as there is no match on OS release.
"""
package_versions = self._get_packages_aicoe()
sieve_context = SieveContext.from_package_versions(package_versions)
# Do not assign runtime environment intentionally - it will default to no environment.
project = Project.from_strings(
pipfile_str=self._PIPFILE_CONTENT_AICOE,
runtime_environment=RuntimeEnvironment.from_dict(
{"operating_system": {
"name": "rhel",
"version": "7.5"
}}))
os_sieve = OperatingSystemSieve(graph=None, project=project)
os_sieve.run(sieve_context)
expected = {
("pytest", "3.0.0", "https://pypi.org/simple"),
("tensorflow", "1.9.0", "https://pypi.org/simple"),
# Filtering out this entry is left on another sieve which ensures runtime environment compatibility.
("tensorflow", "1.9.0",
"https://tensorflow.pypi.thoth-station.ninja/index/manylinux2010/jemalloc/simple/"
),
# These are filtered out:
# ("tensorflow", "1.9.0", "https://tensorflow.pypi.thoth-station.ninja/index/os/fedora/30/jemalloc/simple/"),
# ("tensorflow", "1.9.0", "https://tensorflow.pypi.thoth-station.ninja/index/os/rhel/7.6/jemalloc/simple/")
}
assert set(sieve_context.iter_direct_dependencies_tuple()) == expected
def test_stride_remove_stack(self):
project = Project.from_strings(_PIPFILE_STR)
direct_dependency = PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=False,
)
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): []},
)
pipeline = Pipeline(
graph=None, # We avoid low-level testing down to thoth-storages.
project=project,
sieves=[],
steps=[],
strides=[
(_MockStrideRemoveStack, None)
],
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
# We raise an exception on first walk call, no stacks should be produced.
pipeline_products = list(pipeline.conduct(limit=None, count=None))
assert len(pipeline_products) == 0
def test_os_sieve_no_error(self):
"""Test no error raised if no packages satisfy OS specific requirements."""
package_versions = [
PackageVersion(
name="tensorflow",
version="==1.9.0",
index=Source(
"https://tensorflow.pypi.thoth-station.ninja/index/fedora/30/jemalloc/simple/"
),
develop=False,
)
]
sieve_context = SieveContext.from_package_versions(package_versions)
project = Project.from_strings(
pipfile_str=self._PIPFILE_CONTENT_AICOE,
runtime_environment=RuntimeEnvironment.from_dict(
{"operating_system": {
"name": "ubi",
"version": "9"
}}))
os_sieve = OperatingSystemSieve(graph=None, project=project)
os_sieve.run(sieve_context)
assert set(sieve_context.iter_direct_dependencies_tuple()) == {
("tensorflow", "1.9.0",
"https://tensorflow.pypi.thoth-station.ninja/index/fedora/30/jemalloc/simple/"
),
}
def test_pre_releases_disallowed_removal(self, context: Context,
package_name: str,
package_version: str) -> None:
"""Test no removals if pre-releases are allowed."""
pv = PackageVersion(
name=package_name,
version=f"=={package_version}",
index=Source("https://pypi.org/simple"),
develop=False,
)
project = Project.from_strings(self._CASE_GLOBAL_DISALLOWED_PIPFILE)
context.project = project
sieve = self.UNIT_TESTED()
sieve.update_configuration({
"package_name":
None,
"allow_prereleases":
project.pipfile.thoth.allow_prereleases
})
assert not context.stack_info
with self.UNIT_TESTED.assigned_context(context):
assert list(sieve.run(p for p in [pv])) == []
assert len(context.stack_info) == 1
assert self.verify_justification_schema(context.stack_info)
def test_remove_pre_releases_disallowed_noop(self, context: Context,
package_name: str,
package_version: str) -> None:
"""Test NOT removing dependencies based on pre-release configuration."""
pv = PackageVersion(
name=package_name,
version=f"=={package_version}",
index=Source("https://pypi.org/simple"),
develop=False,
)
project = Project.from_strings(self._CASE_GLOBAL_DISALLOWED_PIPFILE)
context.project = project
sieve = self.UNIT_TESTED()
sieve.update_configuration({
"package_name":
None,
"allow_prereleases":
project.pipfile.thoth.allow_prereleases
})
assert not context.stack_info
with self.UNIT_TESTED.assigned_context(context):
assert list(sieve.run(p for p in [pv])) == [pv]
assert not context.stack_info, "No stack information should be provided"
def test_remove_all_transitive_error(self):
"""Test raising of an error if all the transitive deps of a type were removed."""
source = Source("https://pypi.org/simple")
direct_dependencies = {
("tensorflow", "2.0.0", source.url):
PackageVersion(
name="tensorflow",
version="==2.0.0",
index=source,
develop=False,
)
}
paths = {
("tensorflow", "2.0.0", "https://pypi.org/simple"): [
(("tensorflow", "2.0.0", "https://pypi.org/simple"),
("numpy", "1.0.0rc1", "https://thoth-station.ninja/simple")),
]
}
step_context = StepContext.from_paths(direct_dependencies, paths)
project = Project.from_strings(self._CASE_PIPFILE)
restrict_indexes = CutPreReleases(
graph=None,
project=project,
library_usage=None,
)
with pytest.raises(CannotRemovePackage):
restrict_indexes.run(step_context)
def test_no_clash(self) -> None:
"""Test raising no exception raised if clash occurs."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_NOT_CLASHED))
boot = VersionClashBoot()
with VersionClashBoot.assigned_context(context):
boot.run()
def test_clash(self) -> None:
"""Test raising an exception if a version clash occurs."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_CLASHED))
with pytest.raises(CannotProduceStack):
boot = VersionClashBoot()
with VersionClashBoot.assigned_context(context):
boot.run()
def test_rhel_assign(self) -> None:
"""Test remapping UBI to RHEL."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE))
context.project.runtime_environment.operating_system.name = "ubi"
boot = UbiBoot()
with UbiBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
def test_no_rhel_assign(self) -> None:
"""Test no change made if operating system is not UBI."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE))
context.project.runtime_environment.operating_system.name = "fedora"
boot = UbiBoot()
with UbiBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "fedora"
def test_include(
self,
builder_context: PipelineBuilderContext,
recommendation_type: RecommendationType,
) -> None:
"""Test including this pipeline unit."""
builder_context.recommendation_type = recommendation_type
builder_context.project = Project.from_strings(self._CASE_DISALLOWED_PIPFILE)
assert builder_context.is_adviser_pipeline()
assert list(CutPreReleasesSieve.should_include(builder_context)) == [{}]
def _get_case(self) -> Tuple[PackageVersion, Project]:
"""Get all the objects needed for a test case for this sieve."""
project = Project.from_strings(self._CASE_PIPFILE)
flexmock(GraphDatabase)
package_version = PackageVersion(
name="tensorflow",
version="==2.0.0",
index=Source("https://pypi.org/simple"),
develop=False,
)
return package_version, project
def test_version_no_change_no_version(self) -> None:
"""Test no change to RHEL version identifier if OS version was not supplied."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE))
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = None
boot = RHELVersionBoot()
with RHELVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version is None
def test_rhel_assign(self, context: Context) -> None:
"""Test remapping UBI to RHEL."""
context.project = Project.from_strings(self._CASE_PIPFILE)
context.project.runtime_environment.operating_system.name = "ubi"
boot = UbiBoot()
with UbiBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.stack_info, "No stack info provided"
assert self.verify_justification_schema(context.stack_info) is True
def test_version_change(self) -> None:
"""Test changing RHEL version to its major version."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE))
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = "8.1"
boot = RHELVersionBoot()
with RHELVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version == "8"
def test_version_no_change_no_os_match(self) -> None:
"""Test no change to RHEL version identifier if OS does not match."""
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE))
context.project.runtime_environment.operating_system.name = "fedora"
context.project.runtime_environment.operating_system.version = "31"
boot = RHELVersionBoot()
with RHELVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "fedora"
assert context.project.runtime_environment.operating_system.version == "31"
def test_no_cut(self) -> None:
"""Test not removing a locked package based on direct dependencies."""
tf = PackageVersion(
name="tensorflow",
version="==1.9.0",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_LOCKED))
with CutLockedSieve.assigned_context(context):
sieve = CutLockedSieve()
assert list(sieve.run(p for p in [tf])) == [tf]
def test_pre_releases_disallowed_removal(self) -> None:
"""Test no removals if pre-releases are allowed."""
tf_2_0_0rc0 = PackageVersion(
name="tensorflow",
version="==2.0.0rc0",
index=Source("https://tensorflow.pypi.thoth-station.ninja/index/os/fedora/30/jemalloc/simple/"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_DISALLOWED_PIPFILE))
with CutPreReleasesSieve.assigned_context(context):
sieve = CutPreReleasesSieve()
assert list(sieve.run(p for p in [tf_2_0_0rc0])) == []
def test_remove_pre_releases_allowed_noop(self) -> None:
"""Test removing dependencies not hitting limit causes a noop."""
tf_2_0_0rc = PackageVersion(
name="tensorflow",
version="==2.0.0rc0",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_ALLOWED_PIPFILE))
with CutPreReleasesSieve.assigned_context(context):
sieve = CutPreReleasesSieve()
assert list(sieve.run(p for p in [tf_2_0_0rc])) == [tf_2_0_0rc]
def test_noop_dev(self) -> None:
"""Test no operation if dependencies are not locked."""
pytest = PackageVersion(
name="pytest",
version="==5.3.1",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_NOT_LOCKED))
with CutLockedSieve.assigned_context(context):
sieve = CutLockedSieve()
assert list(sieve.run(p for p in [pytest])) == [pytest]
def test_remove2(self):
"""Test removal of both, direct and transitive dependencies in one run."""
direct_dependencies = {
("tensorflow", "2.0.0", "https://pypi.org/simple"):
PackageVersion(
name="tensorflow",
version="==2.0.0",
index=Source("https://pypi.org/simple"),
develop=False,
),
("tensorflow", "1.9.0", "https://thoth-station.ninja/simple"):
PackageVersion(
name="tensorflow",
version="==1.9.0",
index=Source("https://thoth-station.ninja/simple"),
develop=False,
),
}
paths = {
("tensorflow", "2.0.0", "https://pypi.org/simple"): [
(("tensorflow", "2.0.0", "https://pypi.org/simple"),
("numpy", "1.0.0", "https://pypi.org/simple")),
(("tensorflow", "2.0.0", "https://pypi.org/simple"),
("numpy", "2.0.0", "https://thoth-station.ninja/simple")),
],
("tensorflow", "1.9.0", "https://thoth-station.ninja/simple"): [
(("tensorflow", "1.9.0", "https://thoth-station.ninja/simple"),
("numpy", "2.0.0", "https://thoth-station.ninja/simple")),
(("tensorflow", "1.9.0", "https://thoth-station.ninja/simple"),
("numpy", "1.0.0", "https://pypi.org/simple")),
],
}
step_context = StepContext.from_paths(direct_dependencies, paths)
project = Project.from_strings(self._CASE_PIPFILE)
restrict_indexes = RestrictIndexes(
graph=None,
project=project,
library_usage=None,
)
restrict_indexes.run(step_context)
assert step_context.dependency_graph_adaptation.to_scored_package_tuple_pairs(
) == [(0.0, (None, ('tensorflow', '2.0.0',
'https://pypi.org/simple'))),
(0.0, (('tensorflow', '2.0.0', 'https://pypi.org/simple'),
('numpy', '1.0.0', 'https://pypi.org/simple')))]
assert (len(list(step_context.iter_direct_dependencies())) == 1
), "Wrong number of direct dependencies"
def test_noop_no_aicoe(self):
"""Test no changes are made if no AICoE releases are found."""
package_versions = self._get_packages_no_aicoe()
sieve_context = SieveContext.from_package_versions(package_versions)
# Do not assign runtime environment intentionally - it will default to no environment.
project = Project.from_strings(
pipfile_str=self._PIPFILE_CONTENT_NO_AICOE)
os_sieve = OperatingSystemSieve(graph=None, project=project)
os_sieve.run(sieve_context)
assert {pv.to_tuple()
for pv in package_versions
} == set(sieve_context.iter_direct_dependencies_tuple())
def test_not_fully_resolved_error(self):
"""Test trying to running pipeline with unresolved package produces an error."""
project = Project.from_strings(_PIPFILE_STR)
pipeline = Pipeline(
graph=None, # We avoid low-level testing down to thoth-storages.
project=project,
sieves=[],
steps=[],
strides=[],
)
direct_dependency = PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=False,
)
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): [
(("flask", "1.0.2", "https://pypi.org/simple"), ("werkzeug", None, None))
]},
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
with pytest.raises(NotResolvedError):
pipeline.conduct(limit=None, count=None)
# Try again, now with first item in the tuple list - same error should be produced.
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): [
(("flask", None, None), ("werkzeug", "0.15.6", "https://pypi.org/simple"))
]},
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
with pytest.raises(NotResolvedError):
pipeline.conduct(limit=None, count=None)
def post(self):
"""Store requirements file to disk."""
initial_path = Path.cwd()
input_data = self.get_json_body()
# Path of the repo where we need to store
path_to_store: str = input_data["path_to_store"]
kernel_name: str = input_data["kernel_name"]
requirements: str = input_data["requirements"]
requirements_lock: str = input_data["requirement_lock"]
complete_path: str = input_data["complete_path"]
env_path = Path(complete_path).joinpath(path_to_store).joinpath(
kernel_name)
_LOGGER.info("Path used to store dependencies is: %r",
env_path.as_posix())
# Delete and recreate folder
if env_path.exists():
_ = subprocess.call(
f"rm -rf ./{kernel_name} ",
shell=True,
cwd=Path(complete_path).joinpath(path_to_store))
env_path.mkdir(parents=True, exist_ok=True)
os.chdir(env_path)
requirements_format = "pipenv"
project = Project.from_strings(requirements, requirements_lock)
pipfile_path = env_path.joinpath("Pipfile")
pipfile_lock_path = env_path.joinpath("Pipfile.lock")
if requirements_format == "pipenv":
_LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r", env_path)
project.to_files(pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path)
os.chdir(initial_path)
self.finish(
json.dumps({
"message":
f"Successfully stored requirements at {env_path}!"
}))
