def test_cut_locked(self) -> None:
"""Test removing a locked package based on direct dependencies."""
tf = PackageVersion(
name="tensorflow",
version="==2.0.0",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_LOCKED))
with CutLockedSieve.assigned_context(context):
sieve = CutLockedSieve()
assert list(sieve.run(p for p in [tf])) == []
def test_os_sieve_no_remove(self):
"""Test the TensorFlow package is not removed as it has no other candidate."""
package_versions = [
PackageVersion(
name="tensorflow",
version="==1.9.0",
index=Source(
"https://tensorflow.pypi.thoth-station.ninja/index/fedora/30/jemalloc/simple/"
),
develop=False,
),
PackageVersion(
name="pytest",
version="==3.0.0",
index=Source("https://pypi.org/simple"),
develop=True,
),
]
sieve_context = SieveContext.from_package_versions(package_versions)
# Do not assign runtime environment intentionally - it will default to no environment.
project = Project.from_strings(
pipfile_str=self._PIPFILE_CONTENT_AICOE,
runtime_environment=RuntimeEnvironment.from_dict(
{"operating_system": {
"name": "rhel",
"version": "7.5"
}}))
os_sieve = OperatingSystemSieve(graph=None, project=project)
os_sieve.run(sieve_context)
expected = {
("pytest", "3.0.0", "https://pypi.org/simple"),
("tensorflow", "1.9.0",
"https://tensorflow.pypi.thoth-station.ninja/index/fedora/30/jemalloc/simple/"
),
}
assert set(sieve_context.iter_direct_dependencies_tuple()) == expected
def test_remove_pre_releases_allowed_noop(self) -> None:
"""Test removing dependencies not hitting limit causes a noop."""
tf_2_0_0rc = PackageVersion(
name="tensorflow",
version="==2.0.0rc0",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_ALLOWED_PIPFILE))
with CutPreReleasesSieve.assigned_context(context):
sieve = CutPreReleasesSieve()
assert list(sieve.run(p for p in [tf_2_0_0rc])) == [tf_2_0_0rc]
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]:
"""Score the given package regardless of the state."""
# Using seed set to process on the adviser run affects this call - so adviser
# with same seed set shared scores generated across runs.
score = self._score_history.setdefault(
package_version.to_tuple(),
random.uniform(self.SCORE_MIN, self.SCORE_MAX)
if random.random() <= self.configuration["assign_probability"] else
0.0,
)
return score, None
def register_package_version(self, package_version: PackageVersion) -> bool:
"""Register the given package version to the context."""
package_tuple = package_version.to_tuple()
registered = self.package_versions.get(package_tuple)
if registered:
# If the given package is shared in develop and in the main part, make it main stack part.
registered.develop = registered.develop or package_version.develop
return True
# Direct dependency, no dependency introduced this one.
self._note_dependencies(package_tuple=None, dependency_tuple=package_tuple)
self.package_versions[package_tuple] = package_version
return False
def test_from_pip_compile_files_example_dir3(self) -> None:
"""Test loading only if only requirements.in is present."""
with cwd(os.path.join(self.data_dir, "requirements", "example_dir3")):
project = Project.from_pip_compile_files(allow_without_lock=True)
assert project.pipfile_lock is None
assert list(project.iter_dependencies()) == [
PackageVersion(name="flask",
version="*",
develop=False,
hashes=[],
index=None)
]
def test_db_0_all_versions(self, graph):
"""Check that resolving can gather all versions available in the graph database."""
resolved = PythonGraphSolver(graph_db=graph).solve(
[PackageVersion(name="a", version="*", index=None, develop=False)],
graceful=False,
all_versions=True,
)
assert len(resolved) == 1
assert "a" in resolved
assert resolved == {
"a": [("1.0.0", "index1"), ("1.1.0", "index1"),
("1.2.0", "index2")]
}
def test_noop_dev(self) -> None:
"""Test no operation if dependencies are not locked."""
pytest = PackageVersion(
name="pytest",
version="==5.3.1",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_NOT_LOCKED))
with CutLockedSieve.assigned_context(context):
sieve = CutLockedSieve()
assert list(sieve.run(p for p in [pytest])) == [pytest]
def test_pre_releases_disallowed_removal(self) -> None:
"""Test no removals if pre-releases are allowed."""
tf_2_0_0rc0 = PackageVersion(
name="tensorflow",
version="==2.0.0rc0",
index=Source("https://tensorflow.pypi.thoth-station.ninja/index/os/fedora/30/jemalloc/simple/"),
develop=False,
)
context = flexmock(project=Project.from_strings(self._CASE_DISALLOWED_PIPFILE))
with CutPreReleasesSieve.assigned_context(context):
sieve = CutPreReleasesSieve()
assert list(sieve.run(p for p in [tf_2_0_0rc0])) == []
def test_remove_all_direct_error(self):
"""Test raising an error if all the direct deps of a type were removed."""
source = Source("https://pypi.org/simple")
direct_dependencies = {
("tensorflow", "2.0.0", source.url):
PackageVersion(
name="tensorflow",
version="==2.0.0",
index=source,
develop=False,
),
("numpy", "1.0.0rc1", source.url):
PackageVersion(
name="numpy",
version="==1.0.0rc1",
index=source,
develop=False,
),
}
paths = {
("tensorflow", "2.0.0", "https://pypi.org/simple"): [
(("tensorflow", "2.0.0", "https://pypi.org/simple"),
("absl-py", "1.0.0", "https://pypi.org/simple")),
]
}
step_context = StepContext.from_paths(direct_dependencies, paths)
project = Project.from_strings(self._CASE_PIPFILE)
restrict_indexes = CutPreReleases(
graph=None,
project=project,
library_usage=None,
)
with pytest.raises(CannotRemovePackage):
restrict_indexes.run(step_context)
def test_sieve_develop_true(self, context: Context, develop: bool) -> None:
"""Test sieving packages based on develop flag set to true."""
prescription_str = f"""
name: SieveUnit
type: sieve
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: flask
develop: {'true' if develop else 'false'}
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_SIEVE_SCHEMA(prescription)
SievePrescription.set_prescription(prescription)
package_versions = [
PackageVersion(
name="flask",
version="==1.1.2",
index=Source("https://pypi.org/simple"),
develop=develop,
),
PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=not develop,
),
]
unit = SievePrescription()
unit.pre_run()
with unit.assigned_context(context):
result = list(unit.run((pv for pv in package_versions)))
assert len(result) == 1
assert [result[0]] == [pv for pv in package_versions if pv.develop != develop]
def _python_package_name(n: object) -> None:
"""Validate Python package name."""
if not isinstance(n, str):
raise Invalid(f"Value {n!r} is not a valid package name")
try:
normalized = PackageVersion.normalize_python_package_name(n)
except Exception as exc:
raise Invalid(f"Failed to parse Python package name {n!r}: {str(exc)}")
else:
if normalized != n:
raise Invalid(
f"Python package name {n!r} is not in a normalized form, normalized: {normalized!r}"
)
def test_from_pip_compile_files_example_dir1(self) -> None:
"""Test loading only if requirements.txt is present."""
with cwd(os.path.join(self.data_dir, "requirements", "example_dir1")):
with pytest.raises(FileLoadError):
Project.from_pip_compile_files()
project = Project.from_pip_compile_files(allow_without_lock=True)
assert project.pipfile_lock is None
assert list(project.iter_dependencies()) == [
PackageVersion(
name="click", version="*", develop=False, hashes=[], index=Source("https://pypi.org/simple")
)
]
def _do_try_exclude(sieve_context: SieveContext,
package_version: PackageVersion, config: dict) -> None:
"""Try to exclude the given package, produce warning if exclusion was not successful."""
try:
sieve_context.remove_package(package_version)
except CannotRemovePackage as exc:
# TODO: this should go to sieve context and be reported to the user in final recommendations
_LOGGER.warning(
"Using package %r which was released for different operating system %r, package"
"cannot be removed: %s",
package_version.to_tuple(),
config["os_version"],
str(exc),
)
def test_no_tf_avx2(self) -> None:
"""Test not recommending TensorFlow without AVX2 support."""
package_version = PackageVersion(
name="tensorflow",
version="==2.2.0",
develop=False,
index=Source("https://pypi.org/simple"),
)
# State and context are unused in the actual pipeline run.
state, context = flexmock(), flexmock()
with TensorFlowAVX2Step.assigned_context(context):
unit = TensorFlowAVX2Step()
assert unit.run(state, package_version) is None
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[float, List[Dict[str, str]]]]:
"""Penalize stacks with a CVE."""
try:
cve_records = self.context.graph.get_python_cve_records_all(
package_name=package_version.name,
package_version=package_version.locked_version,
)
except NotFoundError as exc:
_LOGGER.warning("Package %r in version %r not found: %r", exc)
return None
if cve_records:
package_version_tuple = package_version.to_tuple()
_LOGGER.debug("Found a CVEs for %r: %r", package_version_tuple,
cve_records)
justification = []
for cve_record in cve_records:
message = f"Package {package_version_tuple!r} has a CVE {cve_record['cve_id']!r}"
justification.append({
"package_name": package_version.name,
"link": cve_record.get("link") or self._JUSTIFICATION_LINK,
"advisory": cve_record["details"],
"message": message,
"type": "WARNING",
})
if self.context.recommendation_type not in (
RecommendationType.LATEST, RecommendationType.TESTING):
# Penalize only if not latest/testing.
penalization = len(
cve_records) * self.configuration["cve_penalization"]
return max(penalization, -1.0), justification
return 0.0, justification
else:
justification = [{
"package_name":
package_version.name,
"link":
self._JUSTIFICATION_LINK_NO_CVE,
"type":
"INFO",
"message":
f"No known CVE known for {package_version.name!r} in "
f"version {package_version.locked_version!r}",
}]
return 0.0, justification
def _get_packages_no_aicoe():
"""Get packages which are not specific to AICoE."""
source = Source("https://pypi.org/simple")
return [
PackageVersion(
name="thoth-python",
version="==1.0.0",
index=source,
develop=False,
),
PackageVersion(
name="thoth-adviser",
version="==1.0.0",
index=source,
develop=False,
),
PackageVersion(
name="pytest",
version="==3.0.0",
index=source,
develop=True,
),
]
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]:
"""Score the given package."""
package_tuple = package_version.to_tuple()
score = self._history.get(package_tuple)
if score is not None:
return score, None
idx = self._idx
self._idx = (self._idx + 1) % self.configuration["buffer_size"]
self._history[package_tuple] = self._buffer[idx]
return self._buffer[idx], None
def register_package_tuple(
self,
package_tuple: Tuple[str, str, str],
*,
develop: bool,
dependent_tuple: Optional[Tuple[str, str, str]] = None,
extras: Optional[List[str]] = None,
os_name: Optional[str],
os_version: Optional[str],
python_version: Optional[str],
) -> PackageVersion:
"""Register the given package tuple to pipeline context and return its package version representative."""
registered = self.package_versions.get(package_tuple)
if registered:
# If the given package is shared in develop and in the main part, make it main stack part.
registered.develop = registered.develop or develop
self._note_dependencies(
dependent_tuple,
package_tuple,
os_name=os_name,
os_version=os_version,
python_version=python_version,
)
# This method is called solely on transitive dependencies - for those we do not track
# extras as extras are already resolved by solver runs (pre-computed). Keep extras untouched
# in this function call.
return registered
source = self.sources.get(package_tuple[2])
if not source:
source = Source(package_tuple[2])
self.sources[package_tuple[2]] = source
package_version = PackageVersion(
name=package_tuple[0],
version="==" + package_tuple[1],
index=source,
extras=extras,
develop=develop,
)
self.package_versions[package_tuple] = package_version
self._note_dependencies(
dependent_tuple,
package_tuple,
os_name=os_name,
os_version=os_version,
python_version=python_version,
)
return package_version
def test_db_0_package_raises(self, graph: MockedGraphDatabase) -> None:
"""Check that there is raised an exception if no releases were found."""
with pytest.raises(SolverException):
PythonPackageGraphSolver(graph=graph).solve(
[
PackageVersion(
name="nonexisting-foo",
version="==1.0.0",
index=None,
develop=False,
)
],
graceful=False,
)
def test_rule(self, context: Context) -> None:
"""Test if a rule is assigned to a package."""
package_version = PackageVersion(
name="flask",
version="==1.1.2",
index=Source("https://pypi.org/simple"),
develop=False)
(GraphDatabase.should_receive(
"get_python_package_version_solver_rules_all").with_args(
"flask",
"1.1.2",
"https://pypi.org/simple",
).and_return(["foo"]))
(GraphDatabase.should_receive(
"get_python_package_version_solver_rules_all").with_args(
"flask",
"1.1.2",
).and_return(["bar"]))
context.graph = GraphDatabase()
assert not context.stack_info, "No stack info should be provided before test run"
sieve = self.UNIT_TESTED()
sieve.pre_run()
with self.UNIT_TESTED.assigned_context(context):
assert list(sieve.run(p for p in [package_version])) == []
assert context.stack_info == [
{
"link":
"https://thoth-station.ninja/j/rules",
"message":
"Removing package ('flask', '1.1.2', 'https://pypi.org/simple') based on solver "
"rule configured: foo",
"type":
"WARNING",
},
{
"link":
"https://thoth-station.ninja/j/rules",
"message":
"Removing package ('flask', '1.1.2', 'https://pypi.org/simple') based on solver "
"rule configured: bar",
"type":
"WARNING",
},
]
assert self.verify_justification_schema(context.stack_info) is True
def _get_prepared_step_context() -> StepContext:
source = Source("https://pypi.org/simple")
direct_dependencies = {
("flask", "0.12.0", "https://pypi.org/simple"):
PackageVersion(
name="flask",
version="==0.12.0",
index=source,
develop=False,
),
("flask", "0.13.0", "https://pypi.org/simple"):
PackageVersion(
name="flask",
version="==0.13.0",
index=source,
develop=False,
),
}
paths = {
("flask", "0.12.0", "https://pypi.org/simple"): [
(("flask", "0.12.0", "https://pypi.org/simple"),
("click", "2.0", "https://pypi.org/simple")),
(("click", "2.0", "https://pypi.org/simple"),
("pyyaml", "3.12", "https://pypi.org/simple")),
],
("flask", "0.13.0", "https://pypi.org/simple"): [
(("flask", "0.13.0", "https://pypi.org/simple"),
("click", "2.0", "https://pypi.org/simple")),
(("click", "2.0", "https://pypi.org/simple"),
("pyyaml", "3.12", "https://pypi.org/simple")),
(("click", "2.0", "https://pypi.org/simple"),
("pyyaml", "4.0", "https://pypi.org/simple")),
],
}
return StepContext.from_paths(direct_dependencies, paths)
def get_latest_package_version(_, package_name): # noqa: N805
return {
"certifi": PackageVersion.parse_semantic_version("2018.10.15"),
"chardet": PackageVersion.parse_semantic_version("3.0.4"),
"idna": PackageVersion.parse_semantic_version("2.7"),
"requests": PackageVersion.parse_semantic_version("3.0.0"),
"termcolor": PackageVersion.parse_semantic_version("1.1.0"),
"urllib3": PackageVersion.parse_semantic_version("1.23"),
}[package_name]
def test_note_dependencies(self, context: Context) -> None:
"""Test noting dependencies to the context."""
dependency_tuple = ("tensorboard", "2.1.0", "https://pypi.org/simple")
package_tuple = ("tensorflow", "2.0.0", "https://pypi.org/simple")
context.register_package_version(
PackageVersion(
name=package_tuple[0],
version="==" + package_tuple[1],
index=Source(package_tuple[2]),
develop=False,
))
context.register_package_tuple(
dependency_tuple,
develop=True,
extras=None,
dependent_tuple=package_tuple,
os_name="fedora",
os_version="31",
python_version="3.7",
)
package_version = context.get_package_version(package_tuple)
assert package_version is not None
assert package_version.name == package_tuple[0]
assert package_version.locked_version == package_tuple[1]
assert package_version.index.url == package_tuple[2]
package_version = context.get_package_version(dependency_tuple)
assert package_version is not None
assert package_version.name == dependency_tuple[0]
assert package_version.locked_version == dependency_tuple[1]
assert package_version.index.url == dependency_tuple[2]
assert package_tuple[0] in context.dependencies
assert package_tuple in context.dependencies[package_tuple[0]]
assert dependency_tuple in context.dependencies[
package_tuple[0]][package_tuple]
assert dependency_tuple[0] in context.dependents
assert dependency_tuple in context.dependents[dependency_tuple[0]]
entry = context.dependents[dependency_tuple[0]][dependency_tuple]
assert entry == {(package_tuple, "fedora", "31", "3.7")}
# By calling register_package_version we get a notion about direct dependency.
assert package_tuple[0] in context.dependents
assert package_tuple in context.dependents[package_tuple[0]]
assert context.dependents[package_tuple[0]][package_tuple] == set()
def test_no_tf_21(self, context: Context, h5py_version: str, tf_version: str) -> None:
"""Test no blocking when using h5py<3 or TensorFlow!=2.1."""
h5py_package_version = PackageVersion(
name="h5py",
version=f"=={h5py_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name="tensorflow",
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.resolved_dependencies["tensorflow"] = tf_package_version.to_tuple()
# Context is not used during the actual pipeline run.
context.register_package_version(tf_package_version)
with self.UNIT_TESTED.assigned_context(context):
unit = self.UNIT_TESTED()
assert unit.run(state, h5py_package_version) is None
def test_run_no_yield(self, context: Context, package_name: str,
package_version: str) -> None:
"""Test packages the pipeline unit does not yield respecting Python version compatibility."""
package_version = PackageVersion(
name=package_name,
version=f"=={package_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
unit = self.UNIT_TESTED()
with unit.assigned_context(context):
unit.pre_run()
result = list(unit.run((pv for pv in (package_version, ))))
assert len(result) == 0
def test_run(self, context: Context, pandas_version: str) -> None:
"""Test filtering out Pandas that dropped Python 3.6 support."""
package_version = PackageVersion(
name="pandas",
version=f"=={pandas_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
unit = PandasPy36Sieve()
unit.pre_run()
with PandasPy36Sieve.assigned_context(context):
assert unit._message_logged is False
assert list(unit.run(p for p in [package_version])) == []
assert unit._message_logged is True
def test_run_no_filter(self) -> None:
""""Test not filtering a package based on version specifier."""
package_version = PackageVersion(
name="tensorboard",
version="==2.1.0",
index=Source("https://pypi.org/simple"),
develop=False,
)
unit = VersionConstraintSieve()
unit.update_configuration({
"package_name": "tensorboard",
"version_specifier": ">2.0",
})
unit.pre_run()
assert list(unit.run([package_version])) == [package_version]
def test_no_aicoe_release(self) -> None:
"""Make sure the stack score is untouched if not an AICoE release."""
package_version = PackageVersion(
name="tensorflow",
version="==2.0.0",
index=Source("https://pypi.org/simple"),
develop=False,
)
context = flexmock()
with AICoEReleasesStep.assigned_context(context):
step = AICoEReleasesStep()
result = step.run(None, package_version)
assert result is None
def test_remove_latest_versions_noop(self):
"""Test removing direct dependencies not hitting limit causes a noop."""
tf_1_1_0 = PackageVersion(
name="tensorflow",
version="==1.1.0",
index=Source(
"https://tensorflow.pypi.thoth-station.ninja/index/os/fedora/30/jemalloc/simple/"
),
develop=False,
)
tf_1_9_0 = PackageVersion(
name="tensorflow",
version="==1.9.0",
index=Source("https://pypi.org/simple"),
develop=False,
)
tf_2_0_0 = PackageVersion(
name="tensorflow",
version="==2.0.0",
index=Source(
"https://tensorflow.pypi.thoth-station.ninja/index/manylinux2010/jemalloc/simple/"
),
develop=False,
)
sieve = LimitLatestVersionsSieve(graph=None, project=None)
sieve.update_parameters(dict([("limit_latest_versions", 3)]))
sieve_context = SieveContext.from_package_versions(
[tf_1_1_0, tf_1_9_0, tf_2_0_0])
sieve.run(sieve_context)
assert list(sieve_context.iter_direct_dependencies()) == [
tf_1_1_0, tf_1_9_0, tf_2_0_0
]
