def test_run_noop(self, context: Context, state: State, tf_name: str,
tf_version: str) -> None:
"""Test not removing scipy from a TensorFlow stack."""
scipy_package_version = PackageVersion(
name="scipy",
version="==1.2.2",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
assert "tensorflow" not in state.resolved_dependencies
state.resolved_dependencies[
"tensorflow"] = tf_package_version.to_tuple()
context.register_package_version(tf_package_version)
context.dependents["scipy"] = {
scipy_package_version.to_tuple():
{(tf_package_version.to_tuple(), "rhel", "8", "3.6")}
}
unit = TensorFlowRemoveSciPyStep()
with TensorFlowRemoveSciPyStep.assigned_context(context):
assert unit.run(state, scipy_package_version) is None
def test_get_package_version(self, context: Context,
package_version: PackageVersion) -> None:
"""Test getting registering and getting a package version."""
with pytest.raises(NotFound):
context.get_package_version(package_version.to_tuple())
assert context.register_package_version(package_version) is False
assert context.get_package_version(
package_version.to_tuple()) is package_version
def _run_state_with_initiator(self, state: State,
package_version: PackageVersion) -> bool:
"""Check state match respecting also initiator of the give package."""
state_prescription = self.match_prescription.get("state")
if not state_prescription:
# Nothing to check.
return True
package_version_from = state_prescription.get(
"package_version_from") or []
# XXX: we explicitly do not consider runtime environment as we expect to have it only one here.
dependents = {
i[0]
for i in self.context.dependents.get(package_version.name, {}).get(
package_version.to_tuple(), set())
}
for resolved_dependency in package_version_from:
resolved = state.resolved_dependencies.get(
resolved_dependency["name"])
if not resolved:
return False
if not self._check_package_tuple_from_prescription(
resolved, resolved_dependency):
return False
if resolved not in dependents:
_LOGGER.debug(
"Package %r stated in package_version_from not a did not introduce package %r",
resolved,
package_version.to_tuple(),
)
return False
dependents.discard(resolved)
if dependents and not state_prescription.get(
"package_version_from_allow_other", False):
for dependent in dependents:
if dependent == state.resolved_dependencies.get(dependent[0]):
return False
for resolved_dependency in state_prescription.get(
"resolved_dependencies", []):
resolved = state.resolved_dependencies.get(
resolved_dependency["name"])
if not resolved:
return False
if not self._check_package_tuple_from_prescription(
resolved, resolved_dependency):
return False
return True
def test_not_fully_resolved_error(self):
"""Test trying to running pipeline with unresolved package produces an error."""
project = Project.from_strings(_PIPFILE_STR)
pipeline = Pipeline(
graph=None, # We avoid low-level testing down to thoth-storages.
project=project,
sieves=[],
steps=[],
strides=[],
)
direct_dependency = PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=False,
)
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): [
(("flask", "1.0.2", "https://pypi.org/simple"), ("werkzeug", None, None))
]},
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
with pytest.raises(NotResolvedError):
pipeline.conduct(limit=None, count=None)
# Try again, now with first item in the tuple list - same error should be produced.
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): [
(("flask", None, None), ("werkzeug", "0.15.6", "https://pypi.org/simple"))
]},
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
with pytest.raises(NotResolvedError):
pipeline.conduct(limit=None, count=None)
def test_get_package_version_graceful(
self, context: Context, package_version: PackageVersion) -> None:
"""Test getting registered package version, gracefully."""
assert context.get_package_version(package_version.to_tuple(),
graceful=True) is None
with pytest.raises(NotFound):
context.get_package_version(package_version.to_tuple(),
graceful=False)
assert context.register_package_version(package_version) is False
assert context.get_package_version(package_version.to_tuple(),
graceful=True) is package_version
assert context.get_package_version(package_version.to_tuple(),
graceful=False) is package_version
def test_no_tf_21(self, urllib3_version: str, tf_version: str) -> None:
"""Test no penalization for TensorFlow other than 2.1 by this pipeline step."""
urllib3_package_version = PackageVersion(
name="urllib3",
version=f"=={urllib3_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name="tensorflow",
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.resolved_dependencies[
"tensorflow"] = tf_package_version.to_tuple()
# Context is not used during the actual pipeline run.
context = flexmock()
with TensorFlow21Urllib3Step.assigned_context(context):
unit = TensorFlow21Urllib3Step()
assert unit.run(state, urllib3_package_version) is None
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[float, List[Dict[str, str]]]]:
"""Penalize stacks with a CVE."""
try:
cve_records = self.context.graph.get_python_cve_records_all(
package_name=package_version.name,
package_version=package_version.locked_version,
)
except NotFoundError as exc:
_LOGGER.warning("Package %r in version %r not found: %r", str(exc))
return None
if cve_records:
_LOGGER.debug(
"Found a CVEs for %r: %r", package_version.to_tuple(), cve_records
)
penalization = len(cve_records) * self.configuration["cve_penalization"]
# Note down package causing this CVE.
for record in cve_records:
record["package_name"] = package_version.name
return penalization, cve_records
return None
def run(
self, state: State, package_version: PackageVersion
) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]:
"""Remove SciPy dependency from a TensorFlow>2.1<=2.3 stack."""
tensorflow_any = (
state.resolved_dependencies.get("tensorflow")
or state.resolved_dependencies.get("tensorflow-cpu")
or state.resolved_dependencies.get("tensorflow-gpu")
)
if not tensorflow_any:
return None
tf_package_version = self.context.get_package_version(tensorflow_any)
if not tf_package_version:
return None
tf_release = tf_package_version.semantic_version.release[:2]
if tf_release < (2, 1) or tf_release >= (2, 3):
return None
# Now check what package introduced the SciPy dependency. If it is solely TensorFlow, we can
# safely remove SciPy from dependencies.
scipy_dependents = {i[0] for i in self.context.dependents["scipy"][package_version.to_tuple()]}
introduced_by = scipy_dependents & set(state.resolved_dependencies.values())
if len(introduced_by) == 1 and next(iter(introduced_by)) == tensorflow_any:
if not self._message_logged:
self._message_logged = True
_LOGGER.warning("%s - see %s", self._MESSAGE, self._LINK)
self.context.stack_info.append({"type": "WARNING", "message": self._MESSAGE, "link": self._LINK})
raise SkipPackage
return None
def test_run_not_acceptable(self, context: Context, tf_name: str,
tf_version: str, np_version: str) -> None:
"""Test resolutions that are not acceptable."""
package_version = PackageVersion(
name="numpy",
version=f"=={np_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(tf_package_version.to_tuple())
context.register_package_version(tf_package_version)
unit = TensorFlow22NumPyStep()
unit.pre_run()
with unit.assigned_context(context):
assert unit._message_logged is False
with pytest.raises(NotAcceptable):
unit.run(state, package_version)
assert unit._message_logged is True
def test_tf_21(self, context: Context, urllib3_version: str,
tf_version: str) -> None:
"""Test penalizing TensorFlow in version 2.1."""
tf_package_version = PackageVersion(
name="tensorflow",
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
urllib3_package_version = PackageVersion(
name="urllib3",
version=f"=={urllib3_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.resolved_dependencies[
"tensorflow"] = tf_package_version.to_tuple()
assert not context.stack_info
with TensorFlow21Urllib3Step.assigned_context(context):
unit = TensorFlow21Urllib3Step()
unit.pre_run()
assert unit._message_logged is False
with pytest.raises(NotAcceptable):
assert unit.run(state, urllib3_package_version)
assert unit._message_logged is True
assert context.stack_info
assert self.verify_justification_schema(context.stack_info)
def test_run_noop(self, context: Context, tf_name: str, tf_version: str,
gast_version: str) -> None:
"""Test no operation performed when not invalid combination is seen."""
gast_package_version = PackageVersion(
name="gast",
version=f"=={gast_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(tf_package_version.to_tuple())
context.register_package_version(tf_package_version)
unit = TensorFlow114GastStep()
with unit.assigned_context(context):
assert unit._message_logged is False
assert unit.run(state, gast_package_version) is None
assert unit._message_logged is False
def test_register_package_version_existing(
self, context: Context, package_version: PackageVersion) -> None:
"""Test registering an existing package version to context."""
assert context.register_package_version(package_version) is False
assert context.get_package_version(
package_version.to_tuple()) is package_version
assert context.register_package_version(package_version) is True
def test_tf_21(self, context: Context, tf_version: str, h5py_version: str) -> None:
"""Test blocking resolution of h5py with TensorFlow==2.1 or TensorFlow==2.3.1."""
tf_package_version = PackageVersion(
name="tensorflow",
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
h5py_package_version = PackageVersion(
name="h5py",
version=f"=={h5py_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.resolved_dependencies["tensorflow"] = tf_package_version.to_tuple()
context.register_package_version(tf_package_version)
assert not context.stack_info
with self.UNIT_TESTED.assigned_context(context):
unit = self.UNIT_TESTED()
unit.pre_run()
assert unit._message_logged is False
with pytest.raises(NotAcceptable):
unit.run(state, h5py_package_version)
assert unit._message_logged is True
assert context.stack_info
assert self.verify_justification_schema(context.stack_info)
def test_run_not_acceptable(self, context: Context, tf_name: str,
tf_version: str, gast_version: str) -> None:
"""Test not acceptable TensorFlow<=1.14 with gast>0.2.2."""
gast_package_version = PackageVersion(
name="gast",
version=f"=={gast_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(tf_package_version.to_tuple())
context.register_package_version(tf_package_version)
unit = TensorFlow114GastStep()
with unit.assigned_context(context):
assert unit._message_logged is False
with pytest.raises(NotAcceptable):
unit.run(state, gast_package_version)
assert unit._message_logged is True
def test_stride_remove_stack(self):
project = Project.from_strings(_PIPFILE_STR)
direct_dependency = PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=False,
)
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): []},
)
pipeline = Pipeline(
graph=None, # We avoid low-level testing down to thoth-storages.
project=project,
sieves=[],
steps=[],
strides=[
(_MockStrideRemoveStack, None)
],
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
# We raise an exception on first walk call, no stacks should be produced.
pipeline_products = list(pipeline.conduct(limit=None, count=None))
assert len(pipeline_products) == 0
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[float, List[Dict[str, str]]]]:
"""Penalize stacks with a CVE."""
try:
cve_records = self.context.graph.get_python_cve_records_all(
package_name=package_version.name,
package_version=package_version.locked_version,
)
except NotFoundError as exc:
_LOGGER.warning("Package %r in version %r not found: %r", str(exc))
return None
if cve_records:
package_version_tuple = package_version.to_tuple()
_LOGGER.debug("Found a CVEs for %r: %r", package_version_tuple,
cve_records)
if self.context.recommendation_type == RecommendationType.SECURITY:
if package_version_tuple not in self._messages_logged:
self._messages_logged.add(package_version_tuple)
for cve_record in cve_records:
message = (
f"Skipping including package {package_version_tuple!r} as a CVE "
f"{cve_record['cve_id']!r} was found")
_LOGGER.warning(
"%s: %s",
message,
cve_record["details"],
)
self.context.stack_info.append({
"type":
"WARNING",
"message":
message,
"link":
self._JUSTIFICATION_LINK
})
raise NotAcceptable
penalization = len(
cve_records) * self.configuration["cve_penalization"]
justification = []
for cve_record in cve_records:
message = f"Package {package_version_tuple!r} has a CVE {cve_record['cve_id']!r}"
justification.append({
"package_name": package_version.name,
"link": self._JUSTIFICATION_LINK,
"advisory": cve_record["details"],
"message": message,
"type": "WARNING",
})
return max(penalization, -1.0), justification
return None
def test_run_develop(self, context: Context, state: State,
develop: bool) -> None:
"""Test running this pipeline unit resulting in a justification addition when "not" index url is used."""
prescription_str = f"""
name: GHReleaseNotes
type: wrap.GHReleaseNotes
should_include:
adviser_pipeline: true
match:
state:
resolved_dependencies:
name: thoth-solver
develop: {'true' if develop else 'false'}
run:
release_notes:
organization: thoth-station
repository: solver
tag_version_prefix: v
"""
units = list(self._instantiate_gh_release_notes_wrap(prescription_str))
assert len(
units
) == 1, "Multiple units created, expected only one based on prescription"
unit = units[0]
state.resolved_dependencies.clear()
state.justification.clear()
package_version = PackageVersion(
name="thoth-solver",
version="==0.5.0",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state.add_resolved_dependency(package_version.to_tuple())
context.register_package_version(package_version)
unit.pre_run()
with unit.assigned_context(context):
# Run twice to verify the justification is added just once.
assert unit.run(state) is None
assert unit.run(state) is None
self.verify_justification_schema(state.justification)
assert set(tuple(i.items()) for i in state.justification) == {
(
("type", "INFO"),
("message", "Release notes for package 'thoth-solver'"),
("link",
"https://github.com/thoth-station/solver/releases/tag/v0.5.0"
),
("package_name", "thoth-solver"),
),
}
def register_package_version(self, package_version: PackageVersion) -> bool:
"""Register the given package version to the context."""
package_tuple = package_version.to_tuple()
registered = self.package_versions.get(package_tuple)
if registered:
# If the given package is shared in develop and in the main part, make it main stack part.
registered.develop = registered.develop or package_version.develop
return True
# Direct dependency, no dependency introduced this one.
self._note_dependencies(package_tuple=None, dependency_tuple=package_tuple)
self.package_versions[package_tuple] = package_version
return False
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]:
"""Score the given package regardless of the state."""
# Using seed set to process on the adviser run affects this call - so adviser
# with same seed set shared scores generated across runs.
score = self._score_history.setdefault(
package_version.to_tuple(),
random.uniform(self.SCORE_MIN, self.SCORE_MAX)
if random.random() <= self.configuration["assign_probability"] else
0.0,
)
return score, None
def test_run(self) -> None:
"""Test computing package combinations."""
pypi = Source("https://pypi.org/simple")
pv0 = PackageVersion(
name="flask",
version="==1.0.0",
index=pypi,
develop=False,
)
pv1 = PackageVersion(
name="flask",
version="==0.12",
index=pypi,
develop=False,
)
pv2 = PackageVersion(
name="pandas",
version="==1.3.3",
index=pypi,
develop=False,
)
pv3 = PackageVersion(
name="pandas",
version="==1.0.0",
index=pypi,
develop=False,
)
unit = self.UNIT_TESTED()
unit.update_configuration({
"package_name":
None,
"package_combinations": ["flask", "werkzeug"],
})
unit.unit_run = True
unit.pre_run()
assert unit.unit_run is False, "Parent pre_run was probably not called"
assert list(unit.run((pv for pv in (pv0, )))) == [pv0]
assert list(unit.run((pv for pv in (pv1, )))) == [pv1]
assert list(unit.run((pv for pv in (pv2, )))) == [pv2]
assert list(unit.run((pv for pv in (pv3, )))) == []
assert list(unit.run((pv for pv in (pv2, )))) == [pv2]
assert list(unit.run((pv for pv in (pv1, )))) == [pv1]
assert list(unit.run((pv for pv in (pv0, )))) == [pv0]
assert unit._package_combinations == {"flask", "werkzeug"}
assert unit._package_tuples_seen == {
"pandas": pv2.to_tuple(),
}
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[float, List[Dict[str, str]]]]:
"""Penalize stacks with a CVE."""
try:
cve_records = self.context.graph.get_python_cve_records_all(
package_name=package_version.name,
package_version=package_version.locked_version,
)
except NotFoundError as exc:
_LOGGER.warning("Package %r in version %r not found: %r", exc)
return None
if cve_records:
package_version_tuple = package_version.to_tuple()
_LOGGER.debug("Found a CVEs for %r: %r", package_version_tuple,
cve_records)
justification = []
for cve_record in cve_records:
message = f"Package {package_version_tuple!r} has a CVE {cve_record['cve_id']!r}"
justification.append({
"package_name": package_version.name,
"link": cve_record.get("link") or self._JUSTIFICATION_LINK,
"advisory": cve_record["details"],
"message": message,
"type": "WARNING",
})
if self.context.recommendation_type not in (
RecommendationType.LATEST, RecommendationType.TESTING):
# Penalize only if not latest/testing.
penalization = len(
cve_records) * self.configuration["cve_penalization"]
return max(penalization, -1.0), justification
return 0.0, justification
else:
justification = [{
"package_name":
package_version.name,
"link":
self._JUSTIFICATION_LINK_NO_CVE,
"type":
"INFO",
"message":
f"No known CVE known for {package_version.name!r} in "
f"version {package_version.locked_version!r}",
}]
return 0.0, justification
def _do_try_exclude(sieve_context: SieveContext,
package_version: PackageVersion, config: dict) -> None:
"""Try to exclude the given package, produce warning if exclusion was not successful."""
try:
sieve_context.remove_package(package_version)
except CannotRemovePackage as exc:
# TODO: this should go to sieve context and be reported to the user in final recommendations
_LOGGER.warning(
"Using package %r which was released for different operating system %r, package"
"cannot be removed: %s",
package_version.to_tuple(),
config["os_version"],
str(exc),
)
def run(
self, _: State, package_version: PackageVersion
) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]:
"""Score the given package."""
package_tuple = package_version.to_tuple()
score = self._history.get(package_tuple)
if score is not None:
return score, None
idx = self._idx
self._idx = (self._idx + 1) % self.configuration["buffer_size"]
self._history[package_tuple] = self._buffer[idx]
return self._buffer[idx], None
def test_run_deps(self, context: Context, state: State) -> None:
"""Test not removing scipy from a TensorFlow stack if introduced by another dependency."""
scipy_package_version = PackageVersion(
name="scipy",
version="==1.2.2",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name="tensorflow",
version="==2.2.0",
develop=False,
index=Source("https://pypi.org/simple"),
)
another_package_version = PackageVersion(
name="some-package",
version="==1.0.0",
develop=False,
index=Source("https://pypi.org/simple"),
)
assert "tensorflow" not in state.resolved_dependencies
state.resolved_dependencies[
"tensorflow"] = tf_package_version.to_tuple()
state.resolved_dependencies[
another_package_version.name] = another_package_version.to_tuple()
context.register_package_version(tf_package_version)
context.dependents["scipy"] = {
scipy_package_version.to_tuple(): {
(tf_package_version.to_tuple(), "rhel", "8", "3.6"),
(another_package_version.to_tuple(), "rhel", "8", "3.6"),
}
}
unit = TensorFlowRemoveSciPyStep()
with TensorFlowRemoveSciPyStep.assigned_context(context):
assert unit.run(state, scipy_package_version) is None
def run(
self, package_version: PackageVersion
) -> Generator[Tuple[str, str, str], None, None]:
"""Run main entry-point for pseudonyms to map packages to their counterparts."""
if (not self._index_url_check(
self._index_url, package_version.index.url)) or (
self._specifier is not None
and package_version.locked_version not in self._specifier):
yield from ()
return None
to_yield = self.run_prescription["yield"]
to_yield_package_version = to_yield.get("package_version") or {}
if to_yield.get("yield_matched_version"):
pseudonym_package_version = package_version.locked_version
else:
pseudonym_package_version = to_yield_package_version.get(
"locked_version")
if pseudonym_package_version:
pseudonym_package_version = pseudonym_package_version[2:]
runtime_environment = self.context.project.runtime_environment
pseudonyms = self.context.graph.get_solved_python_package_versions_all(
package_name=to_yield_package_version.get("name"),
package_version=pseudonym_package_version,
index_url=to_yield_package_version.get("index_url"),
count=None,
os_name=runtime_environment.operating_system.name,
os_version=runtime_environment.operating_system.version,
python_version=runtime_environment.python_version,
distinct=True,
is_missing=False,
)
prescription_conf = self._configuration["prescription"]
if pseudonyms and not prescription_conf["run"]:
self._run_stack_info()
self._run_log()
prescription_conf["run"] = True
for pseudonym in pseudonyms:
_LOGGER.info(
"%s: Considering package %r as a pseudonym of %r",
self.get_unit_name(),
pseudonym,
package_version.to_tuple(),
)
yield pseudonym[0], pseudonym[1], pseudonym[2]
def test_run_develop_state_match(self, context: Context, state: State,
develop: bool,
state_develop: bool) -> None:
"""Test not running the prescription if develop flag is set also on the state match."""
prescription_str = f"""
name: StepUnit
type: step
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
develop: {'true' if develop else 'false'}
state:
resolved_dependencies:
- name: pytest
develop: {'true' if state_develop else 'false'}
run:
score: 0.5
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STEP_SCHEMA(prescription)
StepPrescription.set_prescription(prescription)
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state_package_version = PackageVersion(
name="pytest",
version="==6.2.4",
index=Source("https://pypi.org/simple"),
develop=state_develop,
)
state.add_resolved_dependency(state_package_version.to_tuple())
context.register_package_version(state_package_version)
unit = StepPrescription()
unit.pre_run()
with unit.assigned_context(context):
result = unit.run(state, package_version)
assert isinstance(result, tuple)
assert result[0] == 0.5
assert result[1] is None
def test_conduct_premature_stream_end(self):
"""Test pipeline reports premature stream error if libdependency_graph.so died."""
def raise_premature_stream_error():
raise PrematureStreamEndError("Premature stream exception")
project = Project.from_strings(_PIPFILE_STR)
pipeline = Pipeline(
graph=None, # We avoid low-level testing down to thoth-storages.
project=project,
sieves=[],
steps=[],
strides=[],
)
flexmock(
DependencyGraphWalker,
walk=raise_premature_stream_error,
)
direct_dependency = PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=False,
)
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): []},
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
# We raise an exception on first walk call, no stacks should be produced.
assert len(list(pipeline.conduct(limit=None, count=None))) == 0
# Get stack info after conduct.
stack_info = pipeline.get_stack_info()
assert len(stack_info) == 1
stack_info_entry = stack_info[0]
assert "type" in stack_info_entry
assert stack_info_entry["type"] == "WARNING"
assert "justification" in stack_info_entry
def test_stride_simple_run(self):
"""Test running a single stride inside pipeline."""
project = Project.from_strings(_PIPFILE_STR)
direct_dependency = PackageVersion(
name="flask",
version="==1.0.2",
index=Source("https://pypi.org/simple"),
develop=False,
)
step_context = StepContext.from_paths(
{direct_dependency.to_tuple(): direct_dependency},
{direct_dependency.to_tuple(): []}
)
pipeline = Pipeline(
graph=None, # We avoid low-level testing down to thoth-storages.
project=project,
sieves=[],
steps=[
(_MockStep, _MOCK_STEP_PARAMETERS)
],
strides=[
(_MockStride, _MOCK_STRIDE_PARAMETERS)
],
)
flexmock(
pipeline,
_prepare_direct_dependencies=lambda with_devel: [direct_dependency],
_resolve_transitive_dependencies=lambda _: step_context,
)
# We raise an exception on first walk call, no stacks should be produced.
pipeline_products = list(pipeline.conduct(limit=None, count=None))
assert len(pipeline_products) == 1
assert pipeline.get_stack_info() == []
pipeline_product = pipeline_products[0]
assert isinstance(pipeline_product, PipelineProduct)
assert pipeline_product.justification == _MOCK_STRIDE_JUSTIFICATION
assert pipeline_product.score == 1.0
assert pipeline_product.project
def test_run_match_develop(self, context: Context, state: State, develop: bool) -> None:
"""Test running this pipeline unit based on develop matching."""
prescription_str = f"""
name: StrideUnit
type: stride
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: flask
develop: {'true' if develop else 'false'}
run:
stack_info:
- type: INFO
message: This message will be shown
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STRIDE_SCHEMA(prescription)
StridePrescription.set_prescription(prescription)
package_version = PackageVersion(
name="flask",
version="==2.0.1",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state.add_resolved_dependency(package_version.to_tuple())
context.register_package_version(package_version)
assert not context.stack_info
unit = StridePrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
# Run one more time to verify the stack info is added just once.
assert unit.run(state) is None
assert context.stack_info == [
{"type": "INFO", "message": "This message will be shown", "link": "https://thoth-station.ninja"}
]
def test_run_develop(self, context: Context, state: State,
develop: bool) -> None:
"""Test running this pipeline unit based on matching develop flag."""
prescription_str = f"""
name: WrapUnit
type: wrap
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: flask
develop: {'true' if develop else 'false'}
run:
justification:
- type: INFO
message: This message will be shown
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_WRAP_SCHEMA(prescription)
WrapPrescription.set_prescription(prescription)
package_version = PackageVersion(
name="flask",
version="==2.0.1",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state.add_resolved_dependency(package_version.to_tuple())
context.register_package_version(package_version)
state.justification.clear()
unit = WrapPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
# Run one more time to verify justification is added only once.
assert unit.run(state) is None
assert state.justification == unit.run_prescription["justification"]
