def test_run_log(self, caplog, context: Context, state: State,
log_level: str) -> None:
"""Check logging messages."""
prescription_str = f"""
name: WrapUnit
type: wrap
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: flask
run:
log:
message: Seen flask in one of the resolved stacks
type: {log_level}
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_WRAP_SCHEMA(prescription)
WrapPrescription.set_prescription(prescription)
state.add_resolved_dependency(
("flask", "0.12", "https://pypi.org/simple"))
self.check_run_log(caplog,
context,
log_level,
WrapPrescription,
state=state)
def test_run_no_resolved(self, context: Context, state: State) -> None:
"""Test running this pipeline unit not matching any resolved dependency."""
prescription_str = """
name: GitHubReleaseNotes
type: wrap.GitHubReleaseNotes
should_include:
adviser_pipeline: true
run:
release_notes:
- organization: thoth-station
repository: adviser
tag_version_prefix: v
package_version:
name: adviser
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_GITHUB_RELEASE_NOTES_WRAP_SCHEMA(prescription)
GitHubReleaseNotesWrapPrescription.set_prescription(prescription)
state.resolved_dependencies.clear()
state.add_resolved_dependency(
("flask", "1.0.0", "https://pypi.org/simple"))
state.justification.clear()
unit = GitHubReleaseNotesWrapPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
assert state.justification == []
def test_run_not_acceptable(self, context: Context, tf_name: str,
np_version: str) -> None:
"""Test wrong resolutions are not acceptable."""
package_version = PackageVersion(
name="numpy",
version=f"=={np_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(
(tf_name, "1.13.1", "https://pypi.org/simple"))
unit = TensorFlow113NumPyStep()
unit.pre_run()
assert not context.stack_info
with unit.assigned_context(context):
with pytest.raises(NotAcceptable):
assert unit._message_logged is False
unit.run(state, package_version)
assert unit._message_logged is True
assert context.stack_info
assert self.verify_justification_schema(context.stack_info) is True
def test_run_not_acceptable(self, context: Context, state: State) -> None:
"""Check raising not acceptable."""
prescription_str = """
name: StrideUnit
type: stride
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: flask
version: "<=1.0.0,>=0.12"
index_url: "https://pypi.org/simple"
- name: connexion
version: "==2.7.0"
index_url: "https://pypi.org/simple"
run:
not_acceptable: This is exception message reported
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STRIDE_SCHEMA(prescription)
StridePrescription.set_prescription(prescription)
state.add_resolved_dependency(("flask", "0.12", "https://pypi.org/simple"))
state.add_resolved_dependency(("connexion", "2.7.0", "https://pypi.org/simple"))
self.check_run_not_acceptable(context, StridePrescription, state=state)
def test_run_not_acceptable(self, context: Context, tf_name: str,
tf_version: str, np_version: str) -> None:
"""Test resolutions that are not acceptable."""
package_version = PackageVersion(
name="numpy",
version=f"=={np_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(tf_package_version.to_tuple())
context.register_package_version(tf_package_version)
unit = TensorFlow22NumPyStep()
unit.pre_run()
with unit.assigned_context(context):
assert unit._message_logged is False
with pytest.raises(NotAcceptable):
unit.run(state, package_version)
assert unit._message_logged is True
def test_run_no_resolved(self, context: Context, state: State) -> None:
"""Test running this pipeline unit not matching any resolved dependency."""
prescription_str = """
name: GHReleaseNotes
type: wrap.GHReleaseNotes
should_include:
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: numpy
run:
release_notes:
organization: thoth-station
repository: adviser
tag_version_prefix: v
"""
units = list(self._instantiate_gh_release_notes_wrap(prescription_str))
assert len(
units
) == 1, "Multiple units created, expected only one based on prescription"
unit = units[0]
state.resolved_dependencies.clear()
state.add_resolved_dependency(
("flask", "1.0.0", "https://pypi.org/simple"))
state.justification.clear()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
assert state.justification == []
def test_environment_gpu_cuda_version(
self,
cuda_version: Optional[str],
trove_classifiers: List[str],
context: Context,
state: State,
justification: List[Dict[str, str]],
) -> None:
"""Test adding justifications for environment GPU CUDA version."""
package_name = "tensorflow-gpu"
package_version = "2.6.0"
index_url = "https://pypi.org/simple"
context.project.runtime_environment.cuda_version = cuda_version
context.graph.should_receive("get_python_package_version_trove_classifiers_all").with_args(
package_name=package_name,
package_version=package_version,
index_url=index_url,
os_name=context.project.runtime_environment.operating_system.name,
os_version=context.project.runtime_environment.operating_system.version,
python_version=context.project.runtime_environment.python_version,
).and_return(trove_classifiers).once()
state.justification.clear()
unit = self.UNIT_TESTED()
unit.pre_run()
state.resolved_dependencies.clear()
state.add_resolved_dependency((package_name, package_version, index_url))
with unit.assigned_context(context):
unit.run(state)
assert state.justification == justification
def test_development_status(
self,
development_status_classifiers: List[str],
recommendation_type: RecommendationType,
justification: List[Dict[str, str]],
state: State,
context: Context,
) -> None:
"""Test adding justifications for development status."""
package_name = "thoth-common"
package_version = "0.26.0"
index_url = "https://pypi.org/simple"
context.graph.should_receive("get_python_package_version_trove_classifiers_all").with_args(
package_name=package_name,
package_version=package_version,
index_url=index_url,
os_name=context.project.runtime_environment.operating_system.name,
os_version=context.project.runtime_environment.operating_system.version,
python_version=context.project.runtime_environment.python_version,
).and_return(development_status_classifiers).once()
context.recommendation_type = recommendation_type
state.justification.clear()
unit = self.UNIT_TESTED()
unit.pre_run()
state.resolved_dependencies.clear()
state.add_resolved_dependency((package_name, package_version, index_url))
with unit.assigned_context(context):
unit.run(state)
assert state.justification == justification
def test_run_noop(self, context: Context, tf_name: str, tf_version: str,
gast_version: str) -> None:
"""Test no operation performed when not invalid combination is seen."""
gast_package_version = PackageVersion(
name="gast",
version=f"=={gast_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(tf_package_version.to_tuple())
context.register_package_version(tf_package_version)
unit = TensorFlow114GastStep()
with unit.assigned_context(context):
assert unit._message_logged is False
assert unit.run(state, gast_package_version) is None
assert unit._message_logged is False
def test_run_not_acceptable(self, context: Context, tf_name: str,
tf_version: str, gast_version: str) -> None:
"""Test not acceptable TensorFlow<=1.14 with gast>0.2.2."""
gast_package_version = PackageVersion(
name="gast",
version=f"=={gast_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
tf_package_version = PackageVersion(
name=tf_name,
version=f"=={tf_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(tf_package_version.to_tuple())
context.register_package_version(tf_package_version)
unit = TensorFlow114GastStep()
with unit.assigned_context(context):
assert unit._message_logged is False
with pytest.raises(NotAcceptable):
unit.run(state, gast_package_version)
assert unit._message_logged is True
def test_set_reward_signal_nan_inf(self, float_case: float) -> None:
"""Test (not) keeping the reward signal for nan/inf."""
predictor = TemporalDifference()
state = State()
state.add_resolved_dependency(
("tensorflow", "2.3.0", "https://pypi.org/simple"))
state.add_resolved_dependency(
("flask", "0.12", "https://pypi.org/simple"))
state.add_unresolved_dependency(
("termial-random", "0.0.2", "https://pypi.org/simple"))
predictor._policy = {
("flask", "0.12", "https://pypi.org/simple"): [0.2, 1],
}
predictor._steps_taken = 2
predictor._steps_reward = 1.2
predictor._next_state = state
assert (predictor.set_reward_signal(
state, ("tensorflow", "2.0.0", "https://pypi.org/simple"),
float_case) is None)
assert predictor._policy == {
("flask", "0.12", "https://pypi.org/simple"): [1.4, 2],
("tensorflow", "2.3.0", "https://pypi.org/simple"): [1.2, 1],
}
assert predictor._steps_taken == 0
assert predictor._steps_reward == 0.0
assert predictor._next_state is None
def test_run_acceptable_tf(self) -> None:
"""Test noop for this pipeline unit."""
package_version_1 = PackageVersion(
name="tensorflow-probability",
version="==0.11.0",
develop=False,
index=Source("https://pypi.org/simple"),
)
package_version_2 = PackageVersion(
name="flask",
version="==0.12",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
unit = TensorFlow22ProbabilityStep()
assert unit.run(state, package_version_1) is None
assert unit.run(state, package_version_2) is None
state.add_resolved_dependency(("tensorflow", "2.3.0", "https://pypi.org/simple"))
assert unit.run(state, package_version_1) is None
assert unit.run(state, package_version_2) is None
def test_run(self, context: Context, package_name: str) -> None:
"""Test recommending not to use TensorFlow 2.2 with tensorflow-probability."""
package_version = PackageVersion(
name="tensorflow-probability",
version="==0.11.0",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency((package_name, "2.2.0", "https://pypi.org/simple"))
unit = TensorFlow22ProbabilityStep()
unit.pre_run()
assert not context.stack_info
with unit.assigned_context(context):
with pytest.raises(NotAcceptable):
assert unit._message_logged is False
unit.run(state, package_version)
assert unit._message_logged is True
assert context.stack_info
assert self.verify_justification_schema(context.stack_info) is True
def test_run_stack_info(self, context: Context, state: State) -> None:
"""Check assigning stack info."""
prescription_str = """
name: WrapUnit
type: wrap
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: werkzeug
version: "<=1.0.0"
index_url: 'https://pypi.org/simple'
run:
stack_info:
- type: WARNING
message: Some message
link: https://pypi.org/project/werkzeug
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_WRAP_SCHEMA(prescription)
WrapPrescription.set_prescription(prescription)
state.add_resolved_dependency(
("werkzeug", "0.5.0", "https://pypi.org/simple"))
self.check_run_stack_info(context, WrapPrescription, state=state)
def test_run_justification_noop(self) -> None:
"""Test no operation when PyTorch is not present."""
state = State()
state.add_resolved_dependency(("micropipenv", "0.1.4", "https://pypi.org/simple"))
assert not state.justification
unit = MKLThreadsWrap()
unit.run(state)
assert len(state.justification) == 0
def test_run_no_justification(self) -> None:
"""Test NOT adding information if the given Python package is not a PyPI release."""
state = State()
state.add_resolved_dependency(
("tensorflow", "2.5.0", "https://thoth-station.ninja"))
unit = self.UNIT_TESTED()
unit.run(state)
assert not state.justification
def test_run_no_justification(self) -> None:
"""Test NOT adding a link to the PyTorch index."""
state = State()
state.add_resolved_dependency(
("torch", "1.10.2", "https://pypi.org/simple"))
unit = self.UNIT_TESTED()
unit.run(state)
assert not state.justification
def test_run_develop(self, context: Context, state: State,
develop: bool) -> None:
"""Test running this pipeline unit resulting in a justification addition when "not" index url is used."""
prescription_str = f"""
name: GHReleaseNotes
type: wrap.GHReleaseNotes
should_include:
adviser_pipeline: true
match:
state:
resolved_dependencies:
name: thoth-solver
develop: {'true' if develop else 'false'}
run:
release_notes:
organization: thoth-station
repository: solver
tag_version_prefix: v
"""
units = list(self._instantiate_gh_release_notes_wrap(prescription_str))
assert len(
units
) == 1, "Multiple units created, expected only one based on prescription"
unit = units[0]
state.resolved_dependencies.clear()
state.justification.clear()
package_version = PackageVersion(
name="thoth-solver",
version="==0.5.0",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state.add_resolved_dependency(package_version.to_tuple())
context.register_package_version(package_version)
unit.pre_run()
with unit.assigned_context(context):
# Run twice to verify the justification is added just once.
assert unit.run(state) is None
assert unit.run(state) is None
self.verify_justification_schema(state.justification)
assert set(tuple(i.items()) for i in state.justification) == {
(
("type", "INFO"),
("message", "Release notes for package 'thoth-solver'"),
("link",
"https://github.com/thoth-station/solver/releases/tag/v0.5.0"
),
("package_name", "thoth-solver"),
),
}
def test_run_noop(self) -> None:
"""Test no justification added if TensorFlow 2.3 is not resolved."""
state = State()
assert not state.justification
assert "tensorflow" not in state.resolved_dependencies
state.add_resolved_dependency(("tensorflow", "2.2.0", "https://pypi.org/simple"))
unit = TensorFlow23DictSummary()
unit.run(state)
assert len(state.justification) == 0
def test_run_no_justification(self) -> None:
"""Test NOT adding a link to Pulp instance for the given package released."""
index_url = "https://pypi.org/simple"
assert not self.UNIT_TESTED._PULP_URL.startswith(index_url)
state = State()
state.add_resolved_dependency(("tensorflow", "2.5.0", index_url))
unit = self.UNIT_TESTED()
unit.run(state)
assert not state.justification
def test_n_step_td_step_adjust(self, context: Context) -> None:
"""Test adjusting steps taken on reward signal propagation."""
predictor = TemporalDifference(step=1)
predictor._temperature = 1.0
predictor._steps_taken = 1
package_tuple = ("tensorflow", "2.3.1", "https://pypi.org/simple")
state = State()
state.add_resolved_dependency(package_tuple)
with predictor.assigned_context(context):
predictor.set_reward_signal(state, package_tuple, 0.33)
assert predictor._policy.get(package_tuple) == [0.33, 1]
assert predictor._steps_taken == 0
def test_run_develop_state_match(self, context: Context, state: State,
develop: bool,
state_develop: bool) -> None:
"""Test not running the prescription if develop flag is set also on the state match."""
prescription_str = f"""
name: StepUnit
type: step
should_include:
times: 1
adviser_pipeline: true
match:
package_version:
name: numpy
develop: {'true' if develop else 'false'}
state:
resolved_dependencies:
- name: pytest
develop: {'true' if state_develop else 'false'}
run:
score: 0.5
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STEP_SCHEMA(prescription)
StepPrescription.set_prescription(prescription)
package_version = PackageVersion(
name="numpy",
version="==1.19.1",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state_package_version = PackageVersion(
name="pytest",
version="==6.2.4",
index=Source("https://pypi.org/simple"),
develop=state_develop,
)
state.add_resolved_dependency(state_package_version.to_tuple())
context.register_package_version(state_package_version)
unit = StepPrescription()
unit.pre_run()
with unit.assigned_context(context):
result = unit.run(state, package_version)
assert isinstance(result, tuple)
assert result[0] == 0.5
assert result[1] is None
def test_run_add_justification(self) -> None:
"""Test adding link to libraries.io about the given package."""
state = State()
state.add_resolved_dependency(
("tensorflow", "2.5.0", "https://pypi.org/simple"))
unit = self.UNIT_TESTED()
unit.run(state)
assert state.justification == [{
"type": "INFO",
"message": "Information about 'tensorflow' on libraries.io",
"link": "https://libraries.io/pypi/tensorflow/",
"package_name": "tensorflow",
}]
self.verify_justification_schema(state.justification)
def test_run_add_justification(self) -> None:
"""Test adding information Intel's MKL environment variable."""
state = State()
state.add_resolved_dependency(("pytorch", "1.4.0", "https://pypi.org/simple"))
assert len(state.advised_manifest_changes) == 0
assert not state.justification
unit = MKLThreadsWrap()
unit.run(state)
assert len(state.justification) == 1
assert set(state.justification[0].keys()) == {"type", "message", "link"}
assert state.justification[0]["type"] == "WARNING"
assert state.justification[0]["link"], "Empty link to justification document provided"
self._check_advised_manifest_changes(state)
def test_run(self, tf_version: str) -> None:
"""Test adding justification added if TensorFlow 2.3 is resolved."""
state = State()
assert not state.justification
assert "tensorflow" not in state.resolved_dependencies
state.add_resolved_dependency(("tensorflow", tf_version, "https://pypi.org/simple"))
unit = TensorFlow23DictSummary()
unit.run(state)
assert len(state.justification) == 1
assert set(state.justification[0].keys()) == {"type", "message", "link"}
assert state.justification[0]["type"] == "WARNING"
assert state.justification[0]["message"], "No justification message provided"
assert state.justification[0]["link"], "Empty link to justification document provided"
def test_advised_manifest_changes(self, state: State, context: Context) -> None:
"""Test advising changes in the manifest files."""
prescription_str = """
name: WrapUnit
type: wrap
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: intel-tensorflow
run:
advised_manifest_changes:
apiVersion: apps.openshift.io/v1
kind: DeploymentConfig
patch:
op: add
path: /spec/template/spec/containers/0/env/0
value:
name: OMP_NUM_THREADS
value: "1"
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_WRAP_SCHEMA(prescription)
WrapPrescription.set_prescription(prescription)
state.add_resolved_dependency(("intel-tensorflow", "2.2.0", "https://pypi.org/simple"))
state.justification.clear()
assert state.advised_manifest_changes == []
unit = WrapPrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
assert state.advised_manifest_changes == [
{
"apiVersion": "apps.openshift.io/v1",
"kind": "DeploymentConfig",
"patch": {
"op": "add",
"path": "/spec/template/spec/containers/0/env/0",
"value": {"name": "OMP_NUM_THREADS", "value": "1"},
},
}
]
def test_run_add_justification(self, context: Context,
state: State) -> None:
"""Test running this pipeline unit resulting in a justification addition."""
prescription_str = """
name: GHReleaseNotes
type: wrap.GHReleaseNotes
should_include:
adviser_pipeline: true
match:
- state:
resolved_dependencies:
- name: thoth-adviser
run:
release_notes:
organization: thoth-station
repository: adviser
tag_version_prefix: v
"""
units = list(self._instantiate_gh_release_notes_wrap(prescription_str))
assert len(
units
) == 1, "Multiple units created, expected only one based on prescription"
unit = units[0]
state.resolved_dependencies.clear()
state.add_resolved_dependency(
("flask", "1.0.0", "https://pypi.org/simple"))
state.add_resolved_dependency(
("thoth-adviser", "1.0.0", "https://pypi.org/simple"))
state.justification.clear()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
self.verify_justification_schema(state.justification)
assert set(tuple(i.items()) for i in state.justification) == {
(
("type", "INFO"),
("message", "Release notes for package 'thoth-adviser'"),
("link",
"https://github.com/thoth-station/adviser/releases/tag/v1.0.0"
),
("package_name", "thoth-adviser"),
),
}
def test_run(self) -> None:
"""Test running this stride to filter same stacks."""
context = flexmock()
state = State()
state.add_resolved_dependency(("tensorflow", "2.2.0", "https://pypi.org/simple"))
unit = UniqueStackStride()
with unit.assigned_context(context):
assert unit.run(state) is None
# Running the stride on the same stack should cause rejecting it.
with pytest.raises(NotAcceptable):
unit.run(state)
# A stack with another package should be included.
state.add_resolved_dependency(("numpy", "1.19.1", "https://pypi.org/simple"))
assert unit.run(state) is None
def test_run_noop(self, tf_name: str, np_version: str) -> None:
"""Test wrong resolutions are not acceptable."""
package_version = PackageVersion(
name="numpy",
version=f"=={np_version}",
develop=False,
index=Source("https://pypi.org/simple"),
)
state = State()
state.add_resolved_dependency(
(tf_name, "1.13.1", "https://pypi.org/simple"))
unit = TensorFlow113NumPyStep()
assert unit._message_logged is False
assert unit.run(state, package_version) is None
assert unit._message_logged is False
def test_run_match_develop(self, context: Context, state: State, develop: bool) -> None:
"""Test running this pipeline unit based on develop matching."""
prescription_str = f"""
name: StrideUnit
type: stride
should_include:
times: 1
adviser_pipeline: true
match:
state:
resolved_dependencies:
- name: flask
develop: {'true' if develop else 'false'}
run:
stack_info:
- type: INFO
message: This message will be shown
link: https://thoth-station.ninja
"""
prescription = yaml.safe_load(prescription_str)
PRESCRIPTION_STRIDE_SCHEMA(prescription)
StridePrescription.set_prescription(prescription)
package_version = PackageVersion(
name="flask",
version="==2.0.1",
index=Source("https://pypi.org/simple"),
develop=develop,
)
state.add_resolved_dependency(package_version.to_tuple())
context.register_package_version(package_version)
assert not context.stack_info
unit = StridePrescription()
unit.pre_run()
with unit.assigned_context(context):
assert unit.run(state) is None
# Run one more time to verify the stack info is added just once.
assert unit.run(state) is None
assert context.stack_info == [
{"type": "INFO", "message": "This message will be shown", "link": "https://thoth-station.ninja"}
]