class PythonVersionBoot(Boot):
"""A boot that checks Python3 configuration used by user."""
_LINK_PY_VER_PIPFILE = jl("py_version")
_LINK_PY_VER_THOTH_CONF = jl("py_version")
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self, always for adviser."""
if builder_context.is_adviser_pipeline(
) and not builder_context.is_included(cls):
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Check Python configuration used by user."""
python_version = self.context.project.runtime_environment.python_version
pipfile_python_version = self.context.project.pipfile.meta.requires.get(
"python_version")
if pipfile_python_version is not None and python_version is None:
msg = (
f"No version of Python specified in the configuration, using Python version "
f"found in Pipfile: {pipfile_python_version!r}")
_LOGGER.warning("%s - see %s", msg, self._LINK_PY_VER_PIPFILE)
self.context.project.runtime_environment.python_version = pipfile_python_version
self.context.stack_info.append({
"type": "WARNING",
"message": msg,
"link": self._LINK_PY_VER_PIPFILE
})
elif python_version is not None and pipfile_python_version is None:
msg = (
f"No version of Python specified explicitly, assigning the one found in "
f"Thoth's configuration: {python_version!r}")
_LOGGER.warning("%s - see %s", msg, self._LINK_PY_VER_THOTH_CONF)
self.context.project.pipfile.meta.requires[
"python_version"] = python_version
self.context.stack_info.append({
"type": "WARNING",
"message": msg,
"link": self._LINK_PY_VER_THOTH_CONF
})
elif python_version != pipfile_python_version:
msg = (
f"Python version stated in Pipfile ({pipfile_python_version!r}) does not match with the one "
f"specified in the Thoth configuration ({python_version!r}), using Python version from Thoth "
f"configuration implicitly")
_LOGGER.warning("%s - see %s", msg, self._LINK_PY_VER_THOTH_CONF)
self.context.project.pipfile.meta.requires[
"python_version"] = python_version
self.context.stack_info.append({
"type": "WARNING",
"message": msg,
"link": self._LINK_PY_VER_THOTH_CONF
})
def _log_unknown_tf_version(self, package_version: PackageVersion) -> None:
"""Log an unhandled TensorFlow release, this pipeline unit needs an update in such cases."""
message = (
f"Unhandled TensorFlow release {package_version.to_tuple()}, gracefully giving up recommending "
f"TensorFlow based on CUDA version {self.context.project.runtime_environment.cuda_version!r}"
)
_LOGGER.error("%s - see %s", message, jl("cuda_unknown_tf"))
self.context.stack_info.append({
"type": "ERROR",
"message": message,
"link": jl("cuda_unknown_tf"),
})
class PlatformBoot(Boot):
"""A boot that checks for platform used and adjust to the default one if not provided explicitly."""
CONFIGURATION_DEFAULT = {"default_platform": "linux-x86_64"}
CONFIGURATION_SCHEMA = Schema(
{
Required("default_platform"): str,
}
)
_JUSTIFICATION_LINK = jl("platform")
@classmethod
def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]:
"""Register self, always."""
if not builder_context.is_included(cls):
return {}
return None
def run(self) -> None:
"""Check for platform configured and adjust to the default one if not provided by user."""
if self.context.project.runtime_environment.platform is None:
msg = (
f"No platform provided in the configuration, setting to "
f"{self.configuration['default_platform']!r} implicitly"
)
_LOGGER.warning("%s - see %s", msg, self._JUSTIFICATION_LINK)
self.context.project.runtime_environment.platform = self.configuration["default_platform"]
self.context.stack_info.append({"type": "WARNING", "message": msg, "link": self._JUSTIFICATION_LINK})
platform = self.context.project.runtime_environment.platform
if not self.context.graph.python_package_version_depends_on_platform_exists(platform):
raise NotAcceptable(f"No platform conforming to {platform!r} found in the database")
class UbiBoot(Boot):
"""Remap UBI to RHEL.
As UBI has ABI compatibility with RHEL, remap any UBI to RHEL.
"""
_MESSAGE = "Using observations for RHEL instead of UBI, RHEL is ABI compatible with UBI"
_JUSTIFICATION_LINK = jl("rhel_ubi")
@classmethod
def should_include(cls, builder_context: "PipelineBuilderContext") -> Generator[Dict[str, Any], None, None]:
"""Register self if UBI is used."""
if (
builder_context.project.runtime_environment.operating_system.name == "ubi"
and not builder_context.is_included(cls)
):
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Remap UBI to RHEL as Thoth keeps track of RHEL and UBI is ABI compatible."""
_LOGGER.info("%s - see %s", self._MESSAGE, self._JUSTIFICATION_LINK)
self.context.stack_info.append({"type": "WARNING", "message": self._MESSAGE, "link": self._JUSTIFICATION_LINK})
self.context.project.runtime_environment.operating_system.name = "rhel"
class TensorFlowPython39Sieve(Sieve):
"""A sieve that makes sure the right TensorFlow version is used when running on Python 3.9."""
CONFIGURATION_DEFAULT = {"package_name": "tensorflow"}
_MESSAGE = "TensorFlow releases that do not support Python 3.9 filtered out"
_LINK = jl("tf_py39")
_message_logged = attr.ib(type=bool, default=False, init=False)
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Optional[Dict[str, Any]]:
"""Register this pipeline unit for adviser when CUDA is present."""
if not builder_context.is_adviser_pipeline():
return None
if builder_context.recommendation_type in (RecommendationType.LATEST,
RecommendationType.TESTING):
# Use any TensorFlow for testing purposes or when resolving latest stack.
return None
python_version = builder_context.project.runtime_environment.python_version
if python_version != "3.9":
return None
# Include this pipeline units in different configurations for tensorflow, intel-tensorflow and tensorflow-gpu.
included_units = builder_context.get_included_sieves(cls)
if len(included_units) == 3:
return None
if len(included_units) == 0:
return {"package_name": "tensorflow"}
elif len(included_units) == 1:
return {"package_name": "tensorflow-gpu"}
elif len(included_units) == 2:
return {"package_name": "intel-tensorflow"}
return None
def pre_run(self) -> None:
"""Initialize this pipeline unit before each run."""
self._message_logged = False
super().pre_run()
def run(
self, package_versions: Generator[PackageVersion, None, None]
) -> Generator[PackageVersion, None, None]:
"""Use specific TensorFlow release based on Python version present in the runtime environment."""
for package_version in package_versions:
if package_version.semantic_version.release[:2] <= (2, 4):
if not self._message_logged:
_LOGGER.warning("%s - %s", self._MESSAGE, self._LINK)
self.context.stack_info.append({
"type": "WARNING",
"message": self._MESSAGE,
"link": self._LINK,
})
continue
yield package_version
class LabelsBoot(Boot):
"""A boot to notify about labels used during the resolution."""
_JUSTIFICATION_LINK = jl("labels")
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self if labels are used."""
if not builder_context.is_included(cls) and builder_context.labels:
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Notify about labels used during the resolution process.."""
for key, value in self.context.labels.items():
msg = f"Considering label {key}={value} in the resolution process"
_LOGGER.info("%s - see %s", msg, self._JUSTIFICATION_LINK)
self.context.stack_info.append({
"message": msg,
"type": "INFO",
"link": self._JUSTIFICATION_LINK,
})
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register this pipeline unit for adviser when CUDA is present."""
cuda_version = builder_context.project.runtime_environment.cuda_version
if (builder_context.is_adviser_pipeline()
and not builder_context.is_included(cls)
and builder_context.recommendation_type
not in (RecommendationType.LATEST, RecommendationType.TESTING)
and cuda_version is not None):
if cuda_version not in cls._KNOWN_CUDA:
_LOGGER.warning(
"Unable to perform recommendations for TensorFlow based on CUDA version: "
"unknown CUDA version supplied - see %s",
jl("tf_unknown_cuda"),
)
yield from ()
return None
yield {"package_name": "tensorflow"}
yield {"package_name": "tensorflow-gpu"}
return None
yield from ()
return None
def test_not_acceptable(self, context: Context) -> None:
"""Test raising an exception when the pipeline unit is included."""
unit = self.UNIT_TESTED()
context.project.runtime_environment.platform = "aarch64"
assert not context.stack_info
# The unit always raises, no need to explicitly assign platform here.
with unit.assigned_context(context):
with pytest.raises(
NotAcceptable,
match=
"Platform 'aarch64' is not supported, possible platforms are: linux-x86_64"
):
unit.run()
assert self.verify_justification_schema(context.stack_info)
assert context.stack_info == [
{
"link":
jl("platform"),
"message":
"Platform 'aarch64' is not supported, possible platforms are: "
"linux-x86_64",
"type":
"ERROR",
},
]
def _prepare_justification_link(cls, entries: List[Dict[str,
Any]]) -> None:
"""Prepare justification links before using them."""
for entry in entries:
link = entry.get("link")
if link and not link.startswith(("https://", "http://")):
entry["link"] = jl(link)
def test_python_version_mismatch(self) -> None:
"""Test when python version stated in Pipfile does not match with the one provided in the configuration."""
context = flexmock(project=Project.from_strings(
self._CASE_PIPFILE_PYTHON),
stack_info=[])
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = "8"
context.project.runtime_environment.python_version = "3.8"
boot = PythonVersionBoot()
with PythonVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version == "8"
assert context.project.runtime_environment.python_version == "3.8"
assert context.stack_info == [{
"message":
"Python version stated in Pipfile ('3.6') does not match with the one "
"specified in the Thoth configuration ('3.8'), using Python version from Thoth "
"configuration implicitly",
"type":
"WARNING",
"link":
jl("py_version"),
}]
def test_pre_run_no_symbols_warning(self, context: Context) -> None:
"""Make sure the pre_run method produces a warning for users."""
context.graph.should_receive("get_thoth_s2i_analyzed_image_symbols_all").with_args(
thoth_s2i_image_name="quay.io/thoth-station/s2i-thoth-ubi8-py38",
thoth_s2i_image_version="1.0.0",
is_external=False,
).and_return(set()).once()
context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v1.0.0"
assert not context.stack_info
unit = self.UNIT_TESTED()
with unit.assigned_context(context):
unit.pre_run()
assert len(context.stack_info) == 1
assert self.verify_justification_schema(context.stack_info)
assert context.stack_info == [
{
"link": jl("no_abi"),
"message": "No ABI symbols found for " "'quay.io/thoth-station/s2i-thoth-ubi8-py38' in version '1.0.0'",
"type": "WARNING",
},
]
class SolversConfiguredBoot(Boot):
"""A boot to notify about runtime environments not supported by solvers enabled in a deployment."""
_SOLVERS_CONFIGURED = os.getenv(
"THOTH_ADVISER_DEPLOYMENT_CONFIGURED_SOLVERS")
_JUSTIFICATION = jl("eol_env")
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self if users use runtime environments not matching solvers in the configmap."""
if not cls._SOLVERS_CONFIGURED or builder_context.is_included(cls):
yield from ()
return None
runtime_environment = (
builder_context.project.runtime_environment.operating_system.name,
builder_context.project.runtime_environment.operating_system.
version,
builder_context.project.python_version,
)
for solver_name in cls._SOLVERS_CONFIGURED.splitlines():
solver_name = solver_name.strip()
if not solver_name:
continue
try:
solver_info = OpenShift.parse_python_solver_name(solver_name)
except SolverNameParseError:
_LOGGER.exception(
"Please report this error to Thoth service administrator")
yield from ()
return None
if runtime_environment == (
solver_info["os_name"],
solver_info["os_version"],
solver_info["python_version"],
):
break
else:
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Inform about runtime environment that is not backed up by a solver."""
self.context.stack_info.append({
"type":
"WARNING",
"message":
"Runtime environment used is no longer supported, it is "
"recommended to switch to another runtime environment",
"link":
self._JUSTIFICATION,
})
class ThothS2IBoot(Boot):
"""A boot that notifies about missing observations."""
_THOTH_S2I_PREFIX = "quay.io/thoth-station/s2i-thoth-"
_JUSTIFICATION = [{
"type": "INFO",
"message":
"It is recommended to use Thoth's s2i to have recommendations specific to runtime environment",
"link": jl("thoth_s2i"),
}]
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[Any, Any], None, None]:
"""Include this boot in adviser if Thoth s2i is not used.."""
base_image = builder_context.project.runtime_environment.base_image
if not builder_context.is_included(cls) and (
base_image is None or
(base_image and not base_image.startswith(cls._THOTH_S2I_PREFIX))):
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Check for no observations made on the given state."""
self.context.stack_info.extend(self._JUSTIFICATION)
def test_no_python_pipfile(self) -> None:
"""Test assigning Python version from Pipfile."""
context = flexmock(project=Project.from_strings(
self._CASE_PIPFILE_NO_PYTHON),
stack_info=[])
context.project.runtime_environment.operating_system.name = "rhel"
context.project.runtime_environment.operating_system.version = "8"
context.project.runtime_environment.python_version = "3.6"
boot = PythonVersionBoot()
with PythonVersionBoot.assigned_context(context):
boot.run()
assert context.project.runtime_environment.operating_system.name == "rhel"
assert context.project.runtime_environment.operating_system.version == "8"
assert context.project.runtime_environment.python_version == "3.6"
assert context.stack_info == [{
"message":
"No version of Python specified explicitly, assigning the one "
"found in Thoth's configuration: '3.6'",
"type":
"WARNING",
"link":
jl("py_version"),
}]
def _load_files(requirements_format: str) -> Tuple[str, Optional[str]]:
"""Load Pipfile/Pipfile.lock or requirements.in/txt from the current directory."""
if requirements_format == "pipenv":
_LOGGER.info("Using Pipenv files located in %r directory", os.getcwd())
pipfile_lock_exists = os.path.exists("Pipfile.lock")
if pipfile_lock_exists:
_LOGGER.info(
"Submitting Pipfile.lock as a base for user's stack scoring - see %s",
jl("user_stack"),
)
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] !=
project.pipfile.hash()["sha256"]):
_LOGGER.error(
"Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile "
"adjusted? This error is not critical.",
project.pipfile_lock.meta.hash["sha256"][:6],
project.pipfile.hash()["sha256"][:6],
)
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
_LOGGER.info("Using requirements.txt file located in %r directory",
os.getcwd())
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
return (
project.pipfile.to_string(),
project.pipfile_lock.to_string() if project.pipfile_lock else None,
)
class PlatformBoot(Boot):
"""A boot to check if a supported platform is used.
We could check this based on the database entries, but as this will change rarely, we can
hardcode supported platforms here.
"""
_JUSTIFICATION_LINK = jl("platform")
_SUPPORTED_PLATFORMS = frozenset({"linux-x86_64"})
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self, always on unsupported platform."""
if (builder_context.project.runtime_environment.platform
not in cls._SUPPORTED_PLATFORMS
and not builder_context.is_included(cls)):
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Check if GPU enabled recommendations should be done."""
platform = self.context.project.runtime_environment.platform
msg = f"Platform {platform!r} is not supported, possible platforms are: " + ", ".join(
self._SUPPORTED_PLATFORMS)
self.context.stack_info.append({
"type": "ERROR",
"message": msg,
"link": self._JUSTIFICATION_LINK,
})
raise NotAcceptable(f"{msg} - see {self._JUSTIFICATION_LINK}")
class NoObservationWrap(Wrap):
"""A wrap that notifies about missing observations."""
_JUSTIFICATION = [{
"type": "INFO",
"message":
"No issues spotted for this stack based on Thoth's database",
"link": jl("no_observations"),
}]
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[Any, Any], None, None]:
"""Include this wrap in adviser, once."""
if not builder_context.is_included(
cls) and builder_context.is_adviser_pipeline():
yield {}
return None
yield from ()
return None
def run(self, state: State) -> None:
"""Check for no observations made on the given state."""
if not state.justification:
state.add_justification(self._JUSTIFICATION)
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Optional[Dict[str, Any]]:
"""Register this pipeline unit for adviser when CUDA is present."""
if not builder_context.is_adviser_pipeline():
return None
if builder_context.recommendation_type in (RecommendationType.LATEST,
RecommendationType.TESTING):
# Use any TensorFlow for testing purposes or when resolving latest stack.
return None
cuda_version = builder_context.project.runtime_environment.cuda_version
if cuda_version is None:
# No CUDA available in the given runtime environment.
return None
if cuda_version not in cls._KNOWN_CUDA:
_LOGGER.warning(
"Unable to perform recommendations for TensorFlow based on CUDA version: "
"unknown CUDA version supplied - see %s",
jl("tf_unknown_cuda"),
)
return None
# Include this pipeline units in two configurations for tensorflow and tensorflow-gpu.
included_units = builder_context.get_included_sieves(cls)
if len(included_units) == 2:
return None
if len(included_units) == 0:
return {"package_name": "tensorflow"}
elif len(included_units) == 1:
return {"package_name": "tensorflow-gpu"}
return None
class TensorFlowGPUPseudonym(Pseudonym):
"""A TensorFlow pseudonym to map tensorflow to tensorflow-gpu packages."""
CONFIGURATION_DEFAULT = {"package_name": "tensorflow"}
_LINK = jl("tf_gpu_alt")
_pseudonyms = attr.ib(type=Optional[FrozenSet[str]], default=None, init=False)
@classmethod
def should_include(cls, builder_context: "PipelineBuilderContext") -> Generator[Dict[str, Any], None, None]:
"""Register self."""
if (
builder_context.project.runtime_environment.cuda_version is not None
and builder_context.is_adviser_pipeline()
and builder_context.recommendation_type != RecommendationType.LATEST
and not builder_context.is_included(cls)
):
yield {}
return None
yield from ()
return None
def pre_run(self) -> None:
"""Initialize this pipeline unit before each run."""
self._pseudonyms = None
super().pre_run()
def run(self, package_version: PackageVersion) -> Generator[Tuple[str, str, str], None, None]:
"""Map TensorFlow packages to their alternatives."""
if package_version.index.url != "https://pypi.org/simple" or package_version.semantic_version.release[0] > 1:
return None
if self._pseudonyms is None:
# Be lazy with queries to the database.
_LOGGER.warning(
"Considering also tensorflow-gpu package as the runtime environment used provides CUDA - see %s",
self._LINK,
)
runtime_environment = self.context.project.runtime_environment
self._pseudonyms = frozenset(
{
i[1]
for i in self.context.graph.get_solved_python_package_versions_all(
package_name="tensorflow-gpu",
package_version=None,
index_url="https://pypi.org/simple",
count=None,
os_name=runtime_environment.operating_system.name,
os_version=runtime_environment.operating_system.version,
python_version=runtime_environment.python_version,
distinct=True,
is_missing=False,
)
}
)
if package_version.locked_version in self._pseudonyms:
yield "tensorflow-gpu", package_version.locked_version, "https://pypi.org/simple"
class IntelTensorFlowWrap(Wrap):
"""A wrap that recommends using Intel TensorFlow if TensorFlow is in resolved dependencies.
https://software.intel.com/content/www/us/en/develop/articles/intel-optimization-for-tensorflow-installation-guide.html#pip_wheels
"""
CONFIGURATION_DEFAULT = {"package_name": "tensorflow"}
# Sandy bridge CPUID taken from https://en.wikipedia.org/wiki/Sandy_Bridge
# Ivy bridge CPUID taken from https://en.wikipedia.org/wiki/Ivy_Bridge_(microarchitecture)
#
# As CPUID encodes family (bits 8 - 11 with mask 0xF00) and model (bits 4 - 7 with mask 0xF0) we got these
# numbers from CPUID.
_CPU_TABLE = frozenset({
# tuple (model, family)
# Sandy Bridge-HE-4, Sandy Bridge-H-2, Sandy Bridge-M-2
# ((0x0206A7 & 0xF0) >> 4, (0x0206A7 & 0xF00) >> 8),
# Sandy Bridge - EP - 8
# ((0x0206D6 & 0xF0) >> 4, (0x0206D6 & 0xF00) >> 8),
# ((0x0206D7 & 0xF0) >> 4, (0x0206D7 & 0xF00) >> 8),
# Sandy Bridge - EP - 4
# ((0x0206D6 & 0xF0) >> 4, (0x0206D6 & 0xF00) >> 8),
# ((0x0206D7 & 0xF0) >> 4, (0x0206D7 & 0xF00) >> 8),
# Ivy Bridge-M-2, Ivy Bridge-H-2, Ivy Bridge-HM-4, Ivy Bridge-HE-4
# ((0x000306A9 & 0xF0) >> 4, (0x000306A9 & 0xF00) >> 8),
# All maps to the following values:
(13, 6),
(10, 6),
})
_JUSTIFICATION = [{
"type": "INFO",
"message":
"Consider using intel-tensorflow which is optimized for CPU detected in your environment",
"link": jl("intel_tensorflow"),
}]
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Optional[Dict[str, Any]]:
"""Include this wrap for x86_64 architecture on CPU models with Ivy/Sandy bridge."""
if builder_context.is_included(cls):
return None
if not builder_context.is_adviser_pipeline():
return None
runtime_environment = builder_context.project.runtime_environment
cpu_tuple = (runtime_environment.hardware.cpu_model,
runtime_environment.hardware.cpu_family)
if runtime_environment.platform == "linux-x86_64" and cpu_tuple in cls._CPU_TABLE:
return {}
return None
def run(self, state: State) -> None:
"""Recommend using intel-tensorflow if tensorflow is resolved."""
if "intel-tensorflow" not in state.resolved_dependencies:
state.add_justification(self._JUSTIFICATION)
class LabelsBoot(Boot):
"""A boot to notify about labels used during the resolution."""
_JUSTIFICATION_LINK_LABELS = jl("labels")
_JUSTIFICATION_LINK_ALLOW_CVE = jl("allow_cve")
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self if labels are used."""
if not builder_context.is_included(cls) and builder_context.labels:
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Notify about labels used during the resolution process.."""
for key, value in self.context.labels.items():
if key != "allow-cve":
msg = f"Considering label {key}={value} in the resolution process"
_LOGGER.info("%s - see %s", msg,
self._JUSTIFICATION_LINK_LABELS)
self.context.stack_info.append({
"message":
msg,
"type":
"INFO",
"link":
self._JUSTIFICATION_LINK_LABELS,
})
else:
for allow_cve in value.split(","):
msg = f"Allowing CVE {allow_cve.upper()!r} to be present in the application"
_LOGGER.warning("%s - see %s", msg,
self._JUSTIFICATION_LINK_ALLOW_CVE)
self.context.stack_info.append({
"message":
msg,
"type":
"WARNING",
"link":
self._JUSTIFICATION_LINK_ALLOW_CVE,
})
class SolvedSoftwareEnvironmentBoot(Boot):
"""A boot to check for solved software environment before running any resolution."""
_JUSTIFICATION_LINK = jl("solved_sw_env")
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self, always."""
if builder_context.project.runtime_environment.is_fully_specified(
) and not builder_context.is_included(cls):
yield {}
return None
yield from ()
return None
def run(self) -> None:
"""Check for version clash in packages."""
if self.context.graph.solved_software_environment_exists(
os_name=self.context.project.runtime_environment.
operating_system.name,
os_version=self.context.project.runtime_environment.
operating_system.version,
python_version=self.context.project.runtime_environment.
python_version,
):
return
runtime_environment = self.context.project.runtime_environment
msg = (
f"No observations found for {runtime_environment.operating_system.name!r} in "
f"version {runtime_environment.operating_system.version!r} using "
f"Python {runtime_environment.python_version!r}")
self.context.stack_info.append({
"type": "ERROR",
"message": msg,
"link": self._JUSTIFICATION_LINK,
})
_LOGGER.warning("%s - %s", msg, self._JUSTIFICATION_LINK)
_LOGGER.warning("Available configurations:")
configurations = self.context.graph.get_solved_python_package_versions_software_environment_all(
)
_LOGGER.warning("{:<16} {:<16} {:<8}".format("OS name", "OS version",
"Python version"))
for conf in sorted(
configurations,
key=lambda i:
(i["os_name"], i["os_version"], i["python_version"]),
):
_LOGGER.warning("{:<16} {:<16} {:<8}".format(
conf["os_name"], conf["os_version"], conf["python_version"]))
raise NotAcceptable(msg)
class TensorFlow22ProbabilityStep(Step):
"""Suggest not to use TensorFlow 2.2 with tensorflow-probability.
https://github.com/tensorflow/tensorflow/issues/40584
"""
CONFIGURATION_DEFAULT = {
"package_name": "tensorflow-probability",
"multi_package_resolution": False
}
_MESSAGE = "TensorFlow in version 2.2 and tensorflow-probability cause runtime errors"
_LINK = jl("tf_40584")
_message_logged = attr.ib(type=bool, default=False, init=False)
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Optional[Dict[str, Any]]:
"""Register this pipeline unit for adviser if not using latest recommendations."""
if (not builder_context.is_adviser_pipeline()
or builder_context.recommendation_type
== RecommendationType.LATEST):
return None
if not builder_context.is_included(cls):
return {}
return None
def pre_run(self) -> None:
"""Initialize this pipeline unit before each run."""
self._message_logged = False
super().pre_run()
def run(
self, state: State, package_version: PackageVersion
) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]:
"""Suggest not to use TensorFlow 2.2 with with tensorflow-probability."""
tensorflow_any = (state.resolved_dependencies.get("tensorflow")
or state.resolved_dependencies.get("tensorflow-cpu")
or state.resolved_dependencies.get("tensorflow-gpu"))
if not tensorflow_any or (tensorflow_any[1] != "2.2" and
not tensorflow_any[1].startswith("2.2.")):
return None
if not self._message_logged:
self._message_logged = True
_LOGGER.warning("%s - see %s", self._MESSAGE, self._LINK)
self.context.stack_info.append({
"type": "WARNING",
"message": self._MESSAGE,
"link": self._LINK
})
raise NotAcceptable
class TensorFlowSlowKerasEmbedding(Wrap):
"""A wrap that notifies a bug in the embedding layer.
https://github.com/tensorflow/tensorflow/issues/42475
"""
CONFIGURATION_DEFAULT = {"package_name": "tensorflow"}
_JUSTIFICATION = [
{
"type": "WARNING",
"message": "TensorFlow in version <=2.4 is slow when tf.keras.layers.Embedding is used",
"link": jl("tf_42475"),
}
]
@classmethod
def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]:
"""Include this wrap in adviser."""
if not builder_context.is_adviser_pipeline():
return None
if builder_context.library_usage is not None and "tensorflow.keras.layers.Embedding" not in (
builder_context.library_usage.get("tensorflow") or []
):
return None
units_included = builder_context.get_included_wraps(cls)
if len(units_included) == 4:
return None
elif len(units_included) == 0:
return {"package_name": "tensorflow"}
elif len(units_included) == 1:
return {"package_name": "tensorflow-cpu"}
elif len(units_included) == 2:
return {"package_name": "tensorflow-gpu"}
elif len(units_included) == 3:
return {"package_name": "intel-tensorflow"}
return None
def run(self, state: State) -> None:
"""Notify about a bug in summary output spotted on TensorFlow 2.3."""
tensorflow_any = (
state.resolved_dependencies.get("tensorflow")
or state.resolved_dependencies.get("tensorflow-cpu")
or state.resolved_dependencies.get("tensorflow-gpu")
or state.resolved_dependencies.get("intel-tensorflow")
)
if tensorflow_any is None:
return None
tf_package_version: PackageVersion = self.context.get_package_version(tensorflow_any)
if tf_package_version.semantic_version.release[:2] <= (2, 4):
state.add_justification(self._JUSTIFICATION)
class TensorFlow23Accuracy(Wrap):
"""A wrap that notifies about accuracy bug on safe()/load_model() calls.
https://github.com/tensorflow/tensorflow/issues/42045
https://github.com/keras-team/keras/issues/14181
https://github.com/tensorflow/tensorflow/commit/5adacc88077ef82f6c4a7f9bb65f9ed89f9d8947
"""
CONFIGURATION_DEFAULT = {"package_name": "tensorflow"}
_JUSTIFICATION = [{
"type":
"WARNING",
"message":
"TensorFlow in version 2.3 produces wrong model accuracy when the model is "
"serialized using `accuracy`, use `sparse_categorical_accuracy` instead",
"link":
jl("tf_42045"),
}]
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Optional[Dict[str, Any]]:
"""Include this wrap in adviser."""
if not builder_context.is_adviser_pipeline():
return None
units_included = builder_context.get_included_wraps(cls)
if len(units_included) == 4:
return None
elif len(units_included) == 0:
return {"package_name": "tensorflow"}
elif len(units_included) == 1:
return {"package_name": "tensorflow-cpu"}
elif len(units_included) == 2:
return {"package_name": "tensorflow-gpu"}
elif len(units_included) == 3:
return {"package_name": "intel-tensorflow"}
return None
def run(self, state: State) -> None:
"""Notify about accuracy bug in safe()/load_model() calls."""
tensorflow_any = (state.resolved_dependencies.get("tensorflow")
or state.resolved_dependencies.get("tensorflow-cpu")
or state.resolved_dependencies.get("tensorflow-gpu")
or
state.resolved_dependencies.get("intel-tensorflow"))
if tensorflow_any is None:
return None
if tensorflow_any[1] == "2.3" or tensorflow_any[1].startswith("2.3."):
state.add_justification(self._JUSTIFICATION)
class PandasPy36Sieve(Sieve):
"""A sieve that makes sure Pandas>=1.2 is not used on Python 3.6 or older.
See https://github.com/pandas-dev/pandas/pull/35214
"""
CONFIGURATION_DEFAULT = {"package_name": "pandas"}
_MESSAGE = f"Pandas in versions >=1.2 dropped Python 3.6 support"
_JUSTIFICATION_LINK = jl("pandas_py36_drop")
_message_logged = attr.ib(type=bool, default=False, init=False)
def pre_run(self) -> None:
"""Initialize this pipeline unit before each run."""
self._message_logged = False
super().pre_run()
@classmethod
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Optional[Dict[str, Any]]:
"""Register this pipeline unit for adviser when Python 3.6 is used, not latest/testing recommendations."""
if not builder_context.is_adviser_pipeline(
) or builder_context.is_included(cls):
return None
if builder_context.recommendation_type in (RecommendationType.LATEST,
RecommendationType.TESTING):
return None
if builder_context.project.runtime_environment.get_python_version_tuple(
) > (3, 6):
# Not a Python 3.6 or older environment.
return None
return {}
def run(
self, package_versions: Generator[PackageVersion, None, None]
) -> Generator[PackageVersion, None, None]:
"""Do not use Pandas>=1.2 on Python 3.6."""
for package_version in package_versions:
if package_version.semantic_version.release[:2] < (1, 2):
yield package_version
elif not self._message_logged:
self._message_logged = True
_LOGGER.warning("%s - see %s", self._MESSAGE,
self._JUSTIFICATION_LINK)
self.context.stack_info.append({
"type": "WARNING",
"message": self._MESSAGE,
"link": self._JUSTIFICATION_LINK
})
def test_run(self, context: Context) -> None:
"""Test if the given software environment is solved."""
context.labels = {
"foo": "bar",
"baz": "qux",
"allow-cve": "pysec-2014-10,PYSEC-2014-22"
}
assert not context.stack_info
unit = self.UNIT_TESTED()
with unit.assigned_context(context):
assert unit.run() is None
assert context.stack_info == [
{
"link": jl("labels"),
"message":
"Considering label foo=bar in the resolution process",
"type": "INFO"
},
{
"link": jl("labels"),
"message":
"Considering label baz=qux in the resolution process",
"type": "INFO"
},
{
"link": jl("allow_cve"),
"message":
"Allowing CVE 'PYSEC-2014-10' to be present in the application",
"type": "WARNING",
},
{
"link": jl("allow_cve"),
"message":
"Allowing CVE 'PYSEC-2014-22' to be present in the application",
"type": "WARNING",
},
]
def run(self) -> None:
"""Check for version clash in packages."""
if not self.context.project.runtime_environment.is_fully_specified():
msg = (
f"Software environment supplied is not fully specified, OS name "
f"is {self.context.project.runtime_environment.operating_system.name!r} "
f"in version {self.context.project.runtime_environment.operating_system.version!r} "
f"using Python {self.context.project.runtime_environment.python_version!r}"
)
self.context.stack_info.append({"message": msg, "type": "ERROR", "link": jl("fully_specified_env")})
raise NotAcceptable(msg)
def run(self) -> None:
"""Discard any minor release in RHEL."""
os_name = self.context.project.runtime_environment.operating_system.name
os_version = self.context.project.runtime_environment.operating_system.version
if os_name == "rhel" and os_version is not None:
version_parts = os_version.split(".", maxsplit=1)
if len(version_parts) > 1:
_LOGGER.info(
"RHEL major releases guarantee ABI compatibility across minor releases; "
"discarding minor release information and using RHEL version %r - see %s",
version_parts[0],
jl("rhel_version"),
)
self.context.project.runtime_environment.operating_system.version = version_parts[0]
def run(self) -> None:
"""Check CVE timestamp and provide it to users."""
cve_timestamp = self.context.graph.get_cve_timestamp()
if cve_timestamp is None:
message = "No CVE timestamp information found in the database"
self.context.stack_info.append(
{
"message": message,
"type": "WARNING",
"link": jl("no_cve_timestamp"),
}
)
return
days = (datetime.utcnow() - cve_timestamp).days
self.context.stack_info.append(
{
"type": "WARNING" if days > self._CVE_TIMESTAMP_DAYS_WARNING else "INFO",
"message": f"CVE database of known vulnerabilities for Python packages was "
f"updated at {datetime2datetime_str(cve_timestamp)!r}",
"link": jl("cve_timestamp"),
}
)
