class PythonVersionBoot(Boot): """A boot that checks Python3 configuration used by user.""" _LINK_PY_VER_PIPFILE = jl("py_version") _LINK_PY_VER_THOTH_CONF = jl("py_version") @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register self, always for adviser.""" if builder_context.is_adviser_pipeline( ) and not builder_context.is_included(cls): yield {} return None yield from () return None def run(self) -> None: """Check Python configuration used by user.""" python_version = self.context.project.runtime_environment.python_version pipfile_python_version = self.context.project.pipfile.meta.requires.get( "python_version") if pipfile_python_version is not None and python_version is None: msg = ( f"No version of Python specified in the configuration, using Python version " f"found in Pipfile: {pipfile_python_version!r}") _LOGGER.warning("%s - see %s", msg, self._LINK_PY_VER_PIPFILE) self.context.project.runtime_environment.python_version = pipfile_python_version self.context.stack_info.append({ "type": "WARNING", "message": msg, "link": self._LINK_PY_VER_PIPFILE }) elif python_version is not None and pipfile_python_version is None: msg = ( f"No version of Python specified explicitly, assigning the one found in " f"Thoth's configuration: {python_version!r}") _LOGGER.warning("%s - see %s", msg, self._LINK_PY_VER_THOTH_CONF) self.context.project.pipfile.meta.requires[ "python_version"] = python_version self.context.stack_info.append({ "type": "WARNING", "message": msg, "link": self._LINK_PY_VER_THOTH_CONF }) elif python_version != pipfile_python_version: msg = ( f"Python version stated in Pipfile ({pipfile_python_version!r}) does not match with the one " f"specified in the Thoth configuration ({python_version!r}), using Python version from Thoth " f"configuration implicitly") _LOGGER.warning("%s - see %s", msg, self._LINK_PY_VER_THOTH_CONF) self.context.project.pipfile.meta.requires[ "python_version"] = python_version self.context.stack_info.append({ "type": "WARNING", "message": msg, "link": self._LINK_PY_VER_THOTH_CONF })
def _log_unknown_tf_version(self, package_version: PackageVersion) -> None: """Log an unhandled TensorFlow release, this pipeline unit needs an update in such cases.""" message = ( f"Unhandled TensorFlow release {package_version.to_tuple()}, gracefully giving up recommending " f"TensorFlow based on CUDA version {self.context.project.runtime_environment.cuda_version!r}" ) _LOGGER.error("%s - see %s", message, jl("cuda_unknown_tf")) self.context.stack_info.append({ "type": "ERROR", "message": message, "link": jl("cuda_unknown_tf"), })
class PlatformBoot(Boot): """A boot that checks for platform used and adjust to the default one if not provided explicitly.""" CONFIGURATION_DEFAULT = {"default_platform": "linux-x86_64"} CONFIGURATION_SCHEMA = Schema( { Required("default_platform"): str, } ) _JUSTIFICATION_LINK = jl("platform") @classmethod def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]: """Register self, always.""" if not builder_context.is_included(cls): return {} return None def run(self) -> None: """Check for platform configured and adjust to the default one if not provided by user.""" if self.context.project.runtime_environment.platform is None: msg = ( f"No platform provided in the configuration, setting to " f"{self.configuration['default_platform']!r} implicitly" ) _LOGGER.warning("%s - see %s", msg, self._JUSTIFICATION_LINK) self.context.project.runtime_environment.platform = self.configuration["default_platform"] self.context.stack_info.append({"type": "WARNING", "message": msg, "link": self._JUSTIFICATION_LINK}) platform = self.context.project.runtime_environment.platform if not self.context.graph.python_package_version_depends_on_platform_exists(platform): raise NotAcceptable(f"No platform conforming to {platform!r} found in the database")
class UbiBoot(Boot): """Remap UBI to RHEL. As UBI has ABI compatibility with RHEL, remap any UBI to RHEL. """ _MESSAGE = "Using observations for RHEL instead of UBI, RHEL is ABI compatible with UBI" _JUSTIFICATION_LINK = jl("rhel_ubi") @classmethod def should_include(cls, builder_context: "PipelineBuilderContext") -> Generator[Dict[str, Any], None, None]: """Register self if UBI is used.""" if ( builder_context.project.runtime_environment.operating_system.name == "ubi" and not builder_context.is_included(cls) ): yield {} return None yield from () return None def run(self) -> None: """Remap UBI to RHEL as Thoth keeps track of RHEL and UBI is ABI compatible.""" _LOGGER.info("%s - see %s", self._MESSAGE, self._JUSTIFICATION_LINK) self.context.stack_info.append({"type": "WARNING", "message": self._MESSAGE, "link": self._JUSTIFICATION_LINK}) self.context.project.runtime_environment.operating_system.name = "rhel"
class TensorFlowPython39Sieve(Sieve): """A sieve that makes sure the right TensorFlow version is used when running on Python 3.9.""" CONFIGURATION_DEFAULT = {"package_name": "tensorflow"} _MESSAGE = "TensorFlow releases that do not support Python 3.9 filtered out" _LINK = jl("tf_py39") _message_logged = attr.ib(type=bool, default=False, init=False) @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Optional[Dict[str, Any]]: """Register this pipeline unit for adviser when CUDA is present.""" if not builder_context.is_adviser_pipeline(): return None if builder_context.recommendation_type in (RecommendationType.LATEST, RecommendationType.TESTING): # Use any TensorFlow for testing purposes or when resolving latest stack. return None python_version = builder_context.project.runtime_environment.python_version if python_version != "3.9": return None # Include this pipeline units in different configurations for tensorflow, intel-tensorflow and tensorflow-gpu. included_units = builder_context.get_included_sieves(cls) if len(included_units) == 3: return None if len(included_units) == 0: return {"package_name": "tensorflow"} elif len(included_units) == 1: return {"package_name": "tensorflow-gpu"} elif len(included_units) == 2: return {"package_name": "intel-tensorflow"} return None def pre_run(self) -> None: """Initialize this pipeline unit before each run.""" self._message_logged = False super().pre_run() def run( self, package_versions: Generator[PackageVersion, None, None] ) -> Generator[PackageVersion, None, None]: """Use specific TensorFlow release based on Python version present in the runtime environment.""" for package_version in package_versions: if package_version.semantic_version.release[:2] <= (2, 4): if not self._message_logged: _LOGGER.warning("%s - %s", self._MESSAGE, self._LINK) self.context.stack_info.append({ "type": "WARNING", "message": self._MESSAGE, "link": self._LINK, }) continue yield package_version
class LabelsBoot(Boot): """A boot to notify about labels used during the resolution.""" _JUSTIFICATION_LINK = jl("labels") @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register self if labels are used.""" if not builder_context.is_included(cls) and builder_context.labels: yield {} return None yield from () return None def run(self) -> None: """Notify about labels used during the resolution process..""" for key, value in self.context.labels.items(): msg = f"Considering label {key}={value} in the resolution process" _LOGGER.info("%s - see %s", msg, self._JUSTIFICATION_LINK) self.context.stack_info.append({ "message": msg, "type": "INFO", "link": self._JUSTIFICATION_LINK, })
def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register this pipeline unit for adviser when CUDA is present.""" cuda_version = builder_context.project.runtime_environment.cuda_version if (builder_context.is_adviser_pipeline() and not builder_context.is_included(cls) and builder_context.recommendation_type not in (RecommendationType.LATEST, RecommendationType.TESTING) and cuda_version is not None): if cuda_version not in cls._KNOWN_CUDA: _LOGGER.warning( "Unable to perform recommendations for TensorFlow based on CUDA version: " "unknown CUDA version supplied - see %s", jl("tf_unknown_cuda"), ) yield from () return None yield {"package_name": "tensorflow"} yield {"package_name": "tensorflow-gpu"} return None yield from () return None
def test_not_acceptable(self, context: Context) -> None: """Test raising an exception when the pipeline unit is included.""" unit = self.UNIT_TESTED() context.project.runtime_environment.platform = "aarch64" assert not context.stack_info # The unit always raises, no need to explicitly assign platform here. with unit.assigned_context(context): with pytest.raises( NotAcceptable, match= "Platform 'aarch64' is not supported, possible platforms are: linux-x86_64" ): unit.run() assert self.verify_justification_schema(context.stack_info) assert context.stack_info == [ { "link": jl("platform"), "message": "Platform 'aarch64' is not supported, possible platforms are: " "linux-x86_64", "type": "ERROR", }, ]
def _prepare_justification_link(cls, entries: List[Dict[str, Any]]) -> None: """Prepare justification links before using them.""" for entry in entries: link = entry.get("link") if link and not link.startswith(("https://", "http://")): entry["link"] = jl(link)
def test_python_version_mismatch(self) -> None: """Test when python version stated in Pipfile does not match with the one provided in the configuration.""" context = flexmock(project=Project.from_strings( self._CASE_PIPFILE_PYTHON), stack_info=[]) context.project.runtime_environment.operating_system.name = "rhel" context.project.runtime_environment.operating_system.version = "8" context.project.runtime_environment.python_version = "3.8" boot = PythonVersionBoot() with PythonVersionBoot.assigned_context(context): boot.run() assert context.project.runtime_environment.operating_system.name == "rhel" assert context.project.runtime_environment.operating_system.version == "8" assert context.project.runtime_environment.python_version == "3.8" assert context.stack_info == [{ "message": "Python version stated in Pipfile ('3.6') does not match with the one " "specified in the Thoth configuration ('3.8'), using Python version from Thoth " "configuration implicitly", "type": "WARNING", "link": jl("py_version"), }]
def test_pre_run_no_symbols_warning(self, context: Context) -> None: """Make sure the pre_run method produces a warning for users.""" context.graph.should_receive("get_thoth_s2i_analyzed_image_symbols_all").with_args( thoth_s2i_image_name="quay.io/thoth-station/s2i-thoth-ubi8-py38", thoth_s2i_image_version="1.0.0", is_external=False, ).and_return(set()).once() context.project.runtime_environment.base_image = "quay.io/thoth-station/s2i-thoth-ubi8-py38:v1.0.0" assert not context.stack_info unit = self.UNIT_TESTED() with unit.assigned_context(context): unit.pre_run() assert len(context.stack_info) == 1 assert self.verify_justification_schema(context.stack_info) assert context.stack_info == [ { "link": jl("no_abi"), "message": "No ABI symbols found for " "'quay.io/thoth-station/s2i-thoth-ubi8-py38' in version '1.0.0'", "type": "WARNING", }, ]
class SolversConfiguredBoot(Boot): """A boot to notify about runtime environments not supported by solvers enabled in a deployment.""" _SOLVERS_CONFIGURED = os.getenv( "THOTH_ADVISER_DEPLOYMENT_CONFIGURED_SOLVERS") _JUSTIFICATION = jl("eol_env") @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register self if users use runtime environments not matching solvers in the configmap.""" if not cls._SOLVERS_CONFIGURED or builder_context.is_included(cls): yield from () return None runtime_environment = ( builder_context.project.runtime_environment.operating_system.name, builder_context.project.runtime_environment.operating_system. version, builder_context.project.python_version, ) for solver_name in cls._SOLVERS_CONFIGURED.splitlines(): solver_name = solver_name.strip() if not solver_name: continue try: solver_info = OpenShift.parse_python_solver_name(solver_name) except SolverNameParseError: _LOGGER.exception( "Please report this error to Thoth service administrator") yield from () return None if runtime_environment == ( solver_info["os_name"], solver_info["os_version"], solver_info["python_version"], ): break else: yield {} return None yield from () return None def run(self) -> None: """Inform about runtime environment that is not backed up by a solver.""" self.context.stack_info.append({ "type": "WARNING", "message": "Runtime environment used is no longer supported, it is " "recommended to switch to another runtime environment", "link": self._JUSTIFICATION, })
class ThothS2IBoot(Boot): """A boot that notifies about missing observations.""" _THOTH_S2I_PREFIX = "quay.io/thoth-station/s2i-thoth-" _JUSTIFICATION = [{ "type": "INFO", "message": "It is recommended to use Thoth's s2i to have recommendations specific to runtime environment", "link": jl("thoth_s2i"), }] @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[Any, Any], None, None]: """Include this boot in adviser if Thoth s2i is not used..""" base_image = builder_context.project.runtime_environment.base_image if not builder_context.is_included(cls) and ( base_image is None or (base_image and not base_image.startswith(cls._THOTH_S2I_PREFIX))): yield {} return None yield from () return None def run(self) -> None: """Check for no observations made on the given state.""" self.context.stack_info.extend(self._JUSTIFICATION)
def test_no_python_pipfile(self) -> None: """Test assigning Python version from Pipfile.""" context = flexmock(project=Project.from_strings( self._CASE_PIPFILE_NO_PYTHON), stack_info=[]) context.project.runtime_environment.operating_system.name = "rhel" context.project.runtime_environment.operating_system.version = "8" context.project.runtime_environment.python_version = "3.6" boot = PythonVersionBoot() with PythonVersionBoot.assigned_context(context): boot.run() assert context.project.runtime_environment.operating_system.name == "rhel" assert context.project.runtime_environment.operating_system.version == "8" assert context.project.runtime_environment.python_version == "3.6" assert context.stack_info == [{ "message": "No version of Python specified explicitly, assigning the one " "found in Thoth's configuration: '3.6'", "type": "WARNING", "link": jl("py_version"), }]
def _load_files(requirements_format: str) -> Tuple[str, Optional[str]]: """Load Pipfile/Pipfile.lock or requirements.in/txt from the current directory.""" if requirements_format == "pipenv": _LOGGER.info("Using Pipenv files located in %r directory", os.getcwd()) pipfile_lock_exists = os.path.exists("Pipfile.lock") if pipfile_lock_exists: _LOGGER.info( "Submitting Pipfile.lock as a base for user's stack scoring - see %s", jl("user_stack"), ) project = Project.from_files( without_pipfile_lock=not os.path.exists("Pipfile.lock")) if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] != project.pipfile.hash()["sha256"]): _LOGGER.error( "Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile " "adjusted? This error is not critical.", project.pipfile_lock.meta.hash["sha256"][:6], project.pipfile.hash()["sha256"][:6], ) elif requirements_format in ("pip", "pip-tools", "pip-compile"): _LOGGER.info("Using requirements.txt file located in %r directory", os.getcwd()) project = Project.from_pip_compile_files(allow_without_lock=True) else: raise ValueError( f"Unknown configuration option for requirements format: {requirements_format!r}" ) return ( project.pipfile.to_string(), project.pipfile_lock.to_string() if project.pipfile_lock else None, )
class PlatformBoot(Boot): """A boot to check if a supported platform is used. We could check this based on the database entries, but as this will change rarely, we can hardcode supported platforms here. """ _JUSTIFICATION_LINK = jl("platform") _SUPPORTED_PLATFORMS = frozenset({"linux-x86_64"}) @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register self, always on unsupported platform.""" if (builder_context.project.runtime_environment.platform not in cls._SUPPORTED_PLATFORMS and not builder_context.is_included(cls)): yield {} return None yield from () return None def run(self) -> None: """Check if GPU enabled recommendations should be done.""" platform = self.context.project.runtime_environment.platform msg = f"Platform {platform!r} is not supported, possible platforms are: " + ", ".join( self._SUPPORTED_PLATFORMS) self.context.stack_info.append({ "type": "ERROR", "message": msg, "link": self._JUSTIFICATION_LINK, }) raise NotAcceptable(f"{msg} - see {self._JUSTIFICATION_LINK}")
class NoObservationWrap(Wrap): """A wrap that notifies about missing observations.""" _JUSTIFICATION = [{ "type": "INFO", "message": "No issues spotted for this stack based on Thoth's database", "link": jl("no_observations"), }] @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[Any, Any], None, None]: """Include this wrap in adviser, once.""" if not builder_context.is_included( cls) and builder_context.is_adviser_pipeline(): yield {} return None yield from () return None def run(self, state: State) -> None: """Check for no observations made on the given state.""" if not state.justification: state.add_justification(self._JUSTIFICATION)
def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Optional[Dict[str, Any]]: """Register this pipeline unit for adviser when CUDA is present.""" if not builder_context.is_adviser_pipeline(): return None if builder_context.recommendation_type in (RecommendationType.LATEST, RecommendationType.TESTING): # Use any TensorFlow for testing purposes or when resolving latest stack. return None cuda_version = builder_context.project.runtime_environment.cuda_version if cuda_version is None: # No CUDA available in the given runtime environment. return None if cuda_version not in cls._KNOWN_CUDA: _LOGGER.warning( "Unable to perform recommendations for TensorFlow based on CUDA version: " "unknown CUDA version supplied - see %s", jl("tf_unknown_cuda"), ) return None # Include this pipeline units in two configurations for tensorflow and tensorflow-gpu. included_units = builder_context.get_included_sieves(cls) if len(included_units) == 2: return None if len(included_units) == 0: return {"package_name": "tensorflow"} elif len(included_units) == 1: return {"package_name": "tensorflow-gpu"} return None
class TensorFlowGPUPseudonym(Pseudonym): """A TensorFlow pseudonym to map tensorflow to tensorflow-gpu packages.""" CONFIGURATION_DEFAULT = {"package_name": "tensorflow"} _LINK = jl("tf_gpu_alt") _pseudonyms = attr.ib(type=Optional[FrozenSet[str]], default=None, init=False) @classmethod def should_include(cls, builder_context: "PipelineBuilderContext") -> Generator[Dict[str, Any], None, None]: """Register self.""" if ( builder_context.project.runtime_environment.cuda_version is not None and builder_context.is_adviser_pipeline() and builder_context.recommendation_type != RecommendationType.LATEST and not builder_context.is_included(cls) ): yield {} return None yield from () return None def pre_run(self) -> None: """Initialize this pipeline unit before each run.""" self._pseudonyms = None super().pre_run() def run(self, package_version: PackageVersion) -> Generator[Tuple[str, str, str], None, None]: """Map TensorFlow packages to their alternatives.""" if package_version.index.url != "https://pypi.org/simple" or package_version.semantic_version.release[0] > 1: return None if self._pseudonyms is None: # Be lazy with queries to the database. _LOGGER.warning( "Considering also tensorflow-gpu package as the runtime environment used provides CUDA - see %s", self._LINK, ) runtime_environment = self.context.project.runtime_environment self._pseudonyms = frozenset( { i[1] for i in self.context.graph.get_solved_python_package_versions_all( package_name="tensorflow-gpu", package_version=None, index_url="https://pypi.org/simple", count=None, os_name=runtime_environment.operating_system.name, os_version=runtime_environment.operating_system.version, python_version=runtime_environment.python_version, distinct=True, is_missing=False, ) } ) if package_version.locked_version in self._pseudonyms: yield "tensorflow-gpu", package_version.locked_version, "https://pypi.org/simple"
class IntelTensorFlowWrap(Wrap): """A wrap that recommends using Intel TensorFlow if TensorFlow is in resolved dependencies. https://software.intel.com/content/www/us/en/develop/articles/intel-optimization-for-tensorflow-installation-guide.html#pip_wheels """ CONFIGURATION_DEFAULT = {"package_name": "tensorflow"} # Sandy bridge CPUID taken from https://en.wikipedia.org/wiki/Sandy_Bridge # Ivy bridge CPUID taken from https://en.wikipedia.org/wiki/Ivy_Bridge_(microarchitecture) # # As CPUID encodes family (bits 8 - 11 with mask 0xF00) and model (bits 4 - 7 with mask 0xF0) we got these # numbers from CPUID. _CPU_TABLE = frozenset({ # tuple (model, family) # Sandy Bridge-HE-4, Sandy Bridge-H-2, Sandy Bridge-M-2 # ((0x0206A7 & 0xF0) >> 4, (0x0206A7 & 0xF00) >> 8), # Sandy Bridge - EP - 8 # ((0x0206D6 & 0xF0) >> 4, (0x0206D6 & 0xF00) >> 8), # ((0x0206D7 & 0xF0) >> 4, (0x0206D7 & 0xF00) >> 8), # Sandy Bridge - EP - 4 # ((0x0206D6 & 0xF0) >> 4, (0x0206D6 & 0xF00) >> 8), # ((0x0206D7 & 0xF0) >> 4, (0x0206D7 & 0xF00) >> 8), # Ivy Bridge-M-2, Ivy Bridge-H-2, Ivy Bridge-HM-4, Ivy Bridge-HE-4 # ((0x000306A9 & 0xF0) >> 4, (0x000306A9 & 0xF00) >> 8), # All maps to the following values: (13, 6), (10, 6), }) _JUSTIFICATION = [{ "type": "INFO", "message": "Consider using intel-tensorflow which is optimized for CPU detected in your environment", "link": jl("intel_tensorflow"), }] @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Optional[Dict[str, Any]]: """Include this wrap for x86_64 architecture on CPU models with Ivy/Sandy bridge.""" if builder_context.is_included(cls): return None if not builder_context.is_adviser_pipeline(): return None runtime_environment = builder_context.project.runtime_environment cpu_tuple = (runtime_environment.hardware.cpu_model, runtime_environment.hardware.cpu_family) if runtime_environment.platform == "linux-x86_64" and cpu_tuple in cls._CPU_TABLE: return {} return None def run(self, state: State) -> None: """Recommend using intel-tensorflow if tensorflow is resolved.""" if "intel-tensorflow" not in state.resolved_dependencies: state.add_justification(self._JUSTIFICATION)
class LabelsBoot(Boot): """A boot to notify about labels used during the resolution.""" _JUSTIFICATION_LINK_LABELS = jl("labels") _JUSTIFICATION_LINK_ALLOW_CVE = jl("allow_cve") @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register self if labels are used.""" if not builder_context.is_included(cls) and builder_context.labels: yield {} return None yield from () return None def run(self) -> None: """Notify about labels used during the resolution process..""" for key, value in self.context.labels.items(): if key != "allow-cve": msg = f"Considering label {key}={value} in the resolution process" _LOGGER.info("%s - see %s", msg, self._JUSTIFICATION_LINK_LABELS) self.context.stack_info.append({ "message": msg, "type": "INFO", "link": self._JUSTIFICATION_LINK_LABELS, }) else: for allow_cve in value.split(","): msg = f"Allowing CVE {allow_cve.upper()!r} to be present in the application" _LOGGER.warning("%s - see %s", msg, self._JUSTIFICATION_LINK_ALLOW_CVE) self.context.stack_info.append({ "message": msg, "type": "WARNING", "link": self._JUSTIFICATION_LINK_ALLOW_CVE, })
class SolvedSoftwareEnvironmentBoot(Boot): """A boot to check for solved software environment before running any resolution.""" _JUSTIFICATION_LINK = jl("solved_sw_env") @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Generator[Dict[str, Any], None, None]: """Register self, always.""" if builder_context.project.runtime_environment.is_fully_specified( ) and not builder_context.is_included(cls): yield {} return None yield from () return None def run(self) -> None: """Check for version clash in packages.""" if self.context.graph.solved_software_environment_exists( os_name=self.context.project.runtime_environment. operating_system.name, os_version=self.context.project.runtime_environment. operating_system.version, python_version=self.context.project.runtime_environment. python_version, ): return runtime_environment = self.context.project.runtime_environment msg = ( f"No observations found for {runtime_environment.operating_system.name!r} in " f"version {runtime_environment.operating_system.version!r} using " f"Python {runtime_environment.python_version!r}") self.context.stack_info.append({ "type": "ERROR", "message": msg, "link": self._JUSTIFICATION_LINK, }) _LOGGER.warning("%s - %s", msg, self._JUSTIFICATION_LINK) _LOGGER.warning("Available configurations:") configurations = self.context.graph.get_solved_python_package_versions_software_environment_all( ) _LOGGER.warning("{:<16} {:<16} {:<8}".format("OS name", "OS version", "Python version")) for conf in sorted( configurations, key=lambda i: (i["os_name"], i["os_version"], i["python_version"]), ): _LOGGER.warning("{:<16} {:<16} {:<8}".format( conf["os_name"], conf["os_version"], conf["python_version"])) raise NotAcceptable(msg)
class TensorFlow22ProbabilityStep(Step): """Suggest not to use TensorFlow 2.2 with tensorflow-probability. https://github.com/tensorflow/tensorflow/issues/40584 """ CONFIGURATION_DEFAULT = { "package_name": "tensorflow-probability", "multi_package_resolution": False } _MESSAGE = "TensorFlow in version 2.2 and tensorflow-probability cause runtime errors" _LINK = jl("tf_40584") _message_logged = attr.ib(type=bool, default=False, init=False) @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Optional[Dict[str, Any]]: """Register this pipeline unit for adviser if not using latest recommendations.""" if (not builder_context.is_adviser_pipeline() or builder_context.recommendation_type == RecommendationType.LATEST): return None if not builder_context.is_included(cls): return {} return None def pre_run(self) -> None: """Initialize this pipeline unit before each run.""" self._message_logged = False super().pre_run() def run( self, state: State, package_version: PackageVersion ) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]: """Suggest not to use TensorFlow 2.2 with with tensorflow-probability.""" tensorflow_any = (state.resolved_dependencies.get("tensorflow") or state.resolved_dependencies.get("tensorflow-cpu") or state.resolved_dependencies.get("tensorflow-gpu")) if not tensorflow_any or (tensorflow_any[1] != "2.2" and not tensorflow_any[1].startswith("2.2.")): return None if not self._message_logged: self._message_logged = True _LOGGER.warning("%s - see %s", self._MESSAGE, self._LINK) self.context.stack_info.append({ "type": "WARNING", "message": self._MESSAGE, "link": self._LINK }) raise NotAcceptable
class TensorFlowSlowKerasEmbedding(Wrap): """A wrap that notifies a bug in the embedding layer. https://github.com/tensorflow/tensorflow/issues/42475 """ CONFIGURATION_DEFAULT = {"package_name": "tensorflow"} _JUSTIFICATION = [ { "type": "WARNING", "message": "TensorFlow in version <=2.4 is slow when tf.keras.layers.Embedding is used", "link": jl("tf_42475"), } ] @classmethod def should_include(cls, builder_context: "PipelineBuilderContext") -> Optional[Dict[str, Any]]: """Include this wrap in adviser.""" if not builder_context.is_adviser_pipeline(): return None if builder_context.library_usage is not None and "tensorflow.keras.layers.Embedding" not in ( builder_context.library_usage.get("tensorflow") or [] ): return None units_included = builder_context.get_included_wraps(cls) if len(units_included) == 4: return None elif len(units_included) == 0: return {"package_name": "tensorflow"} elif len(units_included) == 1: return {"package_name": "tensorflow-cpu"} elif len(units_included) == 2: return {"package_name": "tensorflow-gpu"} elif len(units_included) == 3: return {"package_name": "intel-tensorflow"} return None def run(self, state: State) -> None: """Notify about a bug in summary output spotted on TensorFlow 2.3.""" tensorflow_any = ( state.resolved_dependencies.get("tensorflow") or state.resolved_dependencies.get("tensorflow-cpu") or state.resolved_dependencies.get("tensorflow-gpu") or state.resolved_dependencies.get("intel-tensorflow") ) if tensorflow_any is None: return None tf_package_version: PackageVersion = self.context.get_package_version(tensorflow_any) if tf_package_version.semantic_version.release[:2] <= (2, 4): state.add_justification(self._JUSTIFICATION)
class TensorFlow23Accuracy(Wrap): """A wrap that notifies about accuracy bug on safe()/load_model() calls. https://github.com/tensorflow/tensorflow/issues/42045 https://github.com/keras-team/keras/issues/14181 https://github.com/tensorflow/tensorflow/commit/5adacc88077ef82f6c4a7f9bb65f9ed89f9d8947 """ CONFIGURATION_DEFAULT = {"package_name": "tensorflow"} _JUSTIFICATION = [{ "type": "WARNING", "message": "TensorFlow in version 2.3 produces wrong model accuracy when the model is " "serialized using `accuracy`, use `sparse_categorical_accuracy` instead", "link": jl("tf_42045"), }] @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Optional[Dict[str, Any]]: """Include this wrap in adviser.""" if not builder_context.is_adviser_pipeline(): return None units_included = builder_context.get_included_wraps(cls) if len(units_included) == 4: return None elif len(units_included) == 0: return {"package_name": "tensorflow"} elif len(units_included) == 1: return {"package_name": "tensorflow-cpu"} elif len(units_included) == 2: return {"package_name": "tensorflow-gpu"} elif len(units_included) == 3: return {"package_name": "intel-tensorflow"} return None def run(self, state: State) -> None: """Notify about accuracy bug in safe()/load_model() calls.""" tensorflow_any = (state.resolved_dependencies.get("tensorflow") or state.resolved_dependencies.get("tensorflow-cpu") or state.resolved_dependencies.get("tensorflow-gpu") or state.resolved_dependencies.get("intel-tensorflow")) if tensorflow_any is None: return None if tensorflow_any[1] == "2.3" or tensorflow_any[1].startswith("2.3."): state.add_justification(self._JUSTIFICATION)
class PandasPy36Sieve(Sieve): """A sieve that makes sure Pandas>=1.2 is not used on Python 3.6 or older. See https://github.com/pandas-dev/pandas/pull/35214 """ CONFIGURATION_DEFAULT = {"package_name": "pandas"} _MESSAGE = f"Pandas in versions >=1.2 dropped Python 3.6 support" _JUSTIFICATION_LINK = jl("pandas_py36_drop") _message_logged = attr.ib(type=bool, default=False, init=False) def pre_run(self) -> None: """Initialize this pipeline unit before each run.""" self._message_logged = False super().pre_run() @classmethod def should_include( cls, builder_context: "PipelineBuilderContext" ) -> Optional[Dict[str, Any]]: """Register this pipeline unit for adviser when Python 3.6 is used, not latest/testing recommendations.""" if not builder_context.is_adviser_pipeline( ) or builder_context.is_included(cls): return None if builder_context.recommendation_type in (RecommendationType.LATEST, RecommendationType.TESTING): return None if builder_context.project.runtime_environment.get_python_version_tuple( ) > (3, 6): # Not a Python 3.6 or older environment. return None return {} def run( self, package_versions: Generator[PackageVersion, None, None] ) -> Generator[PackageVersion, None, None]: """Do not use Pandas>=1.2 on Python 3.6.""" for package_version in package_versions: if package_version.semantic_version.release[:2] < (1, 2): yield package_version elif not self._message_logged: self._message_logged = True _LOGGER.warning("%s - see %s", self._MESSAGE, self._JUSTIFICATION_LINK) self.context.stack_info.append({ "type": "WARNING", "message": self._MESSAGE, "link": self._JUSTIFICATION_LINK })
def test_run(self, context: Context) -> None: """Test if the given software environment is solved.""" context.labels = { "foo": "bar", "baz": "qux", "allow-cve": "pysec-2014-10,PYSEC-2014-22" } assert not context.stack_info unit = self.UNIT_TESTED() with unit.assigned_context(context): assert unit.run() is None assert context.stack_info == [ { "link": jl("labels"), "message": "Considering label foo=bar in the resolution process", "type": "INFO" }, { "link": jl("labels"), "message": "Considering label baz=qux in the resolution process", "type": "INFO" }, { "link": jl("allow_cve"), "message": "Allowing CVE 'PYSEC-2014-10' to be present in the application", "type": "WARNING", }, { "link": jl("allow_cve"), "message": "Allowing CVE 'PYSEC-2014-22' to be present in the application", "type": "WARNING", }, ]
def run(self) -> None: """Check for version clash in packages.""" if not self.context.project.runtime_environment.is_fully_specified(): msg = ( f"Software environment supplied is not fully specified, OS name " f"is {self.context.project.runtime_environment.operating_system.name!r} " f"in version {self.context.project.runtime_environment.operating_system.version!r} " f"using Python {self.context.project.runtime_environment.python_version!r}" ) self.context.stack_info.append({"message": msg, "type": "ERROR", "link": jl("fully_specified_env")}) raise NotAcceptable(msg)
def run(self) -> None: """Discard any minor release in RHEL.""" os_name = self.context.project.runtime_environment.operating_system.name os_version = self.context.project.runtime_environment.operating_system.version if os_name == "rhel" and os_version is not None: version_parts = os_version.split(".", maxsplit=1) if len(version_parts) > 1: _LOGGER.info( "RHEL major releases guarantee ABI compatibility across minor releases; " "discarding minor release information and using RHEL version %r - see %s", version_parts[0], jl("rhel_version"), ) self.context.project.runtime_environment.operating_system.version = version_parts[0]
def run(self) -> None: """Check CVE timestamp and provide it to users.""" cve_timestamp = self.context.graph.get_cve_timestamp() if cve_timestamp is None: message = "No CVE timestamp information found in the database" self.context.stack_info.append( { "message": message, "type": "WARNING", "link": jl("no_cve_timestamp"), } ) return days = (datetime.utcnow() - cve_timestamp).days self.context.stack_info.append( { "type": "WARNING" if days > self._CVE_TIMESTAMP_DAYS_WARNING else "INFO", "message": f"CVE database of known vulnerabilities for Python packages was " f"updated at {datetime2datetime_str(cve_timestamp)!r}", "link": jl("cve_timestamp"), } )