def _retrieve_adviser_inputs_statistics(
adviser_files: Dict[str, Any], ) -> List[Dict[str, Any]]:
"""Retrieve adviser inputs statistics.
:param adviser_files: adviser documents
"""
adviser_inputs_collected: List[Dict[str, Any]] = []
for document_id, document in adviser_files.items():
datetime_advise_run = document["metadata"].get("datetime")
datetime_object = datetime.strptime(datetime_advise_run,
"%Y-%m-%dT%H:%M:%S.%f")
cli_arguments = document["metadata"]["arguments"]["thoth-adviser"]
source_type = (cli_arguments.get("metadata")
or {}).get("source_type")
source_type = source_type.upper() if source_type else None
parameters = document["result"]["parameters"]
runtime_environment = parameters["project"].get(
"runtime_environment")
os = runtime_environment.get("operating_system", {})
if os:
os_name = runtime_environment["operating_system"].get("name")
if os_name:
runtime_environment["operating_system"][
"name"] = map_os_name(
os_name=runtime_environment["operating_system"]
["name"], )
# Recommendation type
recommendation_type = parameters["recommendation_type"].upper()
# Solver
os = runtime_environment.get("operating_system", {})
os_name = os.get("name")
os_version = normalize_os_version(os.get("name"),
os.get("version"))
python_interpreter = runtime_environment.get("python_version")
# Base image
base_image = runtime_environment.get("base_image", None)
# Hardware
hardware = runtime_environment.get("hardware", {})
adviser_inputs_collected.append(
{
"document_id": document_id,
"date": datetime_object,
"source_type": source_type,
"recommendation_type": recommendation_type,
"base_image": base_image,
"solver":
f'{os_name}-{os_version}-py{python_interpreter.replace(".", "")}',
"cpu_model": hardware.get("cpu_model", None),
"cpu_family": hardware.get("cpu_family", None),
}, )
return adviser_inputs_collected
def should_include(
cls, builder_context: "PipelineBuilderContext"
) -> Generator[Dict[str, Any], None, None]:
"""Register self if users use runtime environments not matching solvers in the configmap."""
if not cls._SOLVERS_CONFIGURED or builder_context.is_included(cls):
yield from ()
return None
runtime_environment = (
builder_context.project.runtime_environment.operating_system.name,
normalize_os_version(
builder_context.project.runtime_environment.operating_system.
name,
builder_context.project.runtime_environment.operating_system.
version,
),
builder_context.project.python_version,
)
for solver_name in cls._SOLVERS_CONFIGURED.splitlines():
solver_name = solver_name.strip()
if not solver_name:
continue
try:
solver_info = OpenShift.parse_python_solver_name(solver_name)
except SolverNameParseError:
_LOGGER.exception(
"Please report this error to Thoth service administrator")
yield from ()
return None
if runtime_environment == (
solver_info["os_name"],
solver_info["os_version"],
solver_info["python_version"],
):
break
else:
yield {}
return None
yield from ()
return None
def check_runtime_environment(
self, runtime_environment_name: str) -> List[Dict[str, Any]]:
"""Check the given runtime environment entry."""
runtime_environment = self.get_runtime_environment(
runtime_environment_name)
result = []
# CUDA
cuda_version = discover_cuda_version()
conf_cuda_version = runtime_environment.get("cuda_version")
if conf_cuda_version != cuda_version:
if (cuda_version is None or isinstance(cuda_version, str)
and isinstance(conf_cuda_version, str)):
message_type = "ERROR"
else:
message_type = "WARNING"
result.append({
"type":
message_type,
"runtime_environment":
runtime_environment_name,
"message":
f"CUDA version declared in the configuration file "
f"({conf_cuda_version!r}) does not match the one detected ({cuda_version!r})",
})
# Operating system
conf_operating_system = runtime_environment.get("operating_system")
if conf_operating_system:
conf_os_name = map_os_name(conf_operating_system.get("name"))
conf_os_version = normalize_os_version(
conf_os_name, conf_operating_system.get("version"))
if conf_os_name == "ubi":
result.append({
"type":
"INFO",
"runtime_environment":
runtime_environment_name,
"message":
"UBI container images are ABI compatible with RHEL container images",
})
conf_os_name = "rhel"
os_name, os_version = discover_distribution()
os_name = map_os_name(os_name)
os_version = normalize_os_version(os_name, os_version)
if conf_os_name != os_name:
result.append({
"type":
"ERROR",
"runtime_environment":
runtime_environment_name,
"message":
f"Operating system name stated in the configuration file ({conf_os_name!r}) "
f"does not match the one detected ({os_name!r})",
})
elif conf_os_version != os_version:
result.append({
"type":
"ERROR",
"runtime_environment":
runtime_environment_name,
"message":
f"Operating system version stated in the configuration file ({conf_os_version!r}) "
f"does not match the one detected ({os_version!r})",
})
# Python version
python_version = discover_python_version()
conf_python_version = runtime_environment.get("python_version")
if python_version != conf_python_version:
result.append({
"type":
"ERROR",
"runtime_environment":
runtime_environment_name,
"message":
f"Python version detected ({python_version!r}) does not match the one stated in the "
f"configuration file ({conf_python_version!r})",
})
# Check hardware
conf_cpu_family = runtime_environment.get("hardware",
{}).get("cpu_family")
conf_cpu_model = runtime_environment.get("hardware",
{}).get("cpu_model")
cpu_info = discover_cpu()
if cpu_info.get("cpu_family") != conf_cpu_family:
result.append({
"type":
"ERROR" if conf_cpu_family is not None else "WARNING",
"runtime_environment":
runtime_environment_name,
"message":
f"CPU family stated in the configuration file ({conf_cpu_family}) does not match the "
f"one detected ({cpu_info.get('cpu_family')})",
})
if cpu_info.get("cpu_model") != conf_cpu_model:
result.append({
"type":
"ERROR" if conf_cpu_model is not None else "WARNING",
"runtime_environment":
runtime_environment_name,
"message":
f"CPU model stated in the configuration file ({conf_cpu_model}) does not match the "
f"one detected ({cpu_info.get('cpu_model')})",
})
return result
def create_default_config(self,
template: str = None,
nowrite: bool = False) -> Optional[dict]:
"""Place default configuration into the current directory."""
if not os.path.isdir(".git"):
_LOGGER.warning(
"Configuration file is not created in the root of git repo")
template = template or self.DEFAULT_THOTH_CONFIG
_LOGGER.debug("Reading configuration from %r", template)
with open(template, "r") as default_config_file:
default_config = default_config_file.read()
_LOGGER.info("Discovering host runtime environment")
cpu_info = discover_cpu()
cuda_version = discover_cuda_version()
# Add quotes for textual representation in the config file.
cuda_version = f"'{cuda_version}'" if cuda_version is not None else "null"
os_name, os_version = discover_distribution()
os_name = map_os_name(os_name)
os_version = normalize_os_version(os_name, os_version)
python_version = discover_python_version()
platform = discover_platform()
base_image = discover_base_image()
base_image = base_image if base_image is not None else "null"
runtime_environment_name = re.sub(r"[^0-9a-zA-Z-]", "-",
f"{os_name}-{os_version}")
requirements_format = os.getenv("THAMOS_REQUIREMENTS_FORMAT",
self._DEFAULT_REQUIREMENTS_FORMAT)
if requirements_format not in self.REQUIREMENTS_FORMATS:
# This avoids possibly dangerous environment variable expansion.
_LOGGER.warning(
"Unknown requirements format specified, forcing %r: %r",
self._DEFAULT_REQUIREMENTS_FORMAT,
requirements_format,
)
requirements_format = self._DEFAULT_REQUIREMENTS_FORMAT
expand_env = bool(int(os.getenv("THAMOS_CONFIG_EXPAND_ENV", 0)))
default_config = default_config.format(
runtime_environment_name=runtime_environment_name,
cuda_version=cuda_version,
os_name=os_name,
os_version=os_version,
python_version=python_version,
platform=platform,
requirements_format=requirements_format,
base_image=base_image,
cpu_family=cpu_info["cpu_family"] or "null",
cpu_model=cpu_info["cpu_model"] or "null",
cpu_model_name=cpu_info["cpu_model_name"]
or "CPU detection failed",
**(dict(os.environ) if expand_env else {}),
)
if not nowrite:
_LOGGER.debug(
"Writing configuration file to %r",
os.path.join(os.getcwd(), self.CONFIG_NAME),
)
with open(self.CONFIG_NAME, "w") as config_file:
config_file.write(default_config)
return None
else:
return yaml.safe_load(default_config)