def aggregate_inspection_results_dict(list_ids: List[str],
identifier_inspection: List[str],
limit_results: bool = False) -> dict:
"""Aggregate inspection results per identifier from inspection documents stored in Ceph."""
inspection_store = InspectionResultsStore()
inspection_store.connect()
inspection_results_dict = {}
tot = sum([len(r) for r in list_ids.values()])
current_identifier_batch_length = 0
if limit_results:
logger.info(f"Limiting results to 5 per batch to test functions!!")
for identifier in identifier_inspection:
inspection_results_dict[identifier] = []
logger.info("Analyzing inspection identifer batch: %r", identifier)
for n, ids in enumerate(list_ids[identifier]):
document = inspection_store.retrieve_document(ids)
# pop build logs to save some memory (not necessary for now)
document["build_log"] = None
logger.info(
f"Analysis n.{n + 1 + current_identifier_batch_length}/{tot}")
inspection_results_dict[identifier].append(document)
if limit_results:
if n + 1 == 5:
break
current_identifier_batch_length += len(list_ids[identifier])
return inspection_results_dict
def filter_inspection_ids_list(inspection_identifier_list: List[str]) -> dict:
"""Filter inspection ids list according to the inspection identifier selected.
:param inspection_identifier_list: list of identifier to filter out inspection ids
"""
inspection_store = InspectionResultsStore()
inspection_store.connect()
logger.info(f"Retrieving all inspection ids")
inspection_ids_list = list(inspection_store.get_document_listing())
filtered_list_ids = {}
for identifier in inspection_identifier_list:
filtered_list_ids[identifier] = []
for ids in inspection_ids_list:
inspection_filter = "-".join(
ids.split("-")[1:(len(ids.split("-")) - 1)])
if inspection_filter:
if inspection_filter in inspection_identifier_list:
filtered_list_ids[inspection_filter].append(ids)
tot_inspections_selected = sum(
[len(batch_n) for batch_n in filtered_list_ids.values()])
inspection_batches = [
(batch_name, len(batch_count))
for batch_name, batch_count in filtered_list_ids.items()
]
logger.info(
f"There are {tot_inspections_selected} inspection runs selected: {inspection_batches} respectively"
)
return filtered_list_ids
def get_ceph_connection_error_status():
"""Check connection to Ceph instance."""
inspections = InspectionResultsStore()
try:
inspections.connect()
except Exception as excptn:
metrics.ceph_connection_error_status.set(1)
_LOGGER.exception(excptn)
else:
metrics.ceph_connection_error_status.set(0)
def aggregate_inspection_results_per_identifier(
inspection_ids: List[str],
identifier_inspection: List[str],
limit_results: bool = False,
max_ids: int = 5) -> dict:
"""Aggregate inspection results per identifier from inspection documents stored in Ceph.
:param inspection_ids: list of inspection ids
:param inspection_identifier: list of identifier/s to filter inspection ids
:param limit_results: reduce the number of inspection ids used per batch to `max_ids` to test analysis
:param max_ids: maximum number of inspection ids considered
"""
inspection_store = InspectionResultsStore()
inspection_store.connect()
inspection_results_dict = {}
tot = sum([len(r) for r in inspection_ids.values()])
current_identifier_batch_length = 0
if limit_results:
logger.info(
f"Limiting results to {max_ids} per batch to test functions!!")
for identifier in identifier_inspection:
inspection_results_dict[identifier] = []
logger.info("Analyzing inspection identifer batch: %r", identifier)
for n, ids in enumerate(inspection_ids[identifier]):
document = inspection_store.retrieve_document(ids)
# pop build logs to save some memory (not necessary for now)
document["build_log"] = None
logger.info(
f"Analysis n.{n + 1 + current_identifier_batch_length}/{tot}")
inspection_results_dict[identifier].append(document)
if limit_results:
if n + 1 == max_ids:
break
current_identifier_batch_length += len(inspection_ids[identifier])
return inspection_results_dict
def filter_inspection_ids(inspection_identifiers: List[str]) -> dict:
"""Filter inspection ids list according to the inspection identifier selected.
:param inspection_identifiers: list of identifier/s to filter inspection ids
"""
inspection_store = InspectionResultsStore()
inspection_store.connect()
filtered_inspection_ids = inspection_store.filter_document_ids(
inspection_identifiers=inspection_identifiers)
inspections_selected = sum(
[len(batch_n) for batch_n in filtered_inspection_ids.values()])
inspection_batches = [
(batch_name, len(batch_count))
for batch_name, batch_count in filtered_inspection_ids.items()
]
logger.info(
f"There are {inspections_selected} inspection runs selected: {inspection_batches} respectively"
)
return filtered_inspection_ids
def get_inspection_results_per_identifier():
"""Get the total number of inspections in Ceph per identifier."""
store = InspectionResultsStore()
if not store.is_connected():
store.connect()
specific_list_ids = {}
specific_list_ids["without_identifier"] = 0
for ids in store.get_document_listing():
inspection_filter = "_".join(
ids.split("-")[1:(len(ids.split("-")) - 1)])
if inspection_filter:
if inspection_filter not in specific_list_ids.keys():
specific_list_ids[inspection_filter] = 1
else:
specific_list_ids[inspection_filter] += 1
else:
specific_list_ids["without_identifier"] += 1
for identifier, identifier_list in specific_list_ids.items():
metrics.inspection_results_ceph.labels(identifier).set(identifier_list)
_LOGGER.debug(
f"inspection_results_ceph for {identifier} ={identifier_list}")
from thoth.storages import InspectionResultsStore
from thoth.storages import PackageAnalysisResultsStore
from thoth.storages import ProvenanceResultsStore
from thoth.storages import DependencyMonkeyReportsStore
from thoth.common import init_logging
from thoth.common import OpenShift
import thoth.metrics_exporter.metrics as metrics
init_logging()
_LOGGER = logging.getLogger(__name__)
_MONITORED_STORES = (
AdvisersResultsStore(),
AnalysisResultsStore(),
InspectionResultsStore(),
ProvenanceResultsStore(),
PackageAnalysisResultsStore(),
SolverResultsStore(),
DependencyMonkeyReportsStore(),
)
_NAMESPACES_VARIABLES = [
"THOTH_FRONTEND_NAMESPACE",
"THOTH_MIDDLETIER_NAMESPACE",
"THOTH_BACKEND_NAMESPACE",
"THOTH_AMUN_NAMESPACE",
"THOTH_AMUN_INSPECTION_NAMESPACE",
]
_JOBS_LABELS = [
cf.go_offline()
# %% [markdown]
# ---
# %% [markdown]
# ## Retrieve inspection jobs from Ceph
# %% {"init_cell": true}
%env THOTH_DEPLOYMENT_NAME thoth-core-upshift-stage
%env THOTH_CEPH_BUCKET thoth
%env THOTH_CEPH_BUCKET_PREFIX data/thoth
%env THOTH_S3_ENDPOINT_URL https://s3.upshift.redhat.com/
# %% {"init_cell": true}
inspection_store = InspectionResultsStore(region="eu-central-1")
inspection_store.connect()
# %% [markdown]
# ---
# %% [markdown] {"heading_collapsed": true}
# ## Describe the structure of an inspection job log
# %% {"hidden": true}
def extract_structure_json(input_json, upper_key: str, level: int, json_structure):
"""Convert a json file structure into a list with rows showing tree depths, keys and values"""
level += 1
for key in input_json.keys():
if type(input_json[key]) is dict:
json_structure.append(