def main(
kebechet_analysis: Optional[bool],
kebechet_merge: Optional[bool],
gh_repo_analysis: Optional[bool],
subdir: Optional[str],
):
"""MI-Scheduler entrypoint."""
gh = Github(login_or_token=GITHUB_ACCESS_TOKEN)
oc = OpenShift()
# regular mi schedule
if gh_repo_analysis:
repos, orgs = oc.get_mi_repositories_and_organizations()
Schedule(
github=gh, openshift=oc, organizations=orgs, repositories=repos, subdir=subdir
).schedule_for_mi_analysis()
if kebechet_analysis:
graph = GraphDatabase()
graph.connect()
kebechet_repos = graph.get_active_kebechet_github_installations_repos()
Schedule(github=gh, openshift=oc, repositories=kebechet_repos, subdir=subdir).schedule_for_kebechet_analysis()
if kebechet_merge:
Schedule(openshift=oc, subdir=subdir).schedule_for_kebechet_merge()
def qeb_hwt_thamos_advise() -> None:
"""Qeb-Hwt Thamos Advise Task."""
if not Configuration._REPO_PATH:
raise Exception(
f"No path has been provided to REPO_PATH env variable.")
if not Path(Configuration._REPO_PATH).exists():
raise FileNotFoundError(
f"Cannot find the file on this path: {Configuration._REPO_PATH}")
OpenShift.verify_github_app_inputs(
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
)
os.chdir(Configuration._REPO_PATH)
thoth_yaml_config = _Configuration()
if not thoth_yaml_config.config_file_exists():
exception_message = _create_message_config_file_error(no_file=True)
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
return
try:
analysis_id = advise_here(
nowait=True,
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
source_type=ThothAdviserIntegrationEnum.GITHUB_APP,
)
_LOGGER.info("Successfully submitted thamos advise call.")
except Exception as exception:
if isinstance(
exception,
(
NoRuntimeEnvironmentError,
NoRequirementsFormatError,
FileNotFoundError,
FileLoadError,
KeyError,
ValueError,
AttributeError,
),
):
_LOGGER.debug(exception)
exception_message = str(exception)
else:
_LOGGER.debug(json.loads(exception.body)["error"])
exception_message = json.loads(exception.body)["error"]
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
def schedule_repositories(repositories: Set[str]) -> None:
"""Schedule workflows for repositories.
Repositories are also gathered from all of the organizations passed.
:param repositories:str: List of repositories in string format: repo1,repo2,...
"""
oc = OpenShift()
for repo in repositories:
workflow_id = oc.schedule_srcopsmetrics(repository=repo)
_LOGGER.info("Scheduled srcopsmetrics with id %r", workflow_id)
async def _schedule_security_indicator(openshift: OpenShift, package_name: str,
package_version: str,
index_url: str) -> int:
"""Schedule Security Indicator."""
try:
await wait_for_limit(
openshift,
workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE)
analysis_id = openshift.schedule_security_indicator(
python_package_name=package_name,
python_package_version=package_version,
python_package_index=index_url,
aggregation_function="process_data",
)
_LOGGER.info(
"Scheduled SI workflow for package %r in version %r from index %r, analysis is %r",
package_name,
package_version,
index_url,
analysis_id,
)
is_scheduled = 1
except Exception as e:
_LOGGER.exception(
f"Failed to schedule SI for package {package_name} in version {package_version} from index {index_url}: {e}"
)
is_scheduled = 0
return is_scheduled
def main():
"""MI-Scheduler entrypoint."""
gh = Github(login_or_token=GITHUB_ACCESS_TOKEN)
repos_raw, orgs = OpenShift().get_mi_repositories_and_organizations()
repos = set()
for org in orgs:
try:
gh_org = gh.get_organization(org)
for repo in gh_org.get_repos():
if repo.archived:
_LOGGER.info(
"repository %s is archived, therefore skipped",
repo.full_name)
else:
repos.add(repo.full_name)
except UnknownObjectException:
_LOGGER.error("organization %s was not recognized by GitHub API",
org)
for repo in repos_raw:
try:
if gh.get_repo(repo).archived:
_LOGGER.info("repository %s is archived, therefore skipped",
repo.full_name)
else:
repos.add(repo)
except UnknownObjectException:
_LOGGER.error("Repository %s was not recognized by GitHub API",
repo)
schedule_repositories(repositories=repos)
async def _schedule_revsolver(openshift: OpenShift, package_name: str,
package_version: str) -> int:
"""Schedule revsolver."""
try:
await wait_for_limit(
openshift,
workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE)
analysis_id = openshift.schedule_revsolver(
package_name=package_name,
package_version=package_version,
debug=Configuration.LOG_REVSOLVER)
_LOGGER.info(
"Scheduled reverse solver for package %r in version %r, analysis is %r",
package_name,
package_version,
analysis_id,
)
is_scheduled = 1
except Exception as e:
_LOGGER.exception(
"Failed to schedule reverse solver for %r in version %r: %r",
package_name, package_version, e)
is_scheduled = 0
return is_scheduled
def _schedule_solver(openshift: OpenShift, package_name: str,
package_version: str, indexes: List[str],
solver_name: str) -> int:
"""Schedule solver."""
try:
packages = f"{package_name}==={package_version}"
analysis_id = openshift.schedule_solver(solver=solver_name,
packages=packages,
indexes=indexes,
transitive=False,
debug=Configuration.LOG_SOLVER)
_LOGGER.info(
"Scheduled solver %r for packages %r from indexes %r, analysis is %r",
solver_name,
packages,
indexes,
analysis_id,
)
is_scheduled = 1
except Exception as e:
_LOGGER.exception(
f"Failed to schedule solver {solver_name} for package {packages} from {indexes}: {e}"
)
is_scheduled = 0
return is_scheduled
async def parse_adviser_trigger_message(adviser_trigger: Dict[str, Any],
openshift: OpenShift,
**kwargs) -> None:
"""Parse adviser trigger message."""
await wait_for_limit(
openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_id = openshift.schedule_adviser(
application_stack=adviser_trigger["application_stack"],
recommendation_type=adviser_trigger["recommendation_type"],
count=adviser_trigger["count"],
limit=adviser_trigger["limit"],
runtime_environment=adviser_trigger["runtime_environment"],
library_usage=adviser_trigger["library_usage"],
origin=adviser_trigger["origin"],
dev=adviser_trigger["dev"],
debug=adviser_trigger["debug"],
job_id=adviser_trigger["job_id"],
github_event_type=adviser_trigger["github_event_type"],
github_check_run_id=adviser_trigger["github_check_run_id"],
github_installation_id=adviser_trigger["github_installation_id"],
github_base_repo_url=adviser_trigger["github_base_repo_url"],
re_run_adviser_id=adviser_trigger["re_run_adviser_id"],
source_type=adviser_trigger["source_type"],
)
_LOGGER.debug(f"Scheduled adviser workflow {workflow_id}")
adviser_trigger_success.inc()
async def parse_build_analysis_trigger_message(build_analysis_trigger: Dict[
str, Any], openshift: OpenShift, **kwargs) -> None:
"""Parse build_analysis trigger message."""
await wait_for_limit(
openshift, workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE)
workflow_id = openshift.schedule_build_analysis(
base_image=build_analysis_trigger["base_image"],
base_image_analysis_id=build_analysis_trigger[
"base_image_analysis_id"],
base_registry_password=build_analysis_trigger[
"base_registry_password"],
base_registry_user=build_analysis_trigger["base_registry_user"],
base_registry_verify_tls=build_analysis_trigger[
"base_registry_verify_tls"],
output_image=build_analysis_trigger["output_image"],
output_image_analysis_id=build_analysis_trigger[
"output_image_analysis_id"],
output_registry_password=build_analysis_trigger[
"output_registry_password"],
output_registry_user=build_analysis_trigger["output_registry_user"],
output_registry_verify_tls=build_analysis_trigger[
"output_registry_verify_tls"],
buildlog_document_id=build_analysis_trigger["buildlog_document_id"],
buildlog_parser_id=build_analysis_trigger["buildlog_parser_id"],
environment_type=build_analysis_trigger["environment_type"],
origin=build_analysis_trigger["origin"],
debug=build_analysis_trigger["debug"],
job_id=build_analysis_trigger["job_id"],
)
_LOGGER.debug(f"Scheduled build analysis workflow {workflow_id}")
build_analysis_trigger_success.inc()
def _schedule_default_packages_solver_jobs(packages: List[str],
index_urls: List[str]) -> int:
"""Run solver jobs for Python packages list selected."""
openshift = OpenShift()
counter = 0
for index_url in index_urls:
_LOGGER.debug("consider index %r", index_url)
source = Source(index_url)
for package_name in packages:
_LOGGER.debug("Obtaining %r versions", package_name)
versions = []
try:
versions = source.get_package_versions(package_name)
except Exception as exc:
_LOGGER.exception(str(exc))
if versions:
for version in versions:
_LOGGER.info(
"Scheduling package_name %r in package_version %r",
package_name, version)
number_workflows = _do_schedule_solver_jobs(
openshift, index_urls, package_name, version)
counter += number_workflows
_LOGGER.info(f"Already scheduled {counter} solver workflows...")
return counter
async def parse_inspection_completed(inspection_completed: Dict[str, Any], openshift: OpenShift, **kwargs):
"""Schedule graph sync for inspection after completion."""
wait_for_limit(openshift=openshift, workflow_namespace=openshift.middletier_namespace)
workflow_name = openshift.schedule_graph_sync(
inspection_completed["inspection_id"], force_sync=inspection_completed["force_sync"]
)
_LOGGER.debug(f"Graph sync workflow, {workflow_name}, for inspection {inspection_completed['inspection_id']}")
inspection_completed_success.inc()
async def parse_kebechet_trigger_message(kebechet_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None:
"""Parse kebechet_trigger message."""
await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_name = openshift.schedule_kebechet_workflow(
webhook_payload=kebechet_trigger["webhook_payload"],
job_id=kebechet_trigger["job_id"],
)
_LOGGER.debug(f"Scheduled kebechet workflow {workflow_name}")
kebechet_trigger_success.inc()
async def parse_thoth_repo_init_message(repo_init: Dict[str, Any],
openshift: OpenShift,
**kwargs) -> None:
"""Parse thoth_repo_init message."""
await wait_for_limit(
openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_name = openshift.schedule_thoth_repo_init(
project_url=repo_init["project_url"])
_LOGGER.debug(f"Scheduled kebechet workflow {workflow_name}")
thoth_repo_init_success.inc()
def get_configmaps_per_namespace_per_label():
"""Get the total number of configmaps in the namespace based on labels."""
namespaces = get_namespaces()
_OPENSHIFT = OpenShift()
for namespace in namespaces:
_LOGGER.info("Evaluating configmaps metrics for Thoth namespace: %r",
namespace)
for label in _JOBS_LABELS + [
"operator=graph-sync", "operator=workload"
]:
config_maps_items = _OPENSHIFT.get_configmaps(namespace=namespace,
label_selector=label)
number_configmaps = count_configmaps(config_maps_items)
metrics.config_maps_number.labels(namespace,
label).set(number_configmaps)
_LOGGER.debug(
"thoth_config_maps_number=%r, in namespace=%r for label=%r",
number_configmaps, namespace, label)
async def schedule_kebechet_administrator(openshift: OpenShift, message_info: dict, message_name: str) -> int:
"""Schedule Kebechet Administrator from a particular message."""
workflow_id = 0
try:
await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_id = openshift.schedule_kebechet_administrator(message_info=message_info, message_type=message_name)
_LOGGER.info(
f"Scheduled Kebechet Administrator worflow for message type {message_name} with workflow id - {workflow_id}"
)
except Exception as e:
_LOGGER.exception(f"Failed to schedule Kebechet Administrator worflow for message type {message_name}: {e}")
return workflow_id
def openshift(cls):
"""Get instantiated openshift class."""
if not cls._OPENSHIFT:
try:
cls._OPENSHIFT = OpenShift()
except Exception as e:
metrics.openshift_connection_error_status.set(1)
raise Exception("Raise a flag if there is an error connecting to openshift. %r", e)
else:
metrics.openshift_connection_error_status.set(0)
return cls._OPENSHIFT
def _event_producer(queue: Queue, build_watcher_namespace: str) -> None:
"""Accept events from the cluster and queue them into work queue processed by the main process."""
_LOGGER.info("Starting event producer")
openshift = OpenShift()
v1_build = openshift.ocp_client.resources.get(api_version="v1",
kind="Build")
for event in v1_build.watch(namespace=build_watcher_namespace):
event_name = event["object"].metadata.name
build_reference = {
"build_log_reference": _buildlog_metadata(),
"base_input_reference": None,
"output_reference": None,
}
if event["object"].status.phase != "Complete":
_LOGGER.debug(
"Ignoring build event for %r - not completed phase %r",
event_name, event["object"].status.phase)
continue
elif event["object"].status.phase == "Failed":
_LOGGER.debug(
"Submitting base_image and build_log as build event for %r - the phase is %r",
event_name,
event["object"].status.phase,
)
strategy = event["object"].spec.strategy
build_reference = _get_build(openshift, strategy, build_reference,
event["object"].metadata)
_LOGGER.info(
"Queueing build log based on build event %r for further processing",
event_name)
_METRIC_BUILDS_FAILED.inc()
try:
_LOGGER.info("Submitting metrics to Prometheus pushgateway %r",
_THOTH_METRICS_PUSHGATEWAY_URL)
push_to_gateway(_THOTH_METRICS_PUSHGATEWAY_URL,
job="build-watcher",
registry=prometheus_registry)
except Exception as e:
_LOGGER.exception(
f"An error occurred pushing the metrics: {str(e)}")
queue.put(build_reference)
continue
_LOGGER.debug("New build event: %s", str(event))
build_reference["output_reference"] = event[
"object"].status.outputDockerImageReference
strategy = event["object"].spec.strategy
build_reference = _get_build(openshift, strategy, build_reference,
event["object"].metadata)
_LOGGER.info(
"Queueing build log based on build event %r for further processing",
event_name)
queue.put(build_reference)
async def parse_kebechet_run_url_trigger_message(
kebechet_run_url_trigger: Dict[str, Any], openshift: OpenShift, **kwargs
) -> None:
"""Parse kebechet_run_url_trigger message."""
await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_name = openshift.schedule_kebechet_run_url_workflow(
repo_url=kebechet_run_url_trigger["url"],
service_name=kebechet_run_url_trigger["service_name"],
job_id=kebechet_run_url_trigger["job_id"],
)
_LOGGER.debug(f"Scheduled kebechet run url workflow {workflow_name}")
kebechet_run_url_trigger_success.inc()
def main():
"""MI-Scheduler entrypoint."""
oc = OpenShift()
cm = oc.get_configmap(configmap_id="mi-scheduler",
namespace="thoth-test-core")
organizations = cm.get("organizations", "")
repositories = cm.get("repositories", "")
gh = Github()
repos = set()
orgs = organizations.split(",") if organizations is not None else []
for org in orgs:
try:
gh_org = gh.get_organization(org)
for repo in gh_org.get_repos():
if repo.archived:
_LOGGER.info(
"repository %s is archived, therefore skipped",
repo.full_name)
else:
repos.add(repo.full_name)
except GithubException:
_LOGGER.error("organization %s was not recognized by GitHub API",
org)
repos_raw = repositories.split(",") if repositories is not None else []
for repo in repos_raw:
try:
if gh.get_repo(repo).archived:
_LOGGER.info("repository %s is archived, therefore skipped",
repo.full_name)
else:
repos.add(repo)
except GithubException:
_LOGGER.error("Repository %s was not recognized by GitHub API",
repo)
schedule_repositories(repositories=repos)
async def learn_using_solver(
openshift: OpenShift,
graph: GraphDatabase,
is_present: bool,
package_name: str,
index_url: str,
package_version: str,
solver: Optional[str] = None,
) -> int:
"""Learn using solver about Package Version Index dependencies."""
if not is_present:
# Package never seen (schedule all solver workflows to collect all knowledge for Thoth)
are_solvers_scheduled = await _schedule_all_solvers(
openshift=openshift,
package_name=package_name,
package_version=package_version,
indexes=[index_url])
return are_solvers_scheduled
# Select solvers
if not solver:
solvers: List[str] = openshift.get_solver_names()
else:
solvers = [solver]
dependency_indexes = graph.get_python_package_index_urls_all(enabled=True)
# Check for which solver has not been solved and schedule solver workflow
are_solvers_scheduled = 0
for solver_name in solvers:
is_solved = graph.python_package_version_exists(
package_name=package_name,
package_version=package_version,
index_url=index_url,
solver_name=solver_name)
if not is_solved:
is_solver_scheduled = _schedule_solver(
openshift=openshift,
package_name=package_name,
package_version=package_version,
indexes=[index_url],
dependency_indexes=dependency_indexes,
solver_name=solver_name,
)
are_solvers_scheduled += is_solver_scheduled
return are_solvers_scheduled
def _do_schedule_solver_jobs(
openshift: OpenShift, index_urls: List[str], package_name: str, package_version: str
) -> int:
"""Run Python solvers for the given package in specified version."""
_LOGGER.info(
"Running solver job for package %r in version %r, results will be scheduled using Argo workflow",
package_name,
package_version,
)
solvers_run = openshift.schedule_all_solvers(packages=f"{package_name}==={package_version}", indexes=index_urls)
_LOGGER.debug("Response when running solver jobs: %r", solvers_run)
return len(solvers_run)
def event_producer(queue: Queue, scheduler_namespace: str):
"""Listen for relevant events in the cluster and schedule them for further processing by parent process."""
# Note that jobs do not support field selector pointing to phase (we could
# do it on pod level, but that is not desired).
openshift = OpenShift()
v1_jobs = openshift.ocp_client.resources.get(api_version="batch/v1",
kind="Job")
for event in v1_jobs.watch(namespace=scheduler_namespace,
label_selector="operator=graph-sync"):
_LOGGER.debug("Checking event for %r", event["object"].metadata.name)
if event["type"] != "MODIFIED":
# Skip additions and deletions...
_LOGGER.debug("Skipping event, not modification event: %s",
event["type"])
continue
if not event["object"].status.succeeded and not event[
"object"].status.failed:
# Skip modified events signalizing pod being scheduled.
# We also check for success of failed - the monitored jobs are
# configured to run once - if they fail they are not restarted.
# Thus continue on failed.
_LOGGER.debug(
"Skipping event, no succeeded nor failed in status reported: %s",
event["object"].status)
continue
task_name = event["object"].metadata.labels.task
_LOGGER.info("Handling event for %r (task: %r)",
event["object"].metadata.name, task_name)
target = _CONFIG.get(task_name)
if not target:
_LOGGER.error(
"No configuration entry provided for task %r in graph sync operator",
task_name)
continue
method, sync_failed = target
if not sync_failed and event["object"].status.failed:
_LOGGER.info(
"Skipping failed job %r as scheduler was not configured to perform sync on failed jobs",
event["object"].metadata.name,
)
continue
# Document id directly matches job name.
document_id = event["object"].metadata.name
queue.put((method.__name__, document_id))
def _handle_solved_message(Configuration): # noqa: N803
"""Handle all the messages for which Kebechet needs to run on if the sovler type matches the os type."""
solver_string = os.environ["THOTH_SOLVER_NAME"] # ex - solver-fedora-31-py38
if not solver_string:
raise ValueError(
"SolverMessageType has been provided to the MESSAGE_TYPE env variable. \
but solver name is missing."
)
solver_dict = OpenShift.parse_python_solver_name(solver_string)
os_name, os_version, python_version = (
solver_dict["os_name"],
solver_dict["os_version"],
solver_dict["python_version"],
)
repositories: Dict[str, Dict] = GRAPH.get_kebechet_github_installations_info_for_python_package_version(
package_name=Configuration.PACKAGE_NAME,
index_url=Configuration.PACKAGE_INDEX,
os_name=os_name,
os_version=os_version,
python_version=python_version,
) # We query without the package_version.
for key in repositories.keys():
repo_info = repositories[key]
# Construct the message input
if repo_info.get("private"):
continue # We skip for private repo's.
if semver.compare(repo_info.get("package_version"), Configuration.PACKAGE_VERSION):
continue # We dont schedule, if the package version > version of the solved package version.
message_input = KebechetRunUrlMessageContents(
component_name=__COMPONENT_NAME__,
service_version=__service_version__,
url=_URL_PREFIX + key,
service_name="github",
installation_id=repo_info.get("installation_id"),
metadata={
"message_justification": _JUSTIFICATION_MAPPING[Configuration.MESSAGE_TYPE],
"package_name": Configuration.PACKAGE_NAME,
"package_version": Configuration.PACKAGE_VERSION,
"package_index": Configuration.PACKAGE_INDEX,
},
).dict()
# We store the message to put in the output file here.
output_messages.append(
{"topic_name": kebechet_run_url_trigger_message.base_name, "message_contents": message_input}
)
def cli(scheduler_namespace: str,
graph_sync_namespace: str,
verbose: bool = False):
"""Scheduler handling Thoth's graph syncs."""
if verbose:
_LOGGER.setLevel(logging.DEBUG)
_LOGGER.info(
"Graph sync scheduler is running thoth-common in version %r, built from %r",
thoth_common_version, os.getenv("OPENSHIFT_BUILD_COMMIT"))
_LOGGER.info(
"Graph sync scheduler is watching namespace %r and scheduling graph syncs in namespace %r",
scheduler_namespace, graph_sync_namespace)
openshift = OpenShift()
queue = Queue()
producer = Process(target=event_producer,
args=(queue, scheduler_namespace))
producer.start()
while producer.is_alive():
template_name, document_id = queue.get()
try:
graph_sync_id = openshift.schedule_graph_sync(
document_id, graph_sync_namespace, template_name=template_name)
_LOGGER.info("Scheduled new graph sync with id %r", graph_sync_id)
except Exception as exc:
_LOGGER.exception(
"Failed to run sync for document id %r, the template to be triggered was %r: %s",
document_id, template_name, exc)
producer.join()
# Always fail, this should be run forever.
sys.exit(1)
async def parse_provenance_checker_trigger_message(
provenance_checker_trigger: Dict[str, Any], openshift: OpenShift,
**kwargs) -> None:
"""Parse provenance_checker_trigger message."""
await wait_for_limit(
openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_name = openshift.schedule_provenance_checker(
application_stack=provenance_checker_trigger["application_stack"],
origin=provenance_checker_trigger["origin"],
whitelisted_sources=provenance_checker_trigger["whitelisted_sources"],
debug=provenance_checker_trigger["debug"],
job_id=provenance_checker_trigger["job_id"],
)
_LOGGER.debug(f"Scheduled provenance checker workflow {workflow_name}")
provenance_checker_trigger_success.inc()
async def parse_qebhwt_trigger_message(qebhwt_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None:
"""Parse qebhwt_trigger message."""
await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_name = openshift.schedule_qebhwt_workflow(
github_event_type=qebhwt_trigger["github_event_type"],
github_check_run_id=qebhwt_trigger["github_check_run_id"],
github_installation_id=qebhwt_trigger["github_installation_id"],
github_base_repo_url=qebhwt_trigger["github_base_repo_url"],
github_head_repo_url=qebhwt_trigger["github_head_repo_url"],
origin=qebhwt_trigger["origin"],
revision=qebhwt_trigger["revision"],
host=qebhwt_trigger["host"],
job_id=qebhwt_trigger["job_id"],
)
_LOGGER.debug(f"Scheduled qebhwt workflow {workflow_name}")
qebhwt_trigger_success.inc()
def event_producer(queue: Queue, operator_namespace: str):
"""Queue events to be processed coming from the cluster."""
_LOGGER.info("Starting event producer")
openshift = OpenShift()
v1_configmap = openshift.ocp_client.resources.get(api_version="v1",
kind="ConfigMap")
for event in v1_configmap.watch(namespace=operator_namespace,
label_selector="operator=workload"):
if event["type"] != "ADDED":
_LOGGER.debug("Skipping event, not addition event (type: %r)",
event["type"])
continue
configmap_name = event["object"].metadata.name
_LOGGER.info("Queuing event %r for processing", configmap_name)
queue.put(configmap_name)
async def _schedule_all_solvers(
openshift: OpenShift, package_name: str, package_version: str, indexes: List[str]
) -> int:
"""Schedule all solvers."""
packages = f"{package_name}==={package_version}"
try:
await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE)
analysis_ids = openshift.schedule_all_solvers(packages=packages, indexes=indexes)
_LOGGER.info(
"Scheduled solvers for packages %r from indexes %r, analysis ids are %r", packages, indexes, analysis_ids
)
are_scheduled = len(analysis_ids)
except Exception as e:
_LOGGER.exception(f"Failed to schedule solvers for package {packages} from {indexes}: {e}")
raise e
return are_scheduled
async def parse_provenance_checker_trigger_message(
provenance_checker_trigger: Dict[str, Any], openshift: OpenShift,
**kwargs) -> None:
"""Parse provenance_checker_trigger message."""
await wait_for_limit(
openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE)
workflow_name = openshift.schedule_provenance_checker(
origin=provenance_checker_trigger["origin"],
authenticated=provenance_checker_trigger.get("authenticated", False),
whitelisted_sources=provenance_checker_trigger["whitelisted_sources"],
debug=provenance_checker_trigger["debug"],
job_id=provenance_checker_trigger["job_id"],
kebechet_metadata=provenance_checker_trigger.get("kebechet_metadata"),
stack_info=provenance_checker_trigger.get("stack_info"),
justification=provenance_checker_trigger.get("justification"),
)
_LOGGER.debug(f"Scheduled provenance checker workflow {workflow_name}")
provenance_checker_trigger_success.inc()
def _existing_producer(queue: Queue, build_watcher_namespace: str) -> None:
"""Query for existing images in image streams and queue them for analysis."""
openshift = OpenShift()
v1_imagestreams = openshift.ocp_client.resources.get(
api_version="image.openshift.io/v1", kind="ImageStream")
for item in v1_imagestreams.get(namespace=build_watcher_namespace).items:
_LOGGER.debug("Found imagestream item: %s", str(item))
_LOGGER.debug("Listing tags available for %r",
item["metadata"]["name"])
for tag_info in item.status.tags or []:
output_reference = f"{item.status.dockerImageRepository}:{tag_info.tag}"
_LOGGER.info("Queueing already existing image %r for analysis",
output_reference)
queue.put(output_reference)
_LOGGER.info(
"Queuing existing images for analyses has finished, all of them were scheduled for analysis"
)
