def main( kebechet_analysis: Optional[bool], kebechet_merge: Optional[bool], gh_repo_analysis: Optional[bool], subdir: Optional[str], ): """MI-Scheduler entrypoint.""" gh = Github(login_or_token=GITHUB_ACCESS_TOKEN) oc = OpenShift() # regular mi schedule if gh_repo_analysis: repos, orgs = oc.get_mi_repositories_and_organizations() Schedule( github=gh, openshift=oc, organizations=orgs, repositories=repos, subdir=subdir ).schedule_for_mi_analysis() if kebechet_analysis: graph = GraphDatabase() graph.connect() kebechet_repos = graph.get_active_kebechet_github_installations_repos() Schedule(github=gh, openshift=oc, repositories=kebechet_repos, subdir=subdir).schedule_for_kebechet_analysis() if kebechet_merge: Schedule(openshift=oc, subdir=subdir).schedule_for_kebechet_merge()
def qeb_hwt_thamos_advise() -> None: """Qeb-Hwt Thamos Advise Task.""" if not Configuration._REPO_PATH: raise Exception( f"No path has been provided to REPO_PATH env variable.") if not Path(Configuration._REPO_PATH).exists(): raise FileNotFoundError( f"Cannot find the file on this path: {Configuration._REPO_PATH}") OpenShift.verify_github_app_inputs( github_event_type=Configuration._GITHUB_EVENT_TYPE, github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID, github_installation_id=Configuration._GITHUB_INSTALLATION_ID, github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL, origin=Configuration._ORIGIN, ) os.chdir(Configuration._REPO_PATH) thoth_yaml_config = _Configuration() if not thoth_yaml_config.config_file_exists(): exception_message = _create_message_config_file_error(no_file=True) trigger_finished_webhook(exception_message=exception_message, has_error=True) return try: analysis_id = advise_here( nowait=True, github_event_type=Configuration._GITHUB_EVENT_TYPE, github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID, github_installation_id=Configuration._GITHUB_INSTALLATION_ID, github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL, origin=Configuration._ORIGIN, source_type=ThothAdviserIntegrationEnum.GITHUB_APP, ) _LOGGER.info("Successfully submitted thamos advise call.") except Exception as exception: if isinstance( exception, ( NoRuntimeEnvironmentError, NoRequirementsFormatError, FileNotFoundError, FileLoadError, KeyError, ValueError, AttributeError, ), ): _LOGGER.debug(exception) exception_message = str(exception) else: _LOGGER.debug(json.loads(exception.body)["error"]) exception_message = json.loads(exception.body)["error"] trigger_finished_webhook(exception_message=exception_message, has_error=True)
def schedule_repositories(repositories: Set[str]) -> None: """Schedule workflows for repositories. Repositories are also gathered from all of the organizations passed. :param repositories:str: List of repositories in string format: repo1,repo2,... """ oc = OpenShift() for repo in repositories: workflow_id = oc.schedule_srcopsmetrics(repository=repo) _LOGGER.info("Scheduled srcopsmetrics with id %r", workflow_id)
async def _schedule_security_indicator(openshift: OpenShift, package_name: str, package_version: str, index_url: str) -> int: """Schedule Security Indicator.""" try: await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE) analysis_id = openshift.schedule_security_indicator( python_package_name=package_name, python_package_version=package_version, python_package_index=index_url, aggregation_function="process_data", ) _LOGGER.info( "Scheduled SI workflow for package %r in version %r from index %r, analysis is %r", package_name, package_version, index_url, analysis_id, ) is_scheduled = 1 except Exception as e: _LOGGER.exception( f"Failed to schedule SI for package {package_name} in version {package_version} from index {index_url}: {e}" ) is_scheduled = 0 return is_scheduled
def main(): """MI-Scheduler entrypoint.""" gh = Github(login_or_token=GITHUB_ACCESS_TOKEN) repos_raw, orgs = OpenShift().get_mi_repositories_and_organizations() repos = set() for org in orgs: try: gh_org = gh.get_organization(org) for repo in gh_org.get_repos(): if repo.archived: _LOGGER.info( "repository %s is archived, therefore skipped", repo.full_name) else: repos.add(repo.full_name) except UnknownObjectException: _LOGGER.error("organization %s was not recognized by GitHub API", org) for repo in repos_raw: try: if gh.get_repo(repo).archived: _LOGGER.info("repository %s is archived, therefore skipped", repo.full_name) else: repos.add(repo) except UnknownObjectException: _LOGGER.error("Repository %s was not recognized by GitHub API", repo) schedule_repositories(repositories=repos)
async def _schedule_revsolver(openshift: OpenShift, package_name: str, package_version: str) -> int: """Schedule revsolver.""" try: await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE) analysis_id = openshift.schedule_revsolver( package_name=package_name, package_version=package_version, debug=Configuration.LOG_REVSOLVER) _LOGGER.info( "Scheduled reverse solver for package %r in version %r, analysis is %r", package_name, package_version, analysis_id, ) is_scheduled = 1 except Exception as e: _LOGGER.exception( "Failed to schedule reverse solver for %r in version %r: %r", package_name, package_version, e) is_scheduled = 0 return is_scheduled
def _schedule_solver(openshift: OpenShift, package_name: str, package_version: str, indexes: List[str], solver_name: str) -> int: """Schedule solver.""" try: packages = f"{package_name}==={package_version}" analysis_id = openshift.schedule_solver(solver=solver_name, packages=packages, indexes=indexes, transitive=False, debug=Configuration.LOG_SOLVER) _LOGGER.info( "Scheduled solver %r for packages %r from indexes %r, analysis is %r", solver_name, packages, indexes, analysis_id, ) is_scheduled = 1 except Exception as e: _LOGGER.exception( f"Failed to schedule solver {solver_name} for package {packages} from {indexes}: {e}" ) is_scheduled = 0 return is_scheduled
async def parse_adviser_trigger_message(adviser_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None: """Parse adviser trigger message.""" await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_id = openshift.schedule_adviser( application_stack=adviser_trigger["application_stack"], recommendation_type=adviser_trigger["recommendation_type"], count=adviser_trigger["count"], limit=adviser_trigger["limit"], runtime_environment=adviser_trigger["runtime_environment"], library_usage=adviser_trigger["library_usage"], origin=adviser_trigger["origin"], dev=adviser_trigger["dev"], debug=adviser_trigger["debug"], job_id=adviser_trigger["job_id"], github_event_type=adviser_trigger["github_event_type"], github_check_run_id=adviser_trigger["github_check_run_id"], github_installation_id=adviser_trigger["github_installation_id"], github_base_repo_url=adviser_trigger["github_base_repo_url"], re_run_adviser_id=adviser_trigger["re_run_adviser_id"], source_type=adviser_trigger["source_type"], ) _LOGGER.debug(f"Scheduled adviser workflow {workflow_id}") adviser_trigger_success.inc()
async def parse_build_analysis_trigger_message(build_analysis_trigger: Dict[ str, Any], openshift: OpenShift, **kwargs) -> None: """Parse build_analysis trigger message.""" await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE) workflow_id = openshift.schedule_build_analysis( base_image=build_analysis_trigger["base_image"], base_image_analysis_id=build_analysis_trigger[ "base_image_analysis_id"], base_registry_password=build_analysis_trigger[ "base_registry_password"], base_registry_user=build_analysis_trigger["base_registry_user"], base_registry_verify_tls=build_analysis_trigger[ "base_registry_verify_tls"], output_image=build_analysis_trigger["output_image"], output_image_analysis_id=build_analysis_trigger[ "output_image_analysis_id"], output_registry_password=build_analysis_trigger[ "output_registry_password"], output_registry_user=build_analysis_trigger["output_registry_user"], output_registry_verify_tls=build_analysis_trigger[ "output_registry_verify_tls"], buildlog_document_id=build_analysis_trigger["buildlog_document_id"], buildlog_parser_id=build_analysis_trigger["buildlog_parser_id"], environment_type=build_analysis_trigger["environment_type"], origin=build_analysis_trigger["origin"], debug=build_analysis_trigger["debug"], job_id=build_analysis_trigger["job_id"], ) _LOGGER.debug(f"Scheduled build analysis workflow {workflow_id}") build_analysis_trigger_success.inc()
def _schedule_default_packages_solver_jobs(packages: List[str], index_urls: List[str]) -> int: """Run solver jobs for Python packages list selected.""" openshift = OpenShift() counter = 0 for index_url in index_urls: _LOGGER.debug("consider index %r", index_url) source = Source(index_url) for package_name in packages: _LOGGER.debug("Obtaining %r versions", package_name) versions = [] try: versions = source.get_package_versions(package_name) except Exception as exc: _LOGGER.exception(str(exc)) if versions: for version in versions: _LOGGER.info( "Scheduling package_name %r in package_version %r", package_name, version) number_workflows = _do_schedule_solver_jobs( openshift, index_urls, package_name, version) counter += number_workflows _LOGGER.info(f"Already scheduled {counter} solver workflows...") return counter
async def parse_inspection_completed(inspection_completed: Dict[str, Any], openshift: OpenShift, **kwargs): """Schedule graph sync for inspection after completion.""" wait_for_limit(openshift=openshift, workflow_namespace=openshift.middletier_namespace) workflow_name = openshift.schedule_graph_sync( inspection_completed["inspection_id"], force_sync=inspection_completed["force_sync"] ) _LOGGER.debug(f"Graph sync workflow, {workflow_name}, for inspection {inspection_completed['inspection_id']}") inspection_completed_success.inc()
async def parse_kebechet_trigger_message(kebechet_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None: """Parse kebechet_trigger message.""" await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_name = openshift.schedule_kebechet_workflow( webhook_payload=kebechet_trigger["webhook_payload"], job_id=kebechet_trigger["job_id"], ) _LOGGER.debug(f"Scheduled kebechet workflow {workflow_name}") kebechet_trigger_success.inc()
async def parse_thoth_repo_init_message(repo_init: Dict[str, Any], openshift: OpenShift, **kwargs) -> None: """Parse thoth_repo_init message.""" await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_name = openshift.schedule_thoth_repo_init( project_url=repo_init["project_url"]) _LOGGER.debug(f"Scheduled kebechet workflow {workflow_name}") thoth_repo_init_success.inc()
def get_configmaps_per_namespace_per_label(): """Get the total number of configmaps in the namespace based on labels.""" namespaces = get_namespaces() _OPENSHIFT = OpenShift() for namespace in namespaces: _LOGGER.info("Evaluating configmaps metrics for Thoth namespace: %r", namespace) for label in _JOBS_LABELS + [ "operator=graph-sync", "operator=workload" ]: config_maps_items = _OPENSHIFT.get_configmaps(namespace=namespace, label_selector=label) number_configmaps = count_configmaps(config_maps_items) metrics.config_maps_number.labels(namespace, label).set(number_configmaps) _LOGGER.debug( "thoth_config_maps_number=%r, in namespace=%r for label=%r", number_configmaps, namespace, label)
async def schedule_kebechet_administrator(openshift: OpenShift, message_info: dict, message_name: str) -> int: """Schedule Kebechet Administrator from a particular message.""" workflow_id = 0 try: await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_id = openshift.schedule_kebechet_administrator(message_info=message_info, message_type=message_name) _LOGGER.info( f"Scheduled Kebechet Administrator worflow for message type {message_name} with workflow id - {workflow_id}" ) except Exception as e: _LOGGER.exception(f"Failed to schedule Kebechet Administrator worflow for message type {message_name}: {e}") return workflow_id
def openshift(cls): """Get instantiated openshift class.""" if not cls._OPENSHIFT: try: cls._OPENSHIFT = OpenShift() except Exception as e: metrics.openshift_connection_error_status.set(1) raise Exception("Raise a flag if there is an error connecting to openshift. %r", e) else: metrics.openshift_connection_error_status.set(0) return cls._OPENSHIFT
def _event_producer(queue: Queue, build_watcher_namespace: str) -> None: """Accept events from the cluster and queue them into work queue processed by the main process.""" _LOGGER.info("Starting event producer") openshift = OpenShift() v1_build = openshift.ocp_client.resources.get(api_version="v1", kind="Build") for event in v1_build.watch(namespace=build_watcher_namespace): event_name = event["object"].metadata.name build_reference = { "build_log_reference": _buildlog_metadata(), "base_input_reference": None, "output_reference": None, } if event["object"].status.phase != "Complete": _LOGGER.debug( "Ignoring build event for %r - not completed phase %r", event_name, event["object"].status.phase) continue elif event["object"].status.phase == "Failed": _LOGGER.debug( "Submitting base_image and build_log as build event for %r - the phase is %r", event_name, event["object"].status.phase, ) strategy = event["object"].spec.strategy build_reference = _get_build(openshift, strategy, build_reference, event["object"].metadata) _LOGGER.info( "Queueing build log based on build event %r for further processing", event_name) _METRIC_BUILDS_FAILED.inc() try: _LOGGER.info("Submitting metrics to Prometheus pushgateway %r", _THOTH_METRICS_PUSHGATEWAY_URL) push_to_gateway(_THOTH_METRICS_PUSHGATEWAY_URL, job="build-watcher", registry=prometheus_registry) except Exception as e: _LOGGER.exception( f"An error occurred pushing the metrics: {str(e)}") queue.put(build_reference) continue _LOGGER.debug("New build event: %s", str(event)) build_reference["output_reference"] = event[ "object"].status.outputDockerImageReference strategy = event["object"].spec.strategy build_reference = _get_build(openshift, strategy, build_reference, event["object"].metadata) _LOGGER.info( "Queueing build log based on build event %r for further processing", event_name) queue.put(build_reference)
async def parse_kebechet_run_url_trigger_message( kebechet_run_url_trigger: Dict[str, Any], openshift: OpenShift, **kwargs ) -> None: """Parse kebechet_run_url_trigger message.""" await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_name = openshift.schedule_kebechet_run_url_workflow( repo_url=kebechet_run_url_trigger["url"], service_name=kebechet_run_url_trigger["service_name"], job_id=kebechet_run_url_trigger["job_id"], ) _LOGGER.debug(f"Scheduled kebechet run url workflow {workflow_name}") kebechet_run_url_trigger_success.inc()
def main(): """MI-Scheduler entrypoint.""" oc = OpenShift() cm = oc.get_configmap(configmap_id="mi-scheduler", namespace="thoth-test-core") organizations = cm.get("organizations", "") repositories = cm.get("repositories", "") gh = Github() repos = set() orgs = organizations.split(",") if organizations is not None else [] for org in orgs: try: gh_org = gh.get_organization(org) for repo in gh_org.get_repos(): if repo.archived: _LOGGER.info( "repository %s is archived, therefore skipped", repo.full_name) else: repos.add(repo.full_name) except GithubException: _LOGGER.error("organization %s was not recognized by GitHub API", org) repos_raw = repositories.split(",") if repositories is not None else [] for repo in repos_raw: try: if gh.get_repo(repo).archived: _LOGGER.info("repository %s is archived, therefore skipped", repo.full_name) else: repos.add(repo) except GithubException: _LOGGER.error("Repository %s was not recognized by GitHub API", repo) schedule_repositories(repositories=repos)
async def learn_using_solver( openshift: OpenShift, graph: GraphDatabase, is_present: bool, package_name: str, index_url: str, package_version: str, solver: Optional[str] = None, ) -> int: """Learn using solver about Package Version Index dependencies.""" if not is_present: # Package never seen (schedule all solver workflows to collect all knowledge for Thoth) are_solvers_scheduled = await _schedule_all_solvers( openshift=openshift, package_name=package_name, package_version=package_version, indexes=[index_url]) return are_solvers_scheduled # Select solvers if not solver: solvers: List[str] = openshift.get_solver_names() else: solvers = [solver] dependency_indexes = graph.get_python_package_index_urls_all(enabled=True) # Check for which solver has not been solved and schedule solver workflow are_solvers_scheduled = 0 for solver_name in solvers: is_solved = graph.python_package_version_exists( package_name=package_name, package_version=package_version, index_url=index_url, solver_name=solver_name) if not is_solved: is_solver_scheduled = _schedule_solver( openshift=openshift, package_name=package_name, package_version=package_version, indexes=[index_url], dependency_indexes=dependency_indexes, solver_name=solver_name, ) are_solvers_scheduled += is_solver_scheduled return are_solvers_scheduled
def _do_schedule_solver_jobs( openshift: OpenShift, index_urls: List[str], package_name: str, package_version: str ) -> int: """Run Python solvers for the given package in specified version.""" _LOGGER.info( "Running solver job for package %r in version %r, results will be scheduled using Argo workflow", package_name, package_version, ) solvers_run = openshift.schedule_all_solvers(packages=f"{package_name}==={package_version}", indexes=index_urls) _LOGGER.debug("Response when running solver jobs: %r", solvers_run) return len(solvers_run)
def event_producer(queue: Queue, scheduler_namespace: str): """Listen for relevant events in the cluster and schedule them for further processing by parent process.""" # Note that jobs do not support field selector pointing to phase (we could # do it on pod level, but that is not desired). openshift = OpenShift() v1_jobs = openshift.ocp_client.resources.get(api_version="batch/v1", kind="Job") for event in v1_jobs.watch(namespace=scheduler_namespace, label_selector="operator=graph-sync"): _LOGGER.debug("Checking event for %r", event["object"].metadata.name) if event["type"] != "MODIFIED": # Skip additions and deletions... _LOGGER.debug("Skipping event, not modification event: %s", event["type"]) continue if not event["object"].status.succeeded and not event[ "object"].status.failed: # Skip modified events signalizing pod being scheduled. # We also check for success of failed - the monitored jobs are # configured to run once - if they fail they are not restarted. # Thus continue on failed. _LOGGER.debug( "Skipping event, no succeeded nor failed in status reported: %s", event["object"].status) continue task_name = event["object"].metadata.labels.task _LOGGER.info("Handling event for %r (task: %r)", event["object"].metadata.name, task_name) target = _CONFIG.get(task_name) if not target: _LOGGER.error( "No configuration entry provided for task %r in graph sync operator", task_name) continue method, sync_failed = target if not sync_failed and event["object"].status.failed: _LOGGER.info( "Skipping failed job %r as scheduler was not configured to perform sync on failed jobs", event["object"].metadata.name, ) continue # Document id directly matches job name. document_id = event["object"].metadata.name queue.put((method.__name__, document_id))
def _handle_solved_message(Configuration): # noqa: N803 """Handle all the messages for which Kebechet needs to run on if the sovler type matches the os type.""" solver_string = os.environ["THOTH_SOLVER_NAME"] # ex - solver-fedora-31-py38 if not solver_string: raise ValueError( "SolverMessageType has been provided to the MESSAGE_TYPE env variable. \ but solver name is missing." ) solver_dict = OpenShift.parse_python_solver_name(solver_string) os_name, os_version, python_version = ( solver_dict["os_name"], solver_dict["os_version"], solver_dict["python_version"], ) repositories: Dict[str, Dict] = GRAPH.get_kebechet_github_installations_info_for_python_package_version( package_name=Configuration.PACKAGE_NAME, index_url=Configuration.PACKAGE_INDEX, os_name=os_name, os_version=os_version, python_version=python_version, ) # We query without the package_version. for key in repositories.keys(): repo_info = repositories[key] # Construct the message input if repo_info.get("private"): continue # We skip for private repo's. if semver.compare(repo_info.get("package_version"), Configuration.PACKAGE_VERSION): continue # We dont schedule, if the package version > version of the solved package version. message_input = KebechetRunUrlMessageContents( component_name=__COMPONENT_NAME__, service_version=__service_version__, url=_URL_PREFIX + key, service_name="github", installation_id=repo_info.get("installation_id"), metadata={ "message_justification": _JUSTIFICATION_MAPPING[Configuration.MESSAGE_TYPE], "package_name": Configuration.PACKAGE_NAME, "package_version": Configuration.PACKAGE_VERSION, "package_index": Configuration.PACKAGE_INDEX, }, ).dict() # We store the message to put in the output file here. output_messages.append( {"topic_name": kebechet_run_url_trigger_message.base_name, "message_contents": message_input} )
def cli(scheduler_namespace: str, graph_sync_namespace: str, verbose: bool = False): """Scheduler handling Thoth's graph syncs.""" if verbose: _LOGGER.setLevel(logging.DEBUG) _LOGGER.info( "Graph sync scheduler is running thoth-common in version %r, built from %r", thoth_common_version, os.getenv("OPENSHIFT_BUILD_COMMIT")) _LOGGER.info( "Graph sync scheduler is watching namespace %r and scheduling graph syncs in namespace %r", scheduler_namespace, graph_sync_namespace) openshift = OpenShift() queue = Queue() producer = Process(target=event_producer, args=(queue, scheduler_namespace)) producer.start() while producer.is_alive(): template_name, document_id = queue.get() try: graph_sync_id = openshift.schedule_graph_sync( document_id, graph_sync_namespace, template_name=template_name) _LOGGER.info("Scheduled new graph sync with id %r", graph_sync_id) except Exception as exc: _LOGGER.exception( "Failed to run sync for document id %r, the template to be triggered was %r: %s", document_id, template_name, exc) producer.join() # Always fail, this should be run forever. sys.exit(1)
async def parse_provenance_checker_trigger_message( provenance_checker_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None: """Parse provenance_checker_trigger message.""" await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_name = openshift.schedule_provenance_checker( application_stack=provenance_checker_trigger["application_stack"], origin=provenance_checker_trigger["origin"], whitelisted_sources=provenance_checker_trigger["whitelisted_sources"], debug=provenance_checker_trigger["debug"], job_id=provenance_checker_trigger["job_id"], ) _LOGGER.debug(f"Scheduled provenance checker workflow {workflow_name}") provenance_checker_trigger_success.inc()
async def parse_qebhwt_trigger_message(qebhwt_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None: """Parse qebhwt_trigger message.""" await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_name = openshift.schedule_qebhwt_workflow( github_event_type=qebhwt_trigger["github_event_type"], github_check_run_id=qebhwt_trigger["github_check_run_id"], github_installation_id=qebhwt_trigger["github_installation_id"], github_base_repo_url=qebhwt_trigger["github_base_repo_url"], github_head_repo_url=qebhwt_trigger["github_head_repo_url"], origin=qebhwt_trigger["origin"], revision=qebhwt_trigger["revision"], host=qebhwt_trigger["host"], job_id=qebhwt_trigger["job_id"], ) _LOGGER.debug(f"Scheduled qebhwt workflow {workflow_name}") qebhwt_trigger_success.inc()
def event_producer(queue: Queue, operator_namespace: str): """Queue events to be processed coming from the cluster.""" _LOGGER.info("Starting event producer") openshift = OpenShift() v1_configmap = openshift.ocp_client.resources.get(api_version="v1", kind="ConfigMap") for event in v1_configmap.watch(namespace=operator_namespace, label_selector="operator=workload"): if event["type"] != "ADDED": _LOGGER.debug("Skipping event, not addition event (type: %r)", event["type"]) continue configmap_name = event["object"].metadata.name _LOGGER.info("Queuing event %r for processing", configmap_name) queue.put(configmap_name)
async def _schedule_all_solvers( openshift: OpenShift, package_name: str, package_version: str, indexes: List[str] ) -> int: """Schedule all solvers.""" packages = f"{package_name}==={package_version}" try: await wait_for_limit(openshift, workflow_namespace=Configuration.THOTH_MIDDLETIER_NAMESPACE) analysis_ids = openshift.schedule_all_solvers(packages=packages, indexes=indexes) _LOGGER.info( "Scheduled solvers for packages %r from indexes %r, analysis ids are %r", packages, indexes, analysis_ids ) are_scheduled = len(analysis_ids) except Exception as e: _LOGGER.exception(f"Failed to schedule solvers for package {packages} from {indexes}: {e}") raise e return are_scheduled
async def parse_provenance_checker_trigger_message( provenance_checker_trigger: Dict[str, Any], openshift: OpenShift, **kwargs) -> None: """Parse provenance_checker_trigger message.""" await wait_for_limit( openshift, workflow_namespace=Configuration.THOTH_BACKEND_NAMESPACE) workflow_name = openshift.schedule_provenance_checker( origin=provenance_checker_trigger["origin"], authenticated=provenance_checker_trigger.get("authenticated", False), whitelisted_sources=provenance_checker_trigger["whitelisted_sources"], debug=provenance_checker_trigger["debug"], job_id=provenance_checker_trigger["job_id"], kebechet_metadata=provenance_checker_trigger.get("kebechet_metadata"), stack_info=provenance_checker_trigger.get("stack_info"), justification=provenance_checker_trigger.get("justification"), ) _LOGGER.debug(f"Scheduled provenance checker workflow {workflow_name}") provenance_checker_trigger_success.inc()
def _existing_producer(queue: Queue, build_watcher_namespace: str) -> None: """Query for existing images in image streams and queue them for analysis.""" openshift = OpenShift() v1_imagestreams = openshift.ocp_client.resources.get( api_version="image.openshift.io/v1", kind="ImageStream") for item in v1_imagestreams.get(namespace=build_watcher_namespace).items: _LOGGER.debug("Found imagestream item: %s", str(item)) _LOGGER.debug("Listing tags available for %r", item["metadata"]["name"]) for tag_info in item.status.tags or []: output_reference = f"{item.status.dockerImageRepository}:{tag_info.tag}" _LOGGER.info("Queueing already existing image %r for analysis", output_reference) queue.put(output_reference) _LOGGER.info( "Queuing existing images for analyses has finished, all of them were scheduled for analysis" )