def _create_srpm(self): # we want to get packit logs from the SRPM creation process # so we stuff them into a StringIO buffer stream = StringIO() handler = logging.StreamHandler(stream) packit_logger = logging.getLogger("packit") packit_logger.setLevel(logging.DEBUG) packit_logger.addHandler(handler) formatter = PackitFormatter(None, "%H:%M:%S") handler.setFormatter(formatter) srpm_success = True exception: Optional[Exception] = None extra_logs: str = "" try: self._srpm_path = Path( self.api.create_srpm(srpm_dir=self.api.up.local_project.working_dir) ) except SandcastleTimeoutReached as ex: exception = ex extra_logs = f"\nYou have reached 10-minute timeout while creating SRPM.\n" except ApiException as ex: exception = ex # this is an internal error: let's not expose anything to public extra_logs = ( "\nThere was a problem in the environment the packit-service is running in.\n" "Please hang tight, the help is coming." ) except Exception as ex: exception = ex # collect the logs now packit_logger.removeHandler(handler) stream.seek(0) srpm_logs = stream.read() if exception: logger.info(f"exception while running SRPM build: {exception}") logger.debug(f"{exception!r}") srpm_success = False # when do we NOT want to send stuff to sentry? sentry_integration.send_to_sentry(exception) # this needs to be done AFTER we gather logs # so that extra logs are after actual logs srpm_logs += extra_logs if hasattr(exception, "output"): output = getattr(exception, "output", "") # mypy srpm_logs += f"\nOutput of the command in the sandbox:\n{output}\n" srpm_logs += ( f"\nMessage: {exception}\nException: {exception!r}\n{self.msg_retrigger}" "\nPlease join the freenode IRC channel #packit for the latest info.\n" ) self._srpm_model = SRPMBuildModel.create(logs=srpm_logs, success=srpm_success)
def run(self) -> TaskResults: """ Sync the upstream release to dist-git as a pull request. """ self.local_project = LocalProject( git_project=self.project, working_dir=self.service_config.command_handler_work_dir, ) self.api = PackitAPI(self.service_config, self.job_config, self.local_project) errors = {} for branch in get_branches(*self.job_config.metadata.dist_git_branches, default="master"): try: self.api.sync_release(dist_git_branch=branch, version=self.data.tag_name) except Exception as ex: sentry_integration.send_to_sentry(ex) errors[branch] = str(ex) if errors: branch_errors = "" for branch, err in sorted( errors.items(), key=lambda branch_error: branch_error[0]): err_without_new_lines = err.replace("\n", " ") branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n" body_msg = ( f"Packit failed on creating pull-requests in dist-git:\n\n" f"| dist-git branch | error |\n" f"| --------------- | ----- |\n" f"{branch_errors}\n\n" "You can re-trigger the update by adding `/packit propose-update`" " to the issue comment.\n") self.project.create_issue( title= f"[packit] Propose update failed for release {self.data.tag_name}", body=body_msg, ) return TaskResults( success=False, details={ "msg": "Propose update failed.", "errors": errors }, ) return TaskResults(success=True, details={})
def handle_build_submit_error(self, ex): """ Handle errors when submitting Copr build. """ sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", ) self.monitor_not_submitted_copr_builds( len(self.build_targets), "submit_failure" )
def run(self) -> TaskResults: """ Sync the upstream release to dist-git as a pull request. """ self.local_project = LocalProject( git_project=self.project, working_dir=self.service_config.command_handler_work_dir, ) self.api = PackitAPI(self.service_config, self.job_config, self.local_project) errors = {} default_dg_branch = self.api.dg.local_project.git_project.default_branch for branch in get_branches(*self.job_config.metadata.dist_git_branches, default=default_dg_branch): try: self.api.sync_release(dist_git_branch=branch, tag=self.data.tag_name) except Exception as ex: # the archive has not been uploaded to PyPI yet if FILE_DOWNLOAD_FAILURE in str(ex): # retry for the archive to become available logger.info( f"We were not able to download the archive: {ex}") # when the task hits max_retries, it raises MaxRetriesExceededError # and the error handling code would be never executed retries = self.task.request.retries if retries < int( getenv("CELERY_RETRY_LIMIT", DEFAULT_RETRY_LIMIT)): logger.info(f"Retrying for the {retries + 1}. time...") # throw=False so that exception is not raised and task # is not retried also automatically self.task.retry(exc=ex, countdown=15 * 2**retries, throw=False) return TaskResults( success=False, details={ "msg": "Task was retried, we were not able to download the archive." }, ) sentry_integration.send_to_sentry(ex) errors[branch] = str(ex) if errors: branch_errors = "" for branch, err in sorted( errors.items(), key=lambda branch_error: branch_error[0]): err_without_new_lines = err.replace("\n", " ") branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n" msg_retrigger = MSG_RETRIGGER.format(job="update", command="propose-downstream", place="issue") body_msg = ( f"Packit failed on creating pull-requests in dist-git:\n\n" f"| dist-git branch | error |\n" f"| --------------- | ----- |\n" f"{branch_errors}\n\n" f"{msg_retrigger}\n") self.project.create_issue( title= f"[packit] Propose downstream failed for release {self.data.tag_name}", body=body_msg, ) return TaskResults( success=False, details={ "msg": "Propose downstream failed.", "errors": errors }, ) return TaskResults(success=True, details={})
def run(self) -> TaskResults: """ Sync the upstream release to dist-git as a pull request. """ self.local_project = LocalProject( git_project=self.project, working_dir=self.service_config.command_handler_work_dir, ) self.api = PackitAPI(self.service_config, self.job_config, self.local_project) errors = {} for branch in get_branches(*self.job_config.metadata.dist_git_branches, default="master"): try: self.api.sync_release(dist_git_branch=branch, version=self.data.tag_name) except Exception as ex: # the archive has not been uploaded to PyPI yet if FILE_DOWNLOAD_FAILURE in str(ex): # retry for the archive to become available logger.info( f"We were not able to download the archive: {ex}") # when the task hits max_retries, it raises MaxRetriesExceededError # and the error handling code would be never executed retries = self.task.request.retries if retries < RETRY_LIMIT: logger.info(f"Retrying for the {retries + 1}. time...") self.task.retry(exc=ex, countdown=15 * 2**retries) sentry_integration.send_to_sentry(ex) errors[branch] = str(ex) if errors: branch_errors = "" for branch, err in sorted( errors.items(), key=lambda branch_error: branch_error[0]): err_without_new_lines = err.replace("\n", " ") branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n" body_msg = ( f"Packit failed on creating pull-requests in dist-git:\n\n" f"| dist-git branch | error |\n" f"| --------------- | ----- |\n" f"{branch_errors}\n\n" "You can re-trigger the update by adding `/packit propose-update`" " to the issue comment.\n") self.project.create_issue( title= f"[packit] Propose update failed for release {self.data.tag_name}", body=body_msg, ) return TaskResults( success=False, details={ "msg": "Propose update failed.", "errors": errors }, ) return TaskResults(success=True, details={})
def run_testing_farm(self, build_id: int, chroot: str) -> TaskResults: if chroot not in self.tests_targets: # Leaving here just to be sure that we will discover this situation if it occurs. # Currently not possible to trigger this situation. msg = f"Target '{chroot}' not defined for tests but triggered." logger.error(msg) send_to_sentry(PackitConfigException(msg)) return TaskResults( success=False, details={"msg": msg}, ) if chroot not in self.build_targets: self.report_missing_build_chroot(chroot) return TaskResults( success=False, details={ "msg": f"Target '{chroot}' not defined for build. " "Cannot run tests without build." }, ) self.report_status_to_test_for_chroot( state=CommitStatus.pending, description="Build succeeded. Submitting the tests ...", chroot=chroot, ) logger.info("Sending testing farm request...") if self.is_fmf_configured(): payload = self._payload(build_id, chroot) else: payload = self._payload_install_test(build_id, chroot) endpoint = "requests" logger.debug(f"POSTing {payload} to {self.tft_api_url}{endpoint}") req = self.send_testing_farm_request( endpoint=endpoint, method="POST", data=payload, ) logger.debug(f"Request sent: {req}") if not req: msg = "Failed to post request to testing farm API." logger.debug("Failed to post request to testing farm API.") self.report_status_to_test_for_chroot( state=CommitStatus.error, description=msg, chroot=chroot, ) return TaskResults(success=False, details={"msg": msg}) # success set check on pending if req.status_code != 200: # something went wrong if req.json() and "message" in req.json(): msg = req.json()["message"] else: msg = f"Failed to submit tests: {req.reason}" logger.error(msg) self.report_status_to_test_for_chroot( state=CommitStatus.failure, description=msg, chroot=chroot, ) return TaskResults(success=False, details={"msg": msg}) # Response: {"id": "9fa3cbd1-83f2-4326-a118-aad59f5", ...} pipeline_id = req.json()["id"] logger.debug( f"Submitted ({req.status_code}) to testing farm as request {pipeline_id}" ) TFTTestRunModel.create( pipeline_id=pipeline_id, commit_sha=self.metadata.commit_sha, status=TestingFarmResult.new, target=chroot, web_url=None, trigger_model=self.db_trigger, # In _payload() we ask TF to test commit_sha of fork (PR's source). # Store original url. If this proves to work, make it a separate column. data={"base_project_url": self.project.get_web_url()}, ) self.report_status_to_test_for_chroot( state=CommitStatus.pending, description="Tests have been submitted ...", url=f"{self.tft_api_url}requests/{pipeline_id}", chroot=chroot, ) return TaskResults(success=True, details={})
def run_testing_farm(self, chroot: str) -> TaskResults: if chroot not in self.tests_targets: # Leaving here just to be sure that we will discover this situation if it occurs. # Currently not possible to trigger this situation. msg = f"Target '{chroot}' not defined for tests but triggered." logger.error(msg) send_to_sentry(PackitConfigException(msg)) return TaskResults( success=False, details={"msg": msg}, ) if chroot not in self.build_targets: self.report_missing_build_chroot(chroot) return TaskResults( success=False, details={ "msg": f"Target '{chroot}' not defined for build. " "Cannot run tests without build." }, ) self.report_status_to_test_for_chroot( state=CommitStatus.pending, description="Build succeeded. Submitting the tests ...", chroot=chroot, ) pipeline_id = str(uuid.uuid4()) logger.debug(f"Pipeline id: {pipeline_id}") test_run_model = TFTTestRunModel.create( pipeline_id=pipeline_id, commit_sha=self.metadata.commit_sha, status=TestingFarmResult.new, target=chroot, web_url=None, trigger_model=self.db_trigger, ) logger.debug("Sending testing farm request...") payload = self._trigger_payload(pipeline_id, chroot) logger.debug(f"Payload: {payload}") req = self.send_testing_farm_request(TESTING_FARM_TRIGGER_URL, "POST", {}, json.dumps(payload)) logger.debug(f"Request sent: {req}") if not req: msg = "Failed to post request to testing farm API." logger.debug("Failed to post request to testing farm API.") self.report_status_to_test_for_chroot( state=CommitStatus.error, description=msg, chroot=chroot, ) test_run_model.set_status(TestingFarmResult.error) return TaskResults(success=False, details={"msg": msg}) else: logger.debug( f"Submitted to testing farm with return code: {req.status_code}" ) """ Response: { "id": "9fa3cbd1-83f2-4326-a118-aad59f5", "success": true, "url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<id>" } """ # success set check on pending if req.status_code != 200: # something went wrong if req.json() and "message" in req.json(): msg = req.json()["message"] else: msg = f"Failed to submit tests: {req.reason}" logger.error(msg) self.report_status_to_test_for_chroot( state=CommitStatus.failure, description=msg, chroot=chroot, ) test_run_model.set_status(TestingFarmResult.error) return TaskResults(success=False, details={"msg": msg}) test_run_model.set_status(TestingFarmResult.running) self.report_status_to_test_for_chroot( state=CommitStatus.pending, description="Tests are running ...", url=req.json()["url"], chroot=chroot, ) return TaskResults(success=True, details={})
def _create_srpm(self): """ Create SRPM. Returns: Task results if job is done because of merge conflicts, `None` otherwise. """ # we want to get packit logs from the SRPM creation process # so we stuff them into a StringIO buffer stream = StringIO() handler = logging.StreamHandler(stream) packit_logger = logging.getLogger("packit") packit_logger.setLevel(logging.DEBUG) packit_logger.addHandler(handler) formatter = PackitFormatter() handler.setFormatter(formatter) srpm_success = True exception: Optional[Exception] = None extra_logs: str = "" results: Optional[TaskResults] = None try: self._srpm_path = Path( self.api.create_srpm( srpm_dir=self.api.up.local_project.working_dir)) except SandcastleTimeoutReached as ex: exception = ex extra_logs = "\nYou have reached 10-minute timeout while creating SRPM.\n" except ApiException as ex: exception = ex # this is an internal error: let's not expose anything to public extra_logs = ( "\nThere was a problem in the environment the packit-service is running in.\n" "Please hang tight, the help is coming.") except PackitMergeException as ex: exception = ex results = TaskResults( success=True, details={ "msg": "Merge conflicts were detected, cannot build SRPM.", "exception": str(ex), }, ) except Exception as ex: exception = ex # collect the logs now packit_logger.removeHandler(handler) stream.seek(0) srpm_logs = stream.read() if exception: logger.info(f"exception while running SRPM build: {exception}") logger.debug(f"{exception!r}") srpm_success = False # when do we NOT want to send stuff to sentry? if not isinstance(exception, PackitMergeException): sentry_integration.send_to_sentry(exception) # this needs to be done AFTER we gather logs # so that extra logs are after actual logs srpm_logs += extra_logs if hasattr(exception, "output"): output = getattr(exception, "output", "") # mypy srpm_logs += f"\nOutput of the command in the sandbox:\n{output}\n" srpm_logs += ( f"\nMessage: {exception}\nException: {exception!r}\n{self.msg_retrigger}" "\nPlease join #packit on irc.libera.chat if you need help with the error above.\n" ) self._srpm_model, self.run_model = SRPMBuildModel.create_with_new_run( logs=srpm_logs, success=srpm_success, trigger_model=self.db_trigger, ) return results
def _run_copr_build_and_save_output(self) -> BuildMetadata: # we want to get packit logs from the SRPM creation process # so we stuff them into a StringIO buffer stream = StringIO() handler = logging.StreamHandler(stream) packit_logger = logging.getLogger("packit") packit_logger.setLevel(logging.DEBUG) packit_logger.addHandler(handler) formatter = PackitFormatter(None, "%H:%M:%S") handler.setFormatter(formatter) c = BuildMetadata() c.srpm_failed = False ex: Optional[Exception] = None # shut up pycharm extra_logs: str = "" try: c.copr_build_id, c.copr_web_url = self.api.run_copr_build( project=self.job_project, chroots=self.build_chroots, owner=self.job_owner, ) except SandcastleTimeoutReached as e: ex = e extra_logs = f"\nYou have reached 10-minute timeout while creating SRPM.\n" except ApiException as e: ex = e # this is an internal error: let's not expose anything to public extra_logs = ( "\nThere was a problem in the environment the packit-service is running in.\n" "Please hang tight, the help is coming.") except Exception as e: ex = e # shut up mypy # collect the logs now packit_logger.removeHandler(handler) stream.seek(0) c.srpm_logs = stream.read() if ex: logger.info(f"exception while running a copr build: {ex}") logger.debug(f"{ex!r}") c.srpm_failed = True # when do we NOT want to send stuff to sentry? sentry_integration.send_to_sentry(ex) # this needs to be done AFTER we gather logs # so that extra logs are after actual logs c.srpm_logs += extra_logs if hasattr(ex, "output"): output = getattr(ex, "output", "") # mypy c.srpm_logs += f"\nOutput of the command in the sandbox:\n{output}\n" c.srpm_logs += ( f"\nMessage: {ex}\nException: {ex!r}\n{self.msg_retrigger}" "\nPlease join the freenode IRC channel #packit for the latest info.\n" ) return c
def run_koji_build(self) -> TaskResults: if not self.is_scratch: msg = "Non-scratch builds not possible from upstream." self.report_status_to_all( description=msg, state=CommitStatus.error, url=KOJI_PRODUCTION_BUILDS_ISSUE, ) return TaskResults(success=True, details={"msg": msg}) self.report_status_to_all( description="Building SRPM ...", state=CommitStatus.pending ) self.create_srpm_if_needed() if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) try: # We need to do it manually # because we don't use PackitAPI.build, but PackitAPI.up.koji_build self.api.init_kerberos_ticket() except PackitCommandFailedError as ex: msg = f"Kerberos authentication error: {ex.stderr_output}" logger.error(msg) self.report_status_to_all( state=CommitStatus.error, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) errors: Dict[str, str] = {} for target in self.build_targets: if target not in self.supported_koji_targets: msg = f"Target not supported: {target}" self.report_status_to_all_for_chroot( state=CommitStatus.error, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), chroot=target, ) errors[target] = msg continue try: build_id, web_url = self.run_build(target=target) except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all_for_chroot( state=CommitStatus.error, description=f"Submit of the build failed: {ex}", url=get_srpm_log_url_from_flask(self.srpm_model.id), chroot=target, ) errors[target] = str(ex) continue koji_build = KojiBuildModel.get_or_create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, web_url=web_url, target=target, status="pending", srpm_build=self.srpm_model, trigger_model=self.db_trigger, ) url = get_koji_build_info_url_from_flask(id_=koji_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Building RPM ...", url=url, chroot=target, ) if errors: return TaskResults( success=False, details={ "msg": "Koji build submit was not successful for all chroots.", "errors": errors, }, ) # TODO: release the hounds! """ celery_app.send_task( "task.babysit_koji_build", args=(build_metadata.build_id,), countdown=120, # do the first check in 120s ) """ return TaskResults(success=True, details={})
def run_copr_build(self) -> HandlerResults: if not (self.job_build or self.job_tests): msg = "No copr_build or tests job defined." # we can't report it to end-user at this stage return HandlerResults(success=False, details={"msg": msg}) self.report_status_to_all( description="Building SRPM ...", state=CommitStatus.pending, # pagure requires "valid url" url="", ) self.create_srpm_if_needed() if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return HandlerResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=CommitStatus.error, description=f"Submit of the build failed: {ex}", ) return HandlerResults(success=False, details={"error": str(ex)}) for chroot in self.build_targets: copr_build = CoprBuildModel.get_or_create( build_id=str(build_id), commit_sha=self.event.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", srpm_build=self.srpm_model, trigger_model=self.event.db_trigger, ) url = get_copr_build_log_url_from_flask(id_=copr_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Starting RPM build...", url=url, chroot=chroot, ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s ) return HandlerResults(success=True, details={})
class KojiBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.production_build job_type_test = None status_name_build: str = "production-build" status_name_test: str = None def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger, job_config: JobConfig, targets_override: Optional[Set[str]] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="production-build", place="pull request" ) # Lazy properties self._supported_koji_targets = None @property def is_scratch(self) -> bool: return self.job_build and self.job_build.metadata.scratch @property def build_targets_all(self) -> Set[str]: """ Return all valid Koji targets/chroots from config. """ return get_koji_targets(*self.configured_build_targets) @property def tests_targets_all(self) -> Set[str]: """ [not used now] Return all valid test targets/chroots from config. """ return get_koji_targets(*self.configured_tests_targets) @property def supported_koji_targets(self): if self._supported_koji_targets is None: self._supported_koji_targets = get_all_koji_targets() return self._supported_koji_targets def run_koji_build(self) -> TaskResults: self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running ) if results := self.create_srpm_if_needed(): return results if self.srpm_model.status != PG_BUILD_STATUS_SUCCESS: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) try: # We need to do it manually # because we don't use PackitAPI.build, but PackitAPI.up.koji_build self.api.init_kerberos_ticket() except PackitCommandFailedError as ex: msg = f"Kerberos authentication error: {ex.stderr_output}" logger.error(msg) self.report_status_to_all( state=BaseCommitStatus.error, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) errors: Dict[str, str] = {} for target in self.build_targets: if target not in self.supported_koji_targets: msg = f"Target not supported: {target}" self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), chroot=target, ) errors[target] = msg continue try: build_id, web_url = self.run_build(target=target) except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=target, ) errors[target] = str(ex) continue koji_build = KojiBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, web_url=web_url, target=target, status="pending", run_model=self.run_model, ) url = get_koji_build_info_url(id_=koji_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Building RPM ...", url=url, chroot=target, ) if errors: return TaskResults( success=False, details={ "msg": "Koji build submit was not successful for all chroots.", "errors": errors, }, ) # TODO: release the hounds! """ celery_app.send_task( "task.babysit_koji_build", args=(build_metadata.build_id,), countdown=120, # do the first check in 120s ) """ return TaskResults(success=True, details={})
def run_testing_farm(self, chroot: str) -> HandlerResults: if chroot not in self.tests_chroots: # Leaving here just to be sure that we will discover this situation if it occurs. # Currently not possible to trigger this situation. msg = f"Target '{chroot}' not defined for tests but triggered." logger.error(msg) send_to_sentry(PackitConfigException(msg)) return HandlerResults( success=False, details={"msg": msg}, ) if chroot not in self.build_chroots: self.report_missing_build_chroot(chroot) return HandlerResults( success=False, details={ "msg": f"Target '{chroot}' not defined for build. " f"Cannot run tests without build." }, ) self.report_status_to_test_for_chroot( state=CommitStatus.pending, description="Build succeeded. Submitting the tests ...", chroot=chroot, ) pipeline_id = str(uuid.uuid4()) logger.debug(f"Pipeline id: {pipeline_id}") payload: dict = { "pipeline": { "id": pipeline_id }, "api": { "token": self.config.testing_farm_secret }, "response-url": f"{self.api_url}/testing-farm/results", "artifact": { "repo-name": self.project.repo, "repo-namespace": self.project.namespace, "copr-repo-name": f"{self.job_owner}/{self.default_project_name}", "copr-chroot": chroot, "commit-sha": self.event.commit_sha, "git-url": self.event.project_url, "git-ref": self.event.git_ref if self.event.git_ref else self.event.commit_sha, }, } logger.debug("Sending testing farm request...") logger.debug(payload) req = self.send_testing_farm_request(TESTING_FARM_TRIGGER_URL, "POST", {}, json.dumps(payload)) logger.debug(f"Request sent: {req}") if not req: msg = "Failed to post request to testing farm API." logger.debug("Failed to post request to testing farm API.") self.report_status_to_test_for_chroot( state=CommitStatus.failure, description=msg, chroot=chroot, ) return HandlerResults(success=False, details={"msg": msg}) else: logger.debug( f"Submitted to testing farm with return code: {req.status_code}" ) """ Response: { "id": "9fa3cbd1-83f2-4326-a118-aad59f5", "success": true, "url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<id>" } """ # success set check on pending if req.status_code != 200: # something went wrong if req.json() and "message" in req.json(): msg = req.json()["message"] else: msg = f"Failed to submit tests: {req.reason}" logger.error(msg) self.report_status_to_test_for_chroot( state=CommitStatus.failure, description=msg, chroot=chroot, ) return HandlerResults(success=False, details={"msg": msg}) self.report_status_to_test_for_chroot( state=CommitStatus.pending, description="Tests are running ...", url=req.json()["url"], chroot=chroot, ) return HandlerResults(success=True, details={})
class CoprBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.copr_build job_type_test = JobType.tests status_name_build: str = "rpm-build" status_name_test: str = "testing-farm" def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger: AbstractTriggerDbType, job_config: JobConfig, targets_override: Optional[Set[str]] = None, pushgateway: Optional[Pushgateway] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, pushgateway=pushgateway, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="copr-build" if self.job_build else "build", place="pull request", ) @property def default_project_name(self) -> str: """ Project name for copr. * use hostname prefix for non-github service * replace slash in namespace with dash * add `-stg` suffix for the stg app """ service_hostname = parse_git_repo( self.project.service.instance_url).hostname service_prefix = ("" if isinstance(self.project, GithubProject) else f"{service_hostname}-") namespace = self.project.namespace.replace("/", "-") stg = "-stg" if self.service_config.deployment == Deployment.stg else "" # We want to share project between all releases. # More details: https://github.com/packit/packit-service/issues/1044 identifier = "releases" if self.metadata.tag_name else self.metadata.identifier return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}" @property def job_project(self) -> Optional[str]: """ The job definition from the config file. """ if self.job_build and self.job_build.metadata.project: return self.job_build.metadata.project if self.job_tests and self.job_tests.metadata.project: return self.job_tests.metadata.project return self.default_project_name @property def job_owner(self) -> Optional[str]: """ Owner used for the copr build -- search the config or use the copr's config. """ if self.job_build and self.job_build.metadata.owner: return self.job_build.metadata.owner if self.job_tests and self.job_tests.metadata.owner: return self.job_tests.metadata.owner return self.api.copr_helper.copr_client.config.get("username") @property def preserve_project(self) -> Optional[bool]: """ If the project will be preserved or can be removed after 60 days. """ return self.job_build.metadata.preserve_project if self.job_build else None @property def list_on_homepage(self) -> Optional[bool]: """ If the project will be shown on the copr home page. """ return self.job_build.metadata.list_on_homepage if self.job_build else None @property def additional_repos(self) -> Optional[List[str]]: """ Additional repos that will be enable for copr build. """ return self.job_build.metadata.additional_repos if self.job_build else None @property def build_targets_all(self) -> Set[str]: """ Return all valid Copr build targets/chroots from config. """ return get_valid_build_targets(*self.configured_build_targets, default=None) @property def tests_targets_all(self) -> Set[str]: """ Return all valid test targets/chroots from config. """ return get_valid_build_targets(*self.configured_tests_targets, default=None) @property def available_chroots(self) -> Set[str]: """ Returns set of available COPR targets. """ return { *filter( lambda chroot: not chroot.startswith("_"), self.api.copr_helper.get_copr_client().mock_chroot_proxy. get_list().keys(), ) } def get_built_packages(self, build_id: int, chroot: str) -> List: return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages( build_id, chroot).packages def get_build(self, build_id: int): return self.api.copr_helper.copr_client.build_proxy.get(build_id) def monitor_not_submitted_copr_builds(self, number_of_builds: int, reason: str): """ Measure the time it took to set the failed status in case of event (e.g. failed SRPM) that prevents Copr build to be submitted. """ time = measure_time(end=datetime.now(timezone.utc), begin=self.metadata.task_accepted_time) for _ in range(number_of_builds): self.pushgateway.copr_build_not_submitted_time.labels( reason=reason).observe(time) def run_copr_build(self) -> TaskResults: self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running, # pagure requires "valid url" url="", ) if results := self.create_srpm_if_needed(): return results if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "srpm_failure") return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "submit_failure") return TaskResults( success=False, details={ "msg": "Submit of the Copr build failed.", "error": str(ex) }, ) unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=chroot, ) self.monitor_not_submitted_copr_builds(1, "not_supported_target") unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", run_model=self.run_model, task_accepted_time=self.metadata.task_accepted_time, ) url = get_copr_build_info_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.status_reporter.comment( body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id, ), countdown=120, # do the first check in 120s ) return TaskResults(success=True, details={})
def run_testing_farm(self, chroot: str, build: Optional["CoprBuildModel"]) -> TaskResults: if chroot not in self.tests_targets: # Leaving here just to be sure that we will discover this situation if it occurs. # Currently not possible to trigger this situation. msg = f"Target '{chroot}' not defined for tests but triggered." logger.error(msg) send_to_sentry(PackitConfigException(msg)) return TaskResults( success=False, details={"msg": msg}, ) if not self.skip_build and chroot not in self.build_targets: self.report_missing_build_chroot(chroot) return TaskResults( success=False, details={ "msg": f"Target '{chroot}' not defined for build. " "Cannot run tests without build." }, ) if (self.job_config.metadata.use_internal_tf and f"{self.project.service.hostname}/{self.project.full_repo_name}" not in self.service_config.enabled_projects_for_internal_tf): self.report_status_to_test_for_chroot( state=BaseCommitStatus.neutral, description= "Internal TF not allowed for this project. Let us know.", chroot=chroot, url="https://packit.dev/#contact", ) return TaskResults( success=True, details={"msg": "Project not allowed to use internal TF."}, ) self.report_status_to_test_for_chroot( state=BaseCommitStatus.running, description=f"{'Build succeeded. ' if not self.skip_build else ''}" f"Submitting the tests ...", chroot=chroot, ) logger.info("Sending testing farm request...") if self.is_fmf_configured(): artifact = (self._artifact(chroot, int(build.build_id), build.built_packages) if not self.skip_build else None) payload = self._payload(chroot, artifact, build) elif not self.is_fmf_configured() and not self.skip_build: payload = self._payload_install_test(int(build.build_id), chroot) else: return TaskResults( success=True, details={"msg": "No actions for TestingFarmHandler."}) endpoint = "requests" logger.debug(f"POSTing {payload} to {self.tft_api_url}{endpoint}") req = self.send_testing_farm_request( endpoint=endpoint, method="POST", data=payload, ) logger.debug(f"Request sent: {req}") if not req: msg = "Failed to post request to testing farm API." logger.debug("Failed to post request to testing farm API.") self.report_status_to_test_for_chroot( state=BaseCommitStatus.error, description=msg, chroot=chroot, ) return TaskResults(success=False, details={"msg": msg}) # success set check on pending if req.status_code != 200: # something went wrong if req.json() and "errors" in req.json(): msg = req.json()["errors"] # specific case, unsupported arch if nested_get(req.json(), "errors", "environments", "0", "arch"): msg = req.json()["errors"]["environments"]["0"]["arch"] else: msg = f"Failed to submit tests: {req.reason}" logger.error(msg) self.report_status_to_test_for_chroot( state=BaseCommitStatus.failure, description=msg, chroot=chroot, ) return TaskResults(success=False, details={"msg": msg}) # Response: {"id": "9fa3cbd1-83f2-4326-a118-aad59f5", ...} pipeline_id = req.json()["id"] logger.debug( f"Submitted ({req.status_code}) to testing farm as request {pipeline_id}" ) run_model = (RunModel.create( type=self.db_trigger.job_trigger_model_type, trigger_id=self.db_trigger.id, ) if self.skip_build else build.runs[-1]) created_model = TFTTestRunModel.create( pipeline_id=pipeline_id, commit_sha=self.metadata.commit_sha, status=TestingFarmResult.new, target=chroot, web_url=None, run_model=run_model, # In _payload() we ask TF to test commit_sha of fork (PR's source). # Store original url. If this proves to work, make it a separate column. data={"base_project_url": self.project.get_web_url()}, ) self.report_status_to_test_for_chroot( state=BaseCommitStatus.running, description="Tests have been submitted ...", url=get_testing_farm_info_url(created_model.id), chroot=chroot, ) return TaskResults(success=True, details={})
def run_copr_build(self) -> TaskResults: if not (self.job_build or self.job_tests): msg = "No copr_build or tests job defined." # we can't report it to end-user at this stage return TaskResults(success=False, details={"msg": msg}) self.report_status_to_all( description="Building SRPM ...", state=CommitStatus.pending, # pagure requires "valid url" url="", ) self.create_srpm_if_needed() if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() Pushgateway().push_copr_build_created() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=CommitStatus.error, description=f"Submit of the build failed: {ex}", ) return TaskResults( success=False, details={"msg": "Submit of the Copr build failed.", "error": str(ex)}, ) unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=CommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_log_url_from_flask(self.srpm_model.id), chroot=chroot, ) unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.get_or_create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", srpm_build=self.srpm_model, trigger_model=self.db_trigger, ) url = get_copr_build_info_url_from_flask(id_=copr_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.project.pr_comment( pr_id=self.metadata.pr_id, body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s ) return TaskResults(success=True, details={})