def post(self): """ A webhook used by Packit-as-a-Service Gitlab hook. """ msg = request.json if not msg: logger.debug( "/webhooks/gitlab: we haven't received any JSON data.") return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST if all([msg.get("zen"), msg.get("hook_id"), msg.get("hook")]): logger.debug( f"/webhooks/gitlab received ping event: {msg['hook']}") return "Pong!", HTTPStatus.OK try: self.validate_token() except ValidationFailed as exc: logger.info(f"/webhooks/gitlab {exc}") return str(exc), HTTPStatus.UNAUTHORIZED if not self.interested(): return "Thanks but we don't care about this event", HTTPStatus.ACCEPTED celery_app.send_task( name=getenv("CELERY_MAIN_TASK_NAME") or CELERY_DEFAULT_MAIN_TASK_NAME, kwargs={"event": msg}, ) return "Webhook accepted. We thank you, Gitlab.", HTTPStatus.ACCEPTED
def post(self): """ A webhook used by Packit-as-a-Service GitHub App. """ msg = request.json if not msg: logger.debug("/webhooks/github: we haven't received any JSON data.") return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST if all([msg.get("zen"), msg.get("hook_id"), msg.get("hook")]): logger.debug(f"/webhooks/github received ping event: {msg['hook']}") return "Pong!", HTTPStatus.OK try: self.validate_signature() except ValidationFailed as exc: logger.info(f"/webhooks/github {exc}") return str(exc), HTTPStatus.UNAUTHORIZED if not self.interested(): return "Thanks but we don't care about this event", HTTPStatus.ACCEPTED # TODO: define task names at one place celery_app.send_task( name="task.steve_jobs.process_message", kwargs={"event": msg} ) return "Webhook accepted. We thank you, Github.", HTTPStatus.ACCEPTED
def post(self): """ Submit Testing Farm results """ msg = request.json if not msg: logger.debug( "/testing-farm/results: we haven't received any JSON data.") return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST try: self.validate_testing_farm_request() except ValidationFailed as exc: logger.info(f"/testing-farm/results {exc}") return str(exc), HTTPStatus.UNAUTHORIZED # There's only one key in the msg, # so make sure we don't confuse this with something else msg["source"] = "testing-farm" celery_app.send_task( name=getenv("CELERY_MAIN_TASK_NAME") or CELERY_DEFAULT_MAIN_TASK_NAME, kwargs={"event": msg}, ) return "Test results accepted", HTTPStatus.OK
def listen_to_fedmsg(message_id): """ Listen to events on fedmsg and process them. if MESSAGE-ID is specified, process only the selected messages """ consumerino = Consumerino() if message_id: for msg_id in message_id: fedmsg_dict = consumerino.fetch_fedmsg_dict(msg_id) logger.debug(f"Processing {fedmsg_dict}") celery_app.send_task(name="task.steve_jobs.process_message", kwargs={"event": fedmsg_dict}) else: for topic, msg in consumerino.yield_all_messages(): if do_we_process_fedmsg_topic(topic): logger.debug(f"Processing topic {topic}, msg {msg}") celery_app.send_task( name="task.steve_jobs.process_message", kwargs={ "event": msg, "topic": topic }, )
def handle_rpm_build_start( self, build_id: int, web_url: str, waiting_for_srpm: bool = False ): """ Create models for Copr build chroots and report start of RPM build if the SRPM is already built. """ unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=chroot, ) self.monitor_not_submitted_copr_builds(1, "not_supported_target") unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="waiting_for_srpm" if waiting_for_srpm else "pending", run_model=self.run_model, task_accepted_time=self.metadata.task_accepted_time, ) if not waiting_for_srpm: url = get_copr_build_info_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.status_reporter.comment( body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s )
def run_copr_build(self) -> HandlerResults: if not (self.job_build or self.job_tests): msg = "No copr_build or tests job defined." # we can't report it to end-user at this stage return HandlerResults(success=False, details={"msg": msg}) self.report_status_to_all(description="Building SRPM ...", state=CommitStatus.pending) build_metadata = self._run_copr_build_and_save_output() srpm_build_model = SRPMBuild.create(build_metadata.srpm_logs) if build_metadata.srpm_failed: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url(srpm_build_model.id), ) return HandlerResults(success=False, details={"msg": msg}) for chroot in self.build_chroots: copr_build = CoprBuild.get_or_create( pr_id=self.pr_id, build_id=str(build_metadata.copr_build_id), commit_sha=self.event.commit_sha, repo_name=self.project.repo, namespace=self.project.namespace, project_name=self.job_project, owner=self.job_owner, web_url=build_metadata.copr_web_url, target=chroot, status="pending", srpm_build=srpm_build_model, ) url = get_log_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Building RPM ...", url=url, chroot=chroot, ) self.copr_build_model.build_id = build_metadata.copr_build_id self.copr_build_model.save() # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_metadata.copr_build_id, ), countdown=120, # do the first check in 120s ) return HandlerResults(success=True, details={})
def testing_farm_results(): msg = request.get_json() if not msg: logger.debug( "/testing-farm/results: we haven't received any JSON data.") return "We haven't received any JSON data." if not validate_testing_farm_request(): abort(401) # Unauthorized celery_app.send_task(name="task.steve_jobs.process_message", kwargs={"event": msg}) return json.dumps({"status": 200, "message": "Test results accepted"})
def github_webhook(): msg = request.get_json() if not msg: logger.debug("/webhooks/github: we haven't received any JSON data.") return "We haven't received any JSON data." if all([msg.get("zen"), msg.get("hook_id"), msg.get("hook")]): logger.debug(f"/webhooks/github received ping event: {msg['hook']}") return "Pong!" if not _validate_signature(): abort(401) # Unauthorized # TODO: define task names at one place celery_app.send_task(name="task.steve_jobs.process_message", kwargs={"event": msg}) return "Webhook accepted. We thank you, Github."
def post(self): """ Submit Testing Farm results """ msg = request.json if not msg: logger.debug( "/testing-farm/results: we haven't received any JSON data.") return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST try: self.validate_testing_farm_request() except ValidationFailed as exc: logger.info(f"/testing-farm/results {exc}") return str(exc), HTTPStatus.UNAUTHORIZED celery_app.send_task(name="task.steve_jobs.process_message", kwargs={"event": msg}) return "Test results accepted", HTTPStatus.ACCEPTED
def testing_farm_results(): """ Expected format: { "artifact": { "commit-sha": "08bfc38f15082bdf9ba964c3bbd04878666d1d56", "copr-chroot": "fedora-29-x86_64", "copr-repo-name": "packit/packit-service-hello-world-14", "git-ref": "08bfc38f15082bdf9ba964c3bbd04878666d1d56", "git-url": "https://github.com/packit-service/hello-world", "repo-name": "hello-world", "repo-namespace": "packit-service" }, "message": "Command 'git' not found", "pipeline": { "id": "614d240a-1e27-4758-ad6a-ed3d34281924" }, "result": "error", "token": "HERE-IS-A-VALID-TOKEN", "url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<ID>" } :return: {"status": 200, "message": "Test results accepted"} """ msg = request.get_json() if not msg: logger.debug( "/testing-farm/results: we haven't received any JSON data.") return "We haven't received any JSON data." if not validate_testing_farm_request(): abort(401) # Unauthorized celery_app.send_task(name="task.steve_jobs.process_message", kwargs={"event": msg}) return json.dumps({"status": 200, "message": "Test results accepted"})
def run_copr_build(self) -> HandlerResults: if not (self.job_build or self.job_tests): msg = "No copr_build or tests job defined." # we can't report it to end-user at this stage return HandlerResults(success=False, details={"msg": msg}) self.report_status_to_all( description="Building SRPM ...", state=CommitStatus.pending, # pagure requires "valid url" url="", ) self.create_srpm_if_needed() if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return HandlerResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=CommitStatus.error, description=f"Submit of the build failed: {ex}", ) return HandlerResults(success=False, details={"error": str(ex)}) for chroot in self.build_targets: copr_build = CoprBuildModel.get_or_create( build_id=str(build_id), commit_sha=self.event.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", srpm_build=self.srpm_model, trigger_model=self.event.db_trigger, ) url = get_copr_build_log_url_from_flask(id_=copr_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Starting RPM build...", url=url, chroot=chroot, ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s ) return HandlerResults(success=True, details={})
class CoprBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.copr_build job_type_test = JobType.tests status_name_build: str = "rpm-build" status_name_test: str = "testing-farm" def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger: AbstractTriggerDbType, job_config: JobConfig, targets_override: Optional[Set[str]] = None, pushgateway: Optional[Pushgateway] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, pushgateway=pushgateway, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="copr-build" if self.job_build else "build", place="pull request", ) @property def default_project_name(self) -> str: """ Project name for copr. * use hostname prefix for non-github service * replace slash in namespace with dash * add `-stg` suffix for the stg app """ service_hostname = parse_git_repo( self.project.service.instance_url).hostname service_prefix = ("" if isinstance(self.project, GithubProject) else f"{service_hostname}-") namespace = self.project.namespace.replace("/", "-") stg = "-stg" if self.service_config.deployment == Deployment.stg else "" # We want to share project between all releases. # More details: https://github.com/packit/packit-service/issues/1044 identifier = "releases" if self.metadata.tag_name else self.metadata.identifier return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}" @property def job_project(self) -> Optional[str]: """ The job definition from the config file. """ if self.job_build and self.job_build.metadata.project: return self.job_build.metadata.project if self.job_tests and self.job_tests.metadata.project: return self.job_tests.metadata.project return self.default_project_name @property def job_owner(self) -> Optional[str]: """ Owner used for the copr build -- search the config or use the copr's config. """ if self.job_build and self.job_build.metadata.owner: return self.job_build.metadata.owner if self.job_tests and self.job_tests.metadata.owner: return self.job_tests.metadata.owner return self.api.copr_helper.copr_client.config.get("username") @property def preserve_project(self) -> Optional[bool]: """ If the project will be preserved or can be removed after 60 days. """ return self.job_build.metadata.preserve_project if self.job_build else None @property def list_on_homepage(self) -> Optional[bool]: """ If the project will be shown on the copr home page. """ return self.job_build.metadata.list_on_homepage if self.job_build else None @property def additional_repos(self) -> Optional[List[str]]: """ Additional repos that will be enable for copr build. """ return self.job_build.metadata.additional_repos if self.job_build else None @property def build_targets_all(self) -> Set[str]: """ Return all valid Copr build targets/chroots from config. """ return get_valid_build_targets(*self.configured_build_targets, default=None) @property def tests_targets_all(self) -> Set[str]: """ Return all valid test targets/chroots from config. """ return get_valid_build_targets(*self.configured_tests_targets, default=None) @property def available_chroots(self) -> Set[str]: """ Returns set of available COPR targets. """ return { *filter( lambda chroot: not chroot.startswith("_"), self.api.copr_helper.get_copr_client().mock_chroot_proxy. get_list().keys(), ) } def get_built_packages(self, build_id: int, chroot: str) -> List: return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages( build_id, chroot).packages def get_build(self, build_id: int): return self.api.copr_helper.copr_client.build_proxy.get(build_id) def monitor_not_submitted_copr_builds(self, number_of_builds: int, reason: str): """ Measure the time it took to set the failed status in case of event (e.g. failed SRPM) that prevents Copr build to be submitted. """ time = measure_time(end=datetime.now(timezone.utc), begin=self.metadata.task_accepted_time) for _ in range(number_of_builds): self.pushgateway.copr_build_not_submitted_time.labels( reason=reason).observe(time) def run_copr_build(self) -> TaskResults: self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running, # pagure requires "valid url" url="", ) if results := self.create_srpm_if_needed(): return results if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "srpm_failure") return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "submit_failure") return TaskResults( success=False, details={ "msg": "Submit of the Copr build failed.", "error": str(ex) }, ) unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=chroot, ) self.monitor_not_submitted_copr_builds(1, "not_supported_target") unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", run_model=self.run_model, task_accepted_time=self.metadata.task_accepted_time, ) url = get_copr_build_info_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.status_reporter.comment( body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id, ), countdown=120, # do the first check in 120s ) return TaskResults(success=True, details={})
def run_copr_build(self) -> TaskResults: if not (self.job_build or self.job_tests): msg = "No copr_build or tests job defined." # we can't report it to end-user at this stage return TaskResults(success=False, details={"msg": msg}) self.report_status_to_all( description="Building SRPM ...", state=CommitStatus.pending, # pagure requires "valid url" url="", ) self.create_srpm_if_needed() if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() Pushgateway().push_copr_build_created() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=CommitStatus.error, description=f"Submit of the build failed: {ex}", ) return TaskResults( success=False, details={"msg": "Submit of the Copr build failed.", "error": str(ex)}, ) unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=CommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_log_url_from_flask(self.srpm_model.id), chroot=chroot, ) unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.get_or_create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", srpm_build=self.srpm_model, trigger_model=self.db_trigger, ) url = get_copr_build_info_url_from_flask(id_=copr_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.project.pr_comment( pr_id=self.metadata.pr_id, body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s ) return TaskResults(success=True, details={})