def handle_srpm_end(self): url = get_srpm_build_info_url(self.build.id) if self.copr_event.status != COPR_API_SUCC_STATE: failed_msg = "SRPM build failed, check the logs for details." self.copr_build_helper.report_status_to_all( state=BaseCommitStatus.failure, description=failed_msg, url=url, ) self.build.set_status(PG_BUILD_STATUS_FAILURE) self.copr_build_helper.monitor_not_submitted_copr_builds( len(self.copr_build_helper.build_targets), "srpm_failure") return TaskResults(success=False, details={"msg": failed_msg}) for build in CoprBuildModel.get_all_by_build_id( str(self.copr_event.build_id)): # from waiting_for_srpm to pending build.set_status("pending") self.build.set_status(PG_BUILD_STATUS_SUCCESS) self.copr_build_helper.report_status_to_all( state=BaseCommitStatus.running, description= "SRPM build succeeded. Waiting for RPM build to start...", url=url, ) msg = "SRPM build in Copr has finished." logger.debug(msg) return TaskResults(success=True, details={"msg": msg})
def get(self): """List all SRPM builds.""" result = [] first, last = indices() for build in SRPMBuildModel.get(first, last): build_dict = { "srpm_build_id": build.id, "success": build.success, "log_url": get_srpm_build_info_url(build.id), "build_submitted_time": optional_timestamp(build.build_submitted_time), } project = build.get_project() # Its possible that jobtrigger isnt stored in db if project: build_dict["repo_namespace"] = project.namespace build_dict["repo_name"] = project.repo_name build_dict["project_url"] = project.project_url build_dict["pr_id"] = build.get_pr_id() build_dict["branch_name"] = build.get_branch_name() result.append(build_dict) resp = response_maker( result, status=HTTPStatus.PARTIAL_CONTENT.value, ) resp.headers["Content-Range"] = f"srpm-builds {first + 1}-{last}/*" return resp
def run(self): if not self.build: model = ("SRPMBuildDB" if self.copr_event.chroot == COPR_SRPM_CHROOT else "CoprBuildDB") msg = f"Copr build {self.copr_event.build_id} not in {model}." logger.warning(msg) return TaskResults(success=False, details={"msg": msg}) self.set_start_time() self.set_logs_url() if self.copr_event.chroot == COPR_SRPM_CHROOT: url = get_srpm_build_info_url(self.build.id) self.copr_build_helper.report_status_to_all( description="SRPM build is in progress...", state=BaseCommitStatus.running, url=url, ) msg = "SRPM build in Copr has started..." return TaskResults(success=True, details={"msg": msg}) self.pushgateway.copr_builds_started.inc() url = get_copr_build_info_url(self.build.id) self.build.set_status("pending") self.copr_build_helper.report_status_to_all_for_chroot( description="RPM build is in progress...", state=BaseCommitStatus.running, url=url, chroot=self.copr_event.chroot, ) msg = f"Build on {self.copr_event.chroot} in copr has started..." return TaskResults(success=True, details={"msg": msg})
def test_koji_build_failed_kerberos(github_pr_event): trigger = flexmock( job_config_trigger_type=JobConfigTriggerType.pull_request, id=123, job_trigger_model_type=JobTriggerModelType.pull_request, ) flexmock(JobTriggerModel).should_receive("get_or_create").with_args( type=JobTriggerModelType.pull_request, trigger_id=123).and_return( flexmock(id=2, type=JobTriggerModelType.pull_request)) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return( trigger) helper = build_helper( event=github_pr_event, metadata=JobMetadataConfig(_targets=["bright-future"], scratch=True), db_trigger=trigger, ) flexmock(koji_build).should_receive("get_all_koji_targets").and_return( ["dark-past", "bright-future"]).never() flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.running, description="Building SRPM ...", check_name="production-build:bright-future", url="", links_to_external_services=None, markdown_content=None, ).and_return() flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.error, description= "Kerberos authentication error: the bad authentication error", check_name="production-build:bright-future", url=get_srpm_build_info_url(1), links_to_external_services=None, markdown_content=None, ).and_return() flexmock(GitProject).should_receive("get_pr").and_return( flexmock(source_project=flexmock())) flexmock(GitProject).should_receive( "set_commit_status").and_return().never() flexmock(SRPMBuildModel).should_receive("create_with_new_run").and_return( (flexmock(id=1, success=True), flexmock())) flexmock(KojiBuildModel).should_receive("create").and_return( flexmock(id=1)) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") flexmock(PackitAPI).should_receive("init_kerberos_ticket").and_raise( PackitCommandFailedError, "Command failed", stdout_output="", stderr_output="the bad authentication error", ) response = helper.run_koji_build() assert not response["success"] assert ("Kerberos authentication error: the bad authentication error" == response["details"]["msg"])
def test_koji_build_failed(github_pr_event): trigger = flexmock( job_config_trigger_type=JobConfigTriggerType.pull_request, id=123, job_trigger_model_type=JobTriggerModelType.pull_request, ) flexmock(JobTriggerModel).should_receive("get_or_create").with_args( type=JobTriggerModelType.pull_request, trigger_id=123).and_return( flexmock(id=2, type=JobTriggerModelType.pull_request)) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return( trigger) helper = build_helper( event=github_pr_event, metadata=JobMetadataConfig(_targets=["bright-future"], scratch=True), db_trigger=trigger, ) flexmock(koji_build).should_receive("get_all_koji_targets").and_return( ["dark-past", "bright-future"]).once() flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.running, description="Building SRPM ...", check_name="production-build:bright-future", url="", links_to_external_services=None, markdown_content=None, ).and_return() srpm_build_url = get_srpm_build_info_url(2) flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.error, description="Submit of the build failed: some error", check_name="production-build:bright-future", url=srpm_build_url, links_to_external_services=None, markdown_content=None, ).and_return() flexmock(GitProject).should_receive("get_pr").and_return( flexmock(source_project=flexmock())) flexmock(GitProject).should_receive( "set_commit_status").and_return().never() flexmock(SRPMBuildModel).should_receive("create_with_new_run").and_return( (flexmock(id=2, success=True), flexmock())) flexmock(KojiBuildModel).should_receive("create").and_return( flexmock(id=1)) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") # koji build flexmock(sentry_integration).should_receive( "send_to_sentry").and_return().once() flexmock(Upstream).should_receive("koji_build").and_raise( Exception, "some error") result = helper.run_koji_build() assert not result["success"] assert result["details"]["errors"] assert result["details"]["errors"]["bright-future"] == "some error"
def handle_rpm_build_start( self, build_id: int, web_url: str, waiting_for_srpm: bool = False ): """ Create models for Copr build chroots and report start of RPM build if the SRPM is already built. """ unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=chroot, ) self.monitor_not_submitted_copr_builds(1, "not_supported_target") unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="waiting_for_srpm" if waiting_for_srpm else "pending", run_model=self.run_model, task_accepted_time=self.metadata.task_accepted_time, ) if not waiting_for_srpm: url = get_copr_build_info_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.status_reporter.comment( body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s )
def test_get_srpm_logs(client): srpm_build_mock = flexmock() srpm_build_mock.id = 2 srpm_build_mock.logs = "asd\nqwe" flexmock(SRPMBuildModel).should_receive("get_by_id").and_return(srpm_build_mock) logs_url = get_srpm_build_info_url(2) assert logs_url == "https://localhost/results/srpm-builds/2"
def test_koji_build_target_not_supported(github_pr_event): trigger = flexmock( job_config_trigger_type=JobConfigTriggerType.pull_request, id=123, job_trigger_model_type=JobTriggerModelType.pull_request, ) flexmock(JobTriggerModel).should_receive("get_or_create").with_args( type=JobTriggerModelType.pull_request, trigger_id=123).and_return( flexmock(id=2, type=JobTriggerModelType.pull_request)) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return( trigger) helper = build_helper( event=github_pr_event, metadata=JobMetadataConfig(_targets=["nonexisting-target"], scratch=True), db_trigger=trigger, ) flexmock(koji_build).should_receive("get_all_koji_targets").and_return( ["dark-past", "bright-future"]).once() flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.running, description="Building SRPM ...", check_name="production-build:nonexisting-target", url="", links_to_external_services=None, markdown_content=None, ).and_return() flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.error, description="Target not supported: nonexisting-target", check_name="production-build:nonexisting-target", url=get_srpm_build_info_url(1), links_to_external_services=None, markdown_content=None, ).and_return() flexmock(GitProject).should_receive("get_pr").and_return( flexmock(source_project=flexmock())) flexmock(GitProject).should_receive( "set_commit_status").and_return().never() flexmock(SRPMBuildModel).should_receive("create_with_new_run").and_return(( flexmock(status="success", id=1).should_receive("set_url").with_args( "https://some.host/my.srpm").mock().should_receive( "set_start_time").mock().should_receive("set_status").mock(). should_receive("set_logs").mock().should_receive( "set_end_time").mock(), flexmock(), )) flexmock(KojiBuildModel).should_receive("create").and_return( flexmock(id=1)) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") response = helper.run_koji_build() assert not response["success"] assert ("Target not supported: nonexisting-target" == response["details"] ["errors"]["nonexisting-target"])
def test_koji_build_failed_srpm(github_pr_event): trigger = flexmock( job_config_trigger_type=JobConfigTriggerType.pull_request, id=123, job_trigger_model_type=JobTriggerModelType.pull_request, ) flexmock(JobTriggerModel).should_receive("get_or_create").with_args( type=JobTriggerModelType.pull_request, trigger_id=123).and_return( flexmock(id=2, type=JobTriggerModelType.pull_request)) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return( trigger) helper = build_helper( event=github_pr_event, metadata=JobMetadataConfig(_targets=["bright-future"], scratch=True), db_trigger=trigger, ) srpm_build_url = get_srpm_build_info_url(2) flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.running, description="Building SRPM ...", check_name="production-build:bright-future", url="", links_to_external_services=None, markdown_content=None, ).and_return() flexmock(StatusReporter).should_receive("set_status").with_args( state=BaseCommitStatus.failure, description="SRPM build failed, check the logs for details.", check_name="production-build:bright-future", url=srpm_build_url, links_to_external_services=None, markdown_content=None, ).and_return() flexmock(GitProject).should_receive("get_pr").and_return( flexmock(source_project=flexmock())) flexmock(GitProject).should_receive( "set_commit_status").and_return().never() flexmock(PackitAPI).should_receive("create_srpm").and_raise( Exception, "some error") flexmock(SRPMBuildModel).should_receive("create_with_new_run").and_return(( flexmock(status="failure", id=2).should_receive("set_url").with_args( "https://some.host/my.srpm").mock().should_receive( "set_start_time").mock().should_receive("set_status").mock(). should_receive("set_logs").mock().should_receive( "set_end_time").mock(), flexmock(), )) flexmock(KojiBuildModel).should_receive("create").never() flexmock(sentry_integration).should_receive( "send_to_sentry").and_return().once() result = helper.run_koji_build() assert not result["success"] assert "SRPM build failed" in result["details"]["msg"]
def get(self, forge, namespace, repo_name): """Project branches""" branches = GitProjectModel.get_project_branches(forge, namespace, repo_name) if not branches: return response_maker([]) result = [] for branch in branches: branch_info = { "branch": branch.name, "builds": [], "koji_builds": [], "srpm_builds": [], "tests": [], } for build in branch.get_copr_builds(): build_info = { "build_id": build.build_id, "chroot": build.target, "status": build.status, "web_url": build.web_url, } branch_info["builds"].append(build_info) for build in branch.get_koji_builds(): build_info = { "build_id": build.build_id, "chroot": build.target, "status": build.status, "web_url": build.web_url, } branch_info["koji_builds"].append(build_info) for build in branch.get_srpm_builds(): build_info = { "srpm_build_id": build.id, "status": build.status, "log_url": get_srpm_build_info_url(build.id), } branch_info["srpm_builds"].append(build_info) for test_run in branch.get_test_runs(): test_info = { "pipeline_id": test_run.pipeline_id, "chroot": test_run.target, "status": test_run.status, "web_url": test_run.web_url, } branch_info["tests"].append(test_info) result.append(branch_info) return response_maker(result)
def test_srpm_build_end_failure(srpm_build_end, pc_build_pr, srpm_build_model): srpm_build_end["status"] = COPR_API_FAIL_STATE pr = flexmock(source_project=flexmock()) flexmock(GithubProject).should_receive("is_private").and_return(False) flexmock(GithubProject).should_receive("get_pr").and_return(pr) flexmock(AbstractCoprBuildEvent).should_receive( "get_package_config").and_return(pc_build_pr) flexmock(CoprBuildModel).should_receive("get_all_by_build_id").and_return( [flexmock(target="fedora-33-x86_64")]) (flexmock(CoprBuildJobHelper).should_receive("get_build").with_args( 3122876).and_return( flexmock(source_package={"url": "https://my.host/my.srpm" })).at_least().once()) flexmock(Pushgateway).should_receive("push").once().and_return() flexmock(CoprBuildJobHelper).should_receive( "monitor_not_submitted_copr_builds") flexmock(SRPMBuildModel).should_receive("get_by_copr_build_id").and_return( srpm_build_model) srpm_build_model.should_call("set_status").with_args("failure").once() srpm_build_model.should_receive("set_end_time").once() url = get_srpm_build_info_url(1) flexmock(StatusReporter).should_receive("report").with_args( state=BaseCommitStatus.failure, description="SRPM build failed, check the logs for details.", url=url, check_names=["rpm-build:fedora-33-x86_64"], markdown_content=None, ).once() flexmock(Signature).should_receive("apply_async").once() flexmock(srpm_build_model).should_receive("set_url").with_args( "https://my.host/my.srpm").mock() processing_results = SteveJobs().process_message(srpm_build_end) event_dict, job, job_config, package_config = get_parameters_from_results( processing_results) assert json.dumps(event_dict) results = run_copr_build_end_handler( package_config=package_config, event=event_dict, job_config=job_config, ) assert not first_dict_value(results["job"])["success"]
def create_srpm_if_needed(self) -> Optional[TaskResults]: """ Create SRPM if is needed. Returns: Task results if job is cancelled because of merge conflicts, `None` otherwise. """ if self._srpm_path or self._srpm_model: return None results = self._create_srpm() if results: # merge conflict occurred self.report_status_to_all( state=BaseCommitStatus.neutral, description="Merge conflicts present", url=get_srpm_build_info_url(self.srpm_model.id), ) return results
def run_copr_build_from_source_script(self) -> TaskResults: """ Run copr build using custom source method. """ try: pr_id = self.metadata.pr_id script = create_source_script( url=self.metadata.project_url, ref=self.metadata.git_ref, pr_id=str(pr_id) if pr_id else None, merge_pr=self.package_config.merge_pr_in_ci, target_branch=self.project.get_pr(pr_id).target_branch if pr_id else None, job_config=self.job_config, ) build_id, web_url = self.submit_copr_build(script=script) except Exception as ex: self.handle_build_submit_error(ex) return TaskResults( success=False, details={"msg": "Submit of the Copr build failed.", "error": str(ex)}, ) self._srpm_model, self.run_model = SRPMBuildModel.create_with_new_run( copr_build_id=str(build_id), commit_sha=self.metadata.commit_sha, trigger_model=self.db_trigger, copr_web_url=web_url, ) self.report_status_to_all( description="SRPM build in Copr was submitted...", state=BaseCommitStatus.pending, url=get_srpm_build_info_url(self.srpm_model.id), ) self.handle_rpm_build_start(build_id, web_url, waiting_for_srpm=True) return TaskResults(success=True, details={})
def test_srpm_build_start(srpm_build_start, pc_build_pr, srpm_build_model): pr = flexmock(source_project=flexmock()) flexmock(GithubProject).should_receive("is_private").and_return(False) flexmock(GithubProject).should_receive("get_pr").and_return(pr) flexmock(AbstractCoprBuildEvent).should_receive( "get_package_config").and_return(pc_build_pr) flexmock(CoprBuildModel).should_receive("get_all_by_build_id").and_return( [flexmock(target="fedora-33-x86_64")]) flexmock(Pushgateway).should_receive("push").once().and_return() flexmock(SRPMBuildModel).should_receive("get_by_copr_build_id").and_return( srpm_build_model) flexmock(SRPMBuildModel).should_receive("set_start_time") flexmock(SRPMBuildModel).should_receive("set_build_logs_url") url = get_srpm_build_info_url(1) flexmock(StatusReporter).should_receive("report").with_args( state=BaseCommitStatus.running, description="SRPM build is in progress...", url=url, check_names=["rpm-build:fedora-33-x86_64"], markdown_content=None, ).once() flexmock(Signature).should_receive("apply_async").once() processing_results = SteveJobs().process_message(srpm_build_start) event_dict, job, job_config, package_config = get_parameters_from_results( processing_results) assert json.dumps(event_dict) results = run_copr_build_start_handler( package_config=package_config, event=event_dict, job_config=job_config, ) assert first_dict_value(results["job"])["success"]
def get(self, forge, namespace, repo_name): """List PRs""" result = [] first, last = indices() for pr in GitProjectModel.get_project_prs( first, last, forge, namespace, repo_name ): pr_info = { "pr_id": pr.pr_id, "builds": [], "koji_builds": [], "srpm_builds": [], "tests": [], } copr_builds = [] koji_builds = [] test_runs = [] srpm_builds = [] for build in pr.get_copr_builds(): build_info = { "build_id": build.build_id, "chroot": build.target, "status": build.status, "web_url": build.web_url, } copr_builds.append(build_info) pr_info["builds"] = copr_builds for build in pr.get_koji_builds(): build_info = { "build_id": build.build_id, "chroot": build.target, "status": build.status, "web_url": build.web_url, } koji_builds.append(build_info) pr_info["koji_builds"] = koji_builds for build in pr.get_srpm_builds(): build_info = { "srpm_build_id": build.id, "status": build.status, "log_url": get_srpm_build_info_url(build.id), } srpm_builds.append(build_info) pr_info["srpm_builds"] = srpm_builds for test_run in pr.get_test_runs(): test_info = { "pipeline_id": test_run.pipeline_id, "chroot": test_run.target, "status": str(test_run.status), "web_url": test_run.web_url, } test_runs.append(test_info) pr_info["tests"] = test_runs result.append(pr_info) if not result: return response_maker([]) resp = response_maker( result, status=HTTPStatus.PARTIAL_CONTENT.value, ) resp.headers["Content-Range"] = f"git-project-prs {first + 1}-{last}/*" return resp
class CoprBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.copr_build job_type_test = JobType.tests status_name_build: str = "rpm-build" status_name_test: str = "testing-farm" def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger: AbstractTriggerDbType, job_config: JobConfig, targets_override: Optional[Set[str]] = None, pushgateway: Optional[Pushgateway] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, pushgateway=pushgateway, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="copr-build" if self.job_build else "build", place="pull request", ) @property def default_project_name(self) -> str: """ Project name for copr. * use hostname prefix for non-github service * replace slash in namespace with dash * add `-stg` suffix for the stg app """ service_hostname = parse_git_repo( self.project.service.instance_url).hostname service_prefix = ("" if isinstance(self.project, GithubProject) else f"{service_hostname}-") namespace = self.project.namespace.replace("/", "-") stg = "-stg" if self.service_config.deployment == Deployment.stg else "" # We want to share project between all releases. # More details: https://github.com/packit/packit-service/issues/1044 identifier = "releases" if self.metadata.tag_name else self.metadata.identifier return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}" @property def job_project(self) -> Optional[str]: """ The job definition from the config file. """ if self.job_build and self.job_build.metadata.project: return self.job_build.metadata.project if self.job_tests and self.job_tests.metadata.project: return self.job_tests.metadata.project return self.default_project_name @property def job_owner(self) -> Optional[str]: """ Owner used for the copr build -- search the config or use the copr's config. """ if self.job_build and self.job_build.metadata.owner: return self.job_build.metadata.owner if self.job_tests and self.job_tests.metadata.owner: return self.job_tests.metadata.owner return self.api.copr_helper.copr_client.config.get("username") @property def preserve_project(self) -> Optional[bool]: """ If the project will be preserved or can be removed after 60 days. """ return self.job_build.metadata.preserve_project if self.job_build else None @property def list_on_homepage(self) -> Optional[bool]: """ If the project will be shown on the copr home page. """ return self.job_build.metadata.list_on_homepage if self.job_build else None @property def additional_repos(self) -> Optional[List[str]]: """ Additional repos that will be enable for copr build. """ return self.job_build.metadata.additional_repos if self.job_build else None @property def build_targets_all(self) -> Set[str]: """ Return all valid Copr build targets/chroots from config. """ return get_valid_build_targets(*self.configured_build_targets, default=None) @property def tests_targets_all(self) -> Set[str]: """ Return all valid test targets/chroots from config. """ return get_valid_build_targets(*self.configured_tests_targets, default=None) @property def available_chroots(self) -> Set[str]: """ Returns set of available COPR targets. """ return { *filter( lambda chroot: not chroot.startswith("_"), self.api.copr_helper.get_copr_client().mock_chroot_proxy. get_list().keys(), ) } def get_built_packages(self, build_id: int, chroot: str) -> List: return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages( build_id, chroot).packages def get_build(self, build_id: int): return self.api.copr_helper.copr_client.build_proxy.get(build_id) def monitor_not_submitted_copr_builds(self, number_of_builds: int, reason: str): """ Measure the time it took to set the failed status in case of event (e.g. failed SRPM) that prevents Copr build to be submitted. """ time = measure_time(end=datetime.now(timezone.utc), begin=self.metadata.task_accepted_time) for _ in range(number_of_builds): self.pushgateway.copr_build_not_submitted_time.labels( reason=reason).observe(time) def run_copr_build(self) -> TaskResults: self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running, # pagure requires "valid url" url="", ) if results := self.create_srpm_if_needed(): return results if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "srpm_failure") return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "submit_failure") return TaskResults( success=False, details={ "msg": "Submit of the Copr build failed.", "error": str(ex) }, ) unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=chroot, ) self.monitor_not_submitted_copr_builds(1, "not_supported_target") unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", run_model=self.run_model, task_accepted_time=self.metadata.task_accepted_time, ) url = get_copr_build_info_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.status_reporter.comment( body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id, ), countdown=120, # do the first check in 120s ) return TaskResults(success=True, details={})
class KojiBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.production_build job_type_test = None status_name_build: str = "production-build" status_name_test: str = None def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger, job_config: JobConfig, targets_override: Optional[Set[str]] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="production-build", place="pull request" ) # Lazy properties self._supported_koji_targets = None @property def is_scratch(self) -> bool: return self.job_build and self.job_build.metadata.scratch @property def build_targets_all(self) -> Set[str]: """ Return all valid Koji targets/chroots from config. """ return get_koji_targets(*self.configured_build_targets) @property def tests_targets_all(self) -> Set[str]: """ [not used now] Return all valid test targets/chroots from config. """ return get_koji_targets(*self.configured_tests_targets) @property def supported_koji_targets(self): if self._supported_koji_targets is None: self._supported_koji_targets = get_all_koji_targets() return self._supported_koji_targets def run_koji_build(self) -> TaskResults: self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running ) if results := self.create_srpm_if_needed(): return results if self.srpm_model.status != PG_BUILD_STATUS_SUCCESS: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) try: # We need to do it manually # because we don't use PackitAPI.build, but PackitAPI.up.koji_build self.api.init_kerberos_ticket() except PackitCommandFailedError as ex: msg = f"Kerberos authentication error: {ex.stderr_output}" logger.error(msg) self.report_status_to_all( state=BaseCommitStatus.error, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) return TaskResults(success=False, details={"msg": msg}) errors: Dict[str, str] = {} for target in self.build_targets: if target not in self.supported_koji_targets: msg = f"Target not supported: {target}" self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), chroot=target, ) errors[target] = msg continue try: build_id, web_url = self.run_build(target=target) except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=target, ) errors[target] = str(ex) continue koji_build = KojiBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, web_url=web_url, target=target, status="pending", run_model=self.run_model, ) url = get_koji_build_info_url(id_=koji_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Building RPM ...", url=url, chroot=target, ) if errors: return TaskResults( success=False, details={ "msg": "Koji build submit was not successful for all chroots.", "errors": errors, }, ) # TODO: release the hounds! """ celery_app.send_task( "task.babysit_koji_build", args=(build_metadata.build_id,), countdown=120, # do the first check in 120s ) """ return TaskResults(success=True, details={})
class CoprBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.copr_build job_type_test = JobType.tests status_name_build: str = "rpm-build" status_name_test: str = "testing-farm" def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger: AbstractTriggerDbType, job_config: JobConfig, targets_override: Optional[Set[str]] = None, pushgateway: Optional[Pushgateway] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, pushgateway=pushgateway, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="copr-build" if self.job_build else "build", place="pull request", ) @property def default_project_name(self) -> str: """ Project name for copr. * use hostname prefix for non-github service * replace slash in namespace with dash * add `-stg` suffix for the stg app """ service_hostname = parse_git_repo(self.project.service.instance_url).hostname service_prefix = ( "" if isinstance(self.project, GithubProject) else f"{service_hostname}-" ) namespace = self.project.namespace.replace("/", "-") stg = "-stg" if self.service_config.deployment == Deployment.stg else "" # We want to share project between all releases. # More details: https://github.com/packit/packit-service/issues/1044 identifier = "releases" if self.metadata.tag_name else self.metadata.identifier return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}" @property def job_project(self) -> Optional[str]: """ The job definition from the config file. """ if self.job_build and self.job_build.metadata.project: return self.job_build.metadata.project if self.job_tests and self.job_tests.metadata.project: return self.job_tests.metadata.project return self.default_project_name @property def job_owner(self) -> Optional[str]: """ Owner used for the copr build -- search the config or use the copr's config. """ if self.job_build and self.job_build.metadata.owner: return self.job_build.metadata.owner if self.job_tests and self.job_tests.metadata.owner: return self.job_tests.metadata.owner return self.api.copr_helper.copr_client.config.get("username") @property def preserve_project(self) -> Optional[bool]: """ If the project will be preserved or can be removed after 60 days. """ return self.job_build.metadata.preserve_project if self.job_build else None @property def list_on_homepage(self) -> Optional[bool]: """ If the project will be shown on the copr home page. """ return self.job_build.metadata.list_on_homepage if self.job_build else None @property def additional_repos(self) -> Optional[List[str]]: """ Additional repos that will be enable for copr build. """ return self.job_build.metadata.additional_repos if self.job_build else None @property def build_targets_all(self) -> Set[str]: """ Return all valid Copr build targets/chroots from config. """ return get_valid_build_targets(*self.configured_build_targets, default=None) @property def tests_targets_all(self) -> Set[str]: """ Return all valid test targets/chroots from config. """ return get_valid_build_targets(*self.configured_tests_targets, default=None) @property def available_chroots(self) -> Set[str]: """ Returns set of available COPR targets. """ return { *filter( lambda chroot: not chroot.startswith("_"), self.api.copr_helper.get_copr_client() .mock_chroot_proxy.get_list() .keys(), ) } def get_built_packages(self, build_id: int, chroot: str) -> List: return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages( build_id, chroot ).packages def get_build(self, build_id: int): return self.api.copr_helper.copr_client.build_proxy.get(build_id) def monitor_not_submitted_copr_builds(self, number_of_builds: int, reason: str): """ Measure the time it took to set the failed status in case of event (e.g. failed SRPM) that prevents Copr build to be submitted. """ time = measure_time( end=datetime.now(timezone.utc), begin=self.metadata.task_accepted_time ) for _ in range(number_of_builds): self.pushgateway.copr_build_not_submitted_time.labels( reason=reason ).observe(time) def get_packit_copr_download_urls(self) -> List[str]: """ Get packit package download urls for latest succeeded build in Copr project for the given environment (for production packit/packit-stable, else packit/packit-dev, e.g https://download.copr.fedorainfracloud.org/results/packit/ packit-stable/fedora-35-x86_64/03247833-packit/ packit-0.44.1.dev4+g5ec2bd1-1.20220124144110935127.stable.4.g5ec2bd1.fc35.noarch.rpm) Returns: list of urls """ try: latest_successful_build_id = ( self.api.copr_helper.copr_client.package_proxy.get( ownername="packit", projectname="packit-stable" if self.service_config.deployment == Deployment.prod else "packit-dev", packagename="packit", with_latest_succeeded_build=True, ).builds["latest_succeeded"]["id"] ) result_url = self.api.copr_helper.copr_client.build_chroot_proxy.get( latest_successful_build_id, "fedora-35-x86_64" ).result_url package_nvrs = self.get_built_packages( latest_successful_build_id, "fedora-35-x86_64" ) built_packages = get_package_nvrs(package_nvrs) return [f"{result_url}{package}.rpm" for package in built_packages] except Exception as ex: logger.debug( f"Getting download urls for latest packit/packit-stable " f"build was not successful: {ex}" ) raise ex def run_copr_build_from_source_script(self) -> TaskResults: """ Run copr build using custom source method. """ try: pr_id = self.metadata.pr_id script = create_source_script( url=self.metadata.project_url, ref=self.metadata.git_ref, pr_id=str(pr_id) if pr_id else None, merge_pr=self.package_config.merge_pr_in_ci, target_branch=self.project.get_pr(pr_id).target_branch if pr_id else None, job_config=self.job_config, ) build_id, web_url = self.submit_copr_build(script=script) except Exception as ex: self.handle_build_submit_error(ex) return TaskResults( success=False, details={"msg": "Submit of the Copr build failed.", "error": str(ex)}, ) self._srpm_model, self.run_model = SRPMBuildModel.create_with_new_run( copr_build_id=str(build_id), commit_sha=self.metadata.commit_sha, trigger_model=self.db_trigger, copr_web_url=web_url, ) self.report_status_to_all( description="SRPM build in Copr was submitted...", state=BaseCommitStatus.pending, url=get_srpm_build_info_url(self.srpm_model.id), ) self.handle_rpm_build_start(build_id, web_url, waiting_for_srpm=True) return TaskResults(success=True, details={}) def submit_copr_build(self, script: Optional[str] = None) -> Tuple[int, str]: """ Create the project in Copr if not exists and submit a new build using source script method Return: tuple of build ID and web url """ owner = self.create_copr_project_if_not_exists() try: if script: build = self.api.copr_helper.copr_client.build_proxy.create_from_custom( ownername=owner, projectname=self.job_project, script=script, script_builddeps=self.get_packit_copr_download_urls() + SRPM_BUILD_DEPS, buildopts={ "chroots": list(self.build_targets), }, ) else: build = self.api.copr_helper.copr_client.build_proxy.create_from_file( ownername=owner, projectname=self.job_project, path=self.srpm_path, buildopts={ "chroots": list(self.build_targets), }, ) except CoprRequestException as ex: if "You don't have permissions to build in this copr." in str( ex ) or "is not allowed to build in the copr" in str(ex): self.api.copr_helper.copr_client.project_proxy.request_permissions( ownername=owner, projectname=self.job_project, permissions={"builder": True}, ) # notify user, PR if exists, commit comment otherwise permissions_url = self.api.copr_helper.get_copr_settings_url( owner, self.job_project, section="permissions" ) self.status_reporter.comment( body="We have requested the `builder` permissions " f"for the {owner}/{self.job_project} Copr project.\n" "\n" "Please confirm the request on the " f"[{owner}/{self.job_project} Copr project permissions page]" f"({permissions_url})" " and retrigger the build.", ) raise ex return build.id, self.api.copr_helper.copr_web_build_url(build) def run_copr_build(self) -> TaskResults: """ Run copr build using SRPM built by us. """ self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running, # pagure requires "valid url" url="", ) if results := self.create_srpm_if_needed(): return results if self.srpm_model.status != PG_BUILD_STATUS_SUCCESS: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) self.monitor_not_submitted_copr_builds( len(self.build_targets), "srpm_failure" ) return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.submit_copr_build() except Exception as ex: self.handle_build_submit_error(ex) return TaskResults( success=False, details={"msg": "Submit of the Copr build failed.", "error": str(ex)}, ) self.handle_rpm_build_start(build_id, web_url) return TaskResults(success=True, details={})