def copr_build_helper(self) -> CoprBuildJobHelper: # when reporting state of SRPM build built in Copr targets_override = ({ build.target for build in CoprBuildModel.get_all_by_build_id( str(self.copr_event.build_id)) } if self.copr_event.chroot == COPR_SRPM_CHROOT else None) if not self._copr_build_helper: self._copr_build_helper = CoprBuildJobHelper( service_config=self.service_config, package_config=self.package_config, project=self.project, metadata=self.data, db_trigger=self.db_trigger, job_config=self.job_config, pushgateway=self.pushgateway, targets_override=targets_override, ) return self._copr_build_helper
def test_copr_build_success_set_test_check_gitlab(gitlab_mr_event): # status is set for each build-target (4x): # - Building SRPM ... # - Starting RPM build... # status is set for each test-target (4x): # - Building SRPM ... # - Starting RPM build... test_job = JobConfig( type=JobType.tests, trigger=JobConfigTriggerType.pull_request, metadata=JobMetadataConfig(), ) trigger = flexmock(job_config_trigger_type=JobConfigTriggerType.release) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return( trigger) helper = build_helper(jobs=[test_job], event=gitlab_mr_event, db_trigger=trigger) flexmock(GitProject).should_receive( "set_commit_status").and_return().times(16) flexmock(GitProject).should_receive("get_pr").and_return(flexmock()) flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=True)) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1)) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") # copr build flexmock(CoprHelper).should_receive( "create_copr_project_if_not_exists").and_return(None) flexmock(CoprHelper).should_receive("get_copr_client").and_return( flexmock( config={"copr_url": "https://copr.fedorainfracloud.org/"}, build_proxy=flexmock().should_receive( "create_from_file").and_return( flexmock(id=2, projectname="the-project-name", ownername="the-owner")).mock(), )) flexmock(Celery).should_receive("send_task").once() assert helper.run_copr_build()["success"]
def get(self, id): """A specific copr build details. From copr_build hash, filled by worker.""" builds_list = CoprBuildModel.get_all_by_build_id(str(id)) if not bool(builds_list.first()): return response_maker( {"error": "No info about build stored in DB"}, status=HTTPStatus.NOT_FOUND, ) build = builds_list[0] build_dict = { "project": build.project_name, "owner": build.owner, "build_id": build.build_id, "status": build.status, # Legacy, remove later. "status_per_chroot": {}, "chroots": [], "build_submitted_time": optional_time(build.build_submitted_time), "build_start_time": optional_time(build.build_start_time), "build_finished_time": optional_time(build.build_finished_time), "commit_sha": build.commit_sha, "web_url": build.web_url, "srpm_logs": build.srpm_build.logs if build.srpm_build else None, # For backwards compatability with the old redis based API "ref": build.commit_sha, } project = build.get_project() if project: build_dict["repo_namespace"] = project.namespace build_dict["repo_name"] = project.repo_name build_dict["git_repo"] = project.project_url build_dict["pr_id"] = build.get_pr_id() build_dict["branch_name"] = build.get_branch_name() # merge chroots into one for sbid_build in builds_list: build_dict["chroots"].append(sbid_build.target) # Get status per chroot as well build_dict["status_per_chroot"][sbid_build.target] = sbid_build.status return response_maker(build_dict)
def test_copr_build_fails_in_packit(github_pr_event): # status is set for each build-target (4x): # - Building SRPM ... # - Build failed, check latest comment for details. helper = build_helper(event=github_pr_event) templ = "packit-stg/rpm-build-fedora-{ver}-x86_64" flexmock(copr_build).should_receive("get_srpm_log_url_from_flask").and_return( "https://test.url" ) for v in ["29", "30", "31", "rawhide"]: flexmock(GitProject).should_receive("set_commit_status").with_args( "528b803be6f93e19ca4130bf4976f2800a3004c4", CommitStatus.pending, "", "Building SRPM ...", templ.format(ver=v), trim=True, ).and_return().once() for v in ["29", "30", "31", "rawhide"]: flexmock(GitProject).should_receive("set_commit_status").with_args( "528b803be6f93e19ca4130bf4976f2800a3004c4", CommitStatus.failure, "https://test.url", "SRPM build failed, check the logs for details.", templ.format(ver=v), trim=True, ).and_return().once() flexmock(GitProject).should_receive("get_pr").and_return(flexmock()) flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=False, id=2) ) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1) ) flexmock(sentry_integration).should_receive("send_to_sentry").and_return().once() flexmock(PackitAPI).should_receive("create_srpm").and_raise( FailedCreateSRPM, "some error" ) flexmock(CoprBuildJobHelper).should_receive("run_build").never() assert not helper.run_copr_build()["success"]
def from_build_id( cls, topic: str, build_id: int, chroot: str, status: int, owner: str, project_name: str, pkg: str, timestamp, ) -> Optional["AbstractCoprBuildEvent"]: """ Return cls instance or None if build_id not in CoprBuildDB""" build = CoprBuildModel.get_by_build_id(str(build_id), chroot) if not build: logger.warning(f"Build id {build_id} not in CoprBuildDB.") return None return cls(topic, build_id, build, chroot, status, owner, project_name, pkg, timestamp)
def test_copr_build_success_set_test_check(pull_request_event): # status is set for each build-target (4x): # - Building SRPM ... # - Building RPM ... # status is set for each test-target (4x): # - Building SRPM ... # - Building RPM ... test_job = JobConfig(type=JobType.tests, trigger=JobConfigTriggerType.pull_request,) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return( flexmock(job_config_trigger_type=JobConfigTriggerType.release) ) helper = build_helper(jobs=[test_job], event=pull_request_event) flexmock(GitProject).should_receive("set_commit_status").and_return().times(16) flexmock(SRPMBuildModel).should_receive("create").and_return(SRPMBuildModel()) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1) ) flexmock(PackitAPI).should_receive("run_copr_build").and_return(1, None).once() flexmock(Celery).should_receive("send_task").once() assert helper.run_copr_build()["success"]
def packit_build_752(): pr_model = PullRequestModel.get_or_create( pr_id=752, namespace="packit-service", repo_name="packit", project_url="https://github.com/packit-service/packit", ) srpm_build = SRPMBuildModel.create("asd\nqwe\n", success=True) yield CoprBuildModel.get_or_create( build_id=str(BUILD_ID), commit_sha="687abc76d67d", project_name="packit-service-packit-752", owner="packit", web_url=("https://download.copr.fedorainfracloud.org/" "results/packit/packit-service-packit-752"), target="fedora-rawhide-x86_64", status="pending", srpm_build=srpm_build, trigger_model=pr_model, )
def test_copr_build_success(github_pr_event): # status is set for each build-target (4x): # - Building SRPM ... # - Starting RPM build... helper = build_helper( event=github_pr_event, db_trigger=flexmock( job_config_trigger_type=JobConfigTriggerType.pull_request, id=123), ) flexmock(GitProject).should_receive( "set_commit_status").and_return().times(8) flexmock(GitProject).should_receive("get_pr").and_return(flexmock()) flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=True)) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1)) flexmock(PullRequestGithubEvent).should_receive("db_trigger").and_return( flexmock()) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") # copr build flexmock(CoprHelper).should_receive( "create_copr_project_if_not_exists").and_return(None) flexmock(CoprHelper).should_receive("get_copr_client").and_return( flexmock( config={"copr_url": "https://copr.fedorainfracloud.org/"}, build_proxy=flexmock().should_receive( "create_from_file").and_return( flexmock(id=2, projectname="the-project-name", ownername="the-owner")).mock(), mock_chroot_proxy=flexmock().should_receive("get_list").and_return( {target: "" for target in DEFAULT_TARGETS}).mock(), )) flexmock(Pushgateway).should_receive("push_copr_build_created") flexmock(Celery).should_receive("send_task").once() assert helper.run_copr_build()["success"]
def test_copr_and_koji_build_for_one_trigger(clean_before_and_after): pr1 = PullRequestModel.get_or_create( pr_id=1, namespace="the-namespace", repo_name="the-repo-name", project_url="https://github.com/the-namespace/the-repo-name", ) pr1_trigger = JobTriggerModel.get_or_create( type=JobTriggerModelType.pull_request, trigger_id=pr1.id) srpm_build = SRPMBuildModel.create("asd\nqwe\n", success=True, trigger_model=pr1) copr_build = CoprBuildModel.get_or_create( build_id="123456", commit_sha="687abc76d67d", project_name="SomeUser-hello-world-9", owner="packit", web_url="https://copr.something.somewhere/123456", target=SampleValues.target, status="pending", srpm_build=srpm_build, trigger_model=pr1, ) koji_build = KojiBuildModel.get_or_create( build_id="987654", commit_sha="687abc76d67d", web_url="https://copr.something.somewhere/123456", target=SampleValues.target, status="pending", srpm_build=srpm_build, trigger_model=pr1, ) assert copr_build in pr1_trigger.copr_builds assert koji_build in pr1_trigger.koji_builds assert srpm_build in pr1_trigger.srpm_builds assert srpm_build.job_trigger.get_trigger_object() == pr1 assert copr_build.job_trigger.get_trigger_object() == pr1 assert koji_build.job_trigger.get_trigger_object() == pr1
def test_get_logs(client): chroot = "foo-1-x86_64" state = "pending" build_id = 2 project = GitProjectModel() project.namespace = "john-foo" project.repo_name = "bar" pr = PullRequestModel() pr.pr_id = 234 pr.project = project c = CoprBuildModel() c.target = chroot c.build_id = str(build_id) c.srpm_build_id = 11 c.status = state c.web_url = ( "https://copr.fedorainfracloud.org/coprs/john-foo-bar/john-foo-bar/build/2/" ) c.build_logs_url = "https://localhost:5000/build/2/foo-1-x86_64/logs" flexmock(CoprBuildModel).should_receive("get_by_id").and_return(c) flexmock(CoprBuildModel).should_receive("get_project").and_return(project) flexmock(CoprBuildModel).should_receive("job_trigger").and_return( flexmock(get_trigger_object=lambda: pr)) url = "/copr-build/1" logs_url = get_copr_build_info_url_from_flask(1) assert logs_url.endswith(url) resp = client.get(url).data.decode() assert f"srpm-build/{c.srpm_build_id}/logs" in resp assert c.web_url in resp assert c.build_logs_url in resp assert c.target in resp
def test_copr_build_no_targets_gitlab(gitlab_mr_event): # status is set for each build-target (fedora-stable => 2x): # - Building SRPM ... # - Starting RPM build... helper = build_helper( event=gitlab_mr_event, metadata=JobMetadataConfig(owner="nobody") ) flexmock(GitProject).should_receive("set_commit_status").and_return().times(4) flexmock(GitProject).should_receive("get_pr").and_return(flexmock()) flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=True) ) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1) ) flexmock(MergeRequestGitlabEvent).should_receive("db_trigger").and_return( flexmock() ) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") # copr build flexmock(CoprHelper).should_receive("create_copr_project_if_not_exists").and_return( None ) flexmock(CoprHelper).should_receive("get_copr_client").and_return( flexmock( config={"copr_url": "https://copr.fedorainfracloud.org/"}, build_proxy=flexmock() .should_receive("create_from_file") .and_return( flexmock(id=2, projectname="the-project-name", ownername="the-owner") ) .mock(), ) ) flexmock(Celery).should_receive("send_task").once() assert helper.run_copr_build()["success"]
def babysit_copr_build(self, build_id: int): """ check status of a copr build and update it in DB """ logger.debug(f"getting copr build ID {build_id} from DB") builds = CoprBuildModel.get_all_by_build_id(build_id) if builds: copr_client = CoprClient.create_from_config_file() build_copr = copr_client.build_proxy.get(build_id) if not build_copr.ended_on: logger.info("The copr build is still in progress") self.retry() logger.info(f"The status is {build_copr.state}") for build in builds: if build.status != "pending": logger.info(f"DB state says {build.status}, " "things were taken care of already, skipping.") continue chroot_build = copr_client.build_chroot_proxy.get( build_id, build.target) event = CoprBuildEvent( topic=FedmsgTopic.copr_build_finished.value, build_id=build_id, build=build, chroot=build.target, status=(COPR_API_SUCC_STATE if chroot_build.state == COPR_SUCC_STATE else COPR_API_FAIL_STATE), owner=build.owner, project_name=build.project_name, pkg=build_copr.source_package.get( "name", ""), # this seems to be the SRPM name ) CoprBuildEndHandler(ServiceConfig.get_service_config(), job_config=None, event=event).run() else: logger.warning(f"Copr build {build_id} not in DB.")
def _payload_install_test(self, build_id: int, chroot: str) -> dict: """ If the project doesn't use fmf, but still wants to run tests in TF. TF provides 'installation test', we request it in ['test']['fmf']['url']. We don't specify 'artifacts' as in _payload(), but 'variables'. """ copr_build = CoprBuildModel.get_by_build_id(build_id) distro, arch = self.chroot2distro_arch(chroot) compose = self.distro2compose(distro, arch) return { "api_key": self.service_config.testing_farm_secret, "test": { "fmf": { "url": TESTING_FARM_INSTALLABILITY_TEST_URL, "ref": TESTING_FARM_INSTALLABILITY_TEST_REF, "name": "/packit/installation", }, }, "environments": [{ "arch": arch, "os": { "compose": compose }, "variables": { "REPOSITORY": f"{copr_build.owner}/{copr_build.project_name}", }, }], "notification": { "webhook": { "url": f"{self.api_url}/testing-farm/results", "token": self.service_config.testing_farm_secret, }, }, }
def test_copr_build_check_names_gitlab(gitlab_mr_event): trigger = flexmock(job_config_trigger_type=JobConfigTriggerType.release, id=123) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return(trigger) helper = build_helper( event=gitlab_mr_event, metadata=JobMetadataConfig(targets=["bright-future-x86_64"], owner="nobody"), db_trigger=trigger, ) flexmock(copr_build).should_receive( "get_copr_build_info_url_from_flask" ).and_return("https://test.url") flexmock(StatusReporter).should_receive("set_status").with_args( state=CommitStatus.pending, description="Building SRPM ...", check_name="packit-stg/rpm-build-bright-future-x86_64", url="", ).and_return() flexmock(StatusReporter).should_receive("set_status").with_args( state=CommitStatus.pending, description="Starting RPM build...", check_name="packit-stg/rpm-build-bright-future-x86_64", url="https://test.url", ).and_return() flexmock(GitProject).should_receive("set_commit_status").and_return().never() flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=True) ) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1) ) flexmock(MergeRequestGitlabEvent).should_receive("db_trigger").and_return( flexmock() ) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") # copr build flexmock(CoprHelper).should_receive("create_copr_project_if_not_exists").with_args( project="the-example-namespace-the-example-repo-1-stg", chroots=["bright-future-x86_64"], owner="nobody", description=None, instructions=None, preserve_project=None, list_on_homepage=None, additional_repos=[], request_admin_if_needed=True, ).and_return(None) flexmock(CoprHelper).should_receive("get_copr_client").and_return( flexmock( config={"copr_url": "https://copr.fedorainfracloud.org/"}, build_proxy=flexmock() .should_receive("create_from_file") .and_return( flexmock(id=2, projectname="the-project-name", ownername="nobody") ) .mock(), ) ) flexmock(Celery).should_receive("send_task").once() assert helper.run_copr_build()["success"]
def test_copr_build_fails_to_update_copr_project(github_pr_event): # status is set for each build-target (4x): # - Building SRPM ... # - Build failed, check latest comment for details. helper = build_helper(event=github_pr_event) templ = "packit-stg/rpm-build-fedora-{ver}-x86_64" flexmock(copr_build).should_receive("get_srpm_log_url_from_flask").and_return( "https://test.url" ) for v in ["29", "30", "31", "rawhide"]: flexmock(GitProject).should_receive("set_commit_status").with_args( "528b803be6f93e19ca4130bf4976f2800a3004c4", CommitStatus.pending, "", "Building SRPM ...", templ.format(ver=v), trim=True, ).and_return().once() for v in ["29", "30", "31", "rawhide"]: flexmock(GitProject).should_receive("set_commit_status").with_args( "528b803be6f93e19ca4130bf4976f2800a3004c4", CommitStatus.error, "", "Submit of the build failed: Copr project update failed.", templ.format(ver=v), trim=True, ).and_return().once() flexmock(GitProject).should_receive("get_pr").and_return(flexmock()) flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=True, id=2) ) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1) ) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") flexmock(GitProject).should_receive("pr_comment").with_args( pr_id=342, body="Based on your Packit configuration the settings of the " "nobody/the-example-namespace-the-example-repo-342-stg " "Copr project would need to be updated as follows:\n" "\n" "| field | old value | new value |\n" "| ----- | --------- | --------- |\n" "| chroots | ['f30', 'f31'] | ['f31', 'f32'] |\n" "| description | old | new |\n" "\n" "\n" "Packit was unable to update the settings above " "as it is missing `admin` permissions on the " "nobody/the-example-namespace-the-example-repo-342-stg Copr project.\n" "\n" "To fix this you can do one of the following:\n" "\n" "- Grant Packit `admin` permissions on the " "nobody/the-example-namespace-the-example-repo-342-stg Copr project on the " "[permissions page](https://copr.fedorainfracloud.org/" "coprs/nobody/the-example-namespace-the-example-repo-342-stg/permissions/).\n" "- Change the above Copr project settings manually on the " "[settings page](https://copr.fedorainfracloud.org/" "coprs/nobody/the-example-namespace-the-example-repo-342-stg/edit/) " "to match the Packit configuration.\n" "- Update the Packit configuration to match the Copr project settings.\n" "\n" "Please re-trigger the build, once the issue above is fixed.\n", ).and_return().once() flexmock(sentry_integration).should_receive("send_to_sentry").and_return().once() # copr build flexmock(CoprHelper).should_receive("get_copr_settings_url").with_args( "nobody", "the-example-namespace-the-example-repo-342-stg", section="permissions", ).and_return( "https://copr.fedorainfracloud.org/" "coprs/nobody/the-example-namespace-the-example-repo-342-stg/permissions/" ).once() flexmock(CoprHelper).should_receive("get_copr_settings_url").with_args( "nobody", "the-example-namespace-the-example-repo-342-stg", ).and_return( "https://copr.fedorainfracloud.org/" "coprs/nobody/the-example-namespace-the-example-repo-342-stg/edit/" ).once() flexmock(CoprHelper).should_receive("create_copr_project_if_not_exists").and_raise( PackitCoprSettingsException, "Copr project update failed.", fields_to_change={ "chroots": (["f30", "f31"], ["f31", "f32"]), "description": ("old", "new"), }, ) assert not helper.run_copr_build()["success"]
def test_copr_build_set_status(clean_before_and_after, a_copr_build_for_pr): assert a_copr_build_for_pr.status == "pending" a_copr_build_for_pr.set_status("awesome") assert a_copr_build_for_pr.status == "awesome" b = CoprBuildModel.get_by_build_id(a_copr_build_for_pr.build_id, TARGET) assert b.status == "awesome"
def test_copr_build_check_names_multiple_jobs(github_pr_event): trigger = flexmock( job_config_trigger_type=JobConfigTriggerType.pull_request, id=123 ) flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return(trigger) helper = build_helper( event=github_pr_event, jobs=[ # We run only the job it's config is passed to the handler. # Other one(s) has to be run by a different handler instance. JobConfig( type=JobType.copr_build, trigger=JobConfigTriggerType.pull_request, metadata=JobMetadataConfig( targets=["fedora-rawhide-x86_64"], owner="nobody" ), actions={ActionName.post_upstream_clone: "ls /*"}, ), JobConfig( type=JobType.copr_build, trigger=JobConfigTriggerType.pull_request, metadata=JobMetadataConfig( targets=["fedora-32-x86_64"], owner="nobody" ), actions={ActionName.post_upstream_clone: 'bash -c "ls /*"'}, ), ], db_trigger=trigger, selected_job=JobConfig( type=JobType.copr_build, trigger=JobConfigTriggerType.pull_request, metadata=JobMetadataConfig(targets=["fedora-32-x86_64"], owner="nobody"), actions={ActionName.post_upstream_clone: 'bash -c "ls /*"'}, ), ) # we need to make sure that pr_id is set # so we can check it out and add it to spec's release field assert helper.metadata.pr_id flexmock(copr_build).should_receive( "get_copr_build_info_url_from_flask" ).and_return("https://test.url") flexmock(StatusReporter).should_receive("set_status").with_args( state=CommitStatus.pending, description="Building SRPM ...", check_name="packit-stg/rpm-build-fedora-32-x86_64", url="", ).and_return().once() flexmock(StatusReporter).should_receive("set_status").with_args( state=CommitStatus.pending, description="Starting RPM build...", check_name="packit-stg/rpm-build-fedora-32-x86_64", url="https://test.url", ).and_return().once() flexmock(GitProject).should_receive("set_commit_status").and_return().never() flexmock(SRPMBuildModel).should_receive("create").and_return( SRPMBuildModel(success=True) ) flexmock(CoprBuildModel).should_receive("get_or_create").and_return( CoprBuildModel(id=1) ) flexmock(PullRequestGithubEvent).should_receive("db_trigger").and_return(flexmock()) flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm") # copr build flexmock(CoprHelper).should_receive("create_copr_project_if_not_exists").with_args( project="the-example-namespace-the-example-repo-342-stg", chroots=["fedora-32-x86_64"], owner="nobody", description=None, instructions=None, preserve_project=None, list_on_homepage=None, additional_repos=[], request_admin_if_needed=True, ).and_return(None) flexmock(CoprHelper).should_receive("get_copr_client").and_return( flexmock( config={"copr_url": "https://copr.fedorainfracloud.org/"}, build_proxy=flexmock() .should_receive("create_from_file") .and_return( flexmock(id=2, projectname="the-project-name", ownername="packit") ) .mock(), ) ) flexmock(Celery).should_receive("send_task").once() assert helper.run_copr_build()["success"]
def db_trigger(self) -> Optional[AbstractTriggerDbType]: if not self._db_trigger: build = CoprBuildModel.get_by_id(self.build_id) self._db_trigger = build.job_trigger.get_trigger_object() return self._db_trigger
def run(self): build_job_helper = CoprBuildJobHelper( config=self.config, package_config=self.event.package_config, project=self.event.project, event=self.event, ) if self.event.chroot == "srpm-builds": # we don't want to set check for this msg = "SRPM build in copr has finished." logger.debug(msg) return HandlerResults(success=True, details={"msg": msg}) build = CoprBuildModel.get_by_build_id( str(self.event.build_id), self.event.chroot ) if not build: # TODO: how could this happen? msg = f"Copr build {self.event.build_id} not in CoprBuildDB." logger.warning(msg) return HandlerResults(success=False, details={"msg": msg}) if build.status in [ PG_COPR_BUILD_STATUS_FAILURE, PG_COPR_BUILD_STATUS_SUCCESS, ]: msg = ( f"Copr build {self.event.build_id} is already" f" processed (status={build.status})." ) logger.info(msg) return HandlerResults(success=True, details={"msg": msg}) end_time = ( datetime.utcfromtimestamp(self.event.timestamp) if self.event.timestamp else None ) build.set_end_time(end_time) url = get_copr_build_log_url_from_flask(build.id) # https://pagure.io/copr/copr/blob/master/f/common/copr_common/enums.py#_42 if self.event.status != COPR_API_SUCC_STATE: failed_msg = "RPMs failed to be built." build_job_helper.report_status_to_all_for_chroot( state=CommitStatus.failure, description=failed_msg, url=url, chroot=self.event.chroot, ) build.set_status(PG_COPR_BUILD_STATUS_FAILURE) return HandlerResults(success=False, details={"msg": failed_msg}) if ( build_job_helper.job_build and build_job_helper.job_build.trigger == JobConfigTriggerType.pull_request and self.event.pr_id and isinstance(self.event.project, GithubProject) and not self.was_last_packit_comment_with_congratulation() and self.event.package_config.notifications.pull_request.successful_build ): msg = ( f"Congratulations! One of the builds has completed. :champagne:\n\n" "You can install the built RPMs by following these steps:\n\n" "* `sudo yum install -y dnf-plugins-core` on RHEL 8\n" "* `sudo dnf install -y dnf-plugins-core` on Fedora\n" f"* `dnf copr enable {self.event.owner}/{self.event.project_name}`\n" "* And now you can install the packages.\n" "\nPlease note that the RPMs should be used only in a testing environment." ) self.event.project.pr_comment(pr_id=self.event.pr_id, body=msg) build_job_helper.report_status_to_build_for_chroot( state=CommitStatus.success, description="RPMs were built successfully.", url=url, chroot=self.event.chroot, ) build_job_helper.report_status_to_test_for_chroot( state=CommitStatus.pending, description="RPMs were built successfully.", url=url, chroot=self.event.chroot, ) build.set_status(PG_COPR_BUILD_STATUS_SUCCESS) if ( build_job_helper.job_tests and self.event.chroot in build_job_helper.tests_targets ): testing_farm_handler = GithubTestingFarmHandler( config=self.config, job_config=build_job_helper.job_tests, event=self.event, chroot=self.event.chroot, ) testing_farm_handler.run() else: logger.debug("Testing farm not in the job config.") return HandlerResults(success=True, details={})
def test_get_all_builds(clean_before_and_after, multiple_copr_builds): builds_list = list(CoprBuildModel.get_all()) assert len(builds_list) == 3 # we just wanna check if result is iterable # order doesn't matter, so all of them are set to pending in supplied data assert builds_list[1].status == "pending"
def build(self): if not self._build: self._build = CoprBuildModel.get_by_build_id( str(self.copr_event.build_id), self.copr_event.chroot) return self._build
def check_copr_build(build_id: int) -> bool: """ Check the copr_build with given id and refresh the status if needed. Used in the babysit task. :param build_id: id of the copr_build (CoprBuildModel.build.id) :return: True if in case of successful run, False when we need to retry """ logger.debug(f"Getting copr build ID {build_id} from DB.") builds = CoprBuildModel.get_all_by_build_id(build_id) if not builds: logger.warning(f"Copr build {build_id} not in DB.") return True copr_client = CoprClient.create_from_config_file() build_copr = copr_client.build_proxy.get(build_id) if not build_copr.ended_on: logger.info("The copr build is still in progress.") return False logger.info(f"The status is {build_copr.state!r}.") for build in builds: if build.status != "pending": logger.info( f"DB state says {build.status!r}, " "things were taken care of already, skipping." ) continue chroot_build = copr_client.build_chroot_proxy.get(build_id, build.target) event = CoprBuildEvent( topic=FedmsgTopic.copr_build_finished.value, build_id=build_id, build=build, chroot=build.target, status=( COPR_API_SUCC_STATE if chroot_build.state == COPR_SUCC_STATE else COPR_API_FAIL_STATE ), owner=build.owner, project_name=build.project_name, pkg=build_copr.source_package.get( "name", "" ), # this seems to be the SRPM name timestamp=chroot_build.ended_on, ) job_configs = get_config_for_handler_kls( handler_kls=CoprBuildEndHandler, event=event, package_config=event.get_package_config(), ) for job_config in job_configs: CoprBuildEndHandler( ServiceConfig.get_service_config(), job_config=job_config, event=event, ).run() return True
def test_copr_build_set_build_logs_url(clean_before_and_after, a_copr_build_for_pr): url = "https://copr.fp.o/logs/12456/build.log" a_copr_build_for_pr.set_build_logs_url(url) assert a_copr_build_for_pr.build_logs_url == url b = CoprBuildModel.get_by_build_id(a_copr_build_for_pr.build_id, TARGET) assert b.build_logs_url == url
def run_copr_build(self) -> HandlerResults: if not (self.job_build or self.job_tests): msg = "No copr_build or tests job defined." # we can't report it to end-user at this stage return HandlerResults(success=False, details={"msg": msg}) self.report_status_to_all( description="Building SRPM ...", state=CommitStatus.pending, # pagure requires "valid url" url="", ) self.create_srpm_if_needed() if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=CommitStatus.failure, description=msg, url=get_srpm_log_url_from_flask(self.srpm_model.id), ) return HandlerResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=CommitStatus.error, description=f"Submit of the build failed: {ex}", ) return HandlerResults(success=False, details={"error": str(ex)}) for chroot in self.build_targets: copr_build = CoprBuildModel.get_or_create( build_id=str(build_id), commit_sha=self.event.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", srpm_build=self.srpm_model, trigger_model=self.event.db_trigger, ) url = get_copr_build_log_url_from_flask(id_=copr_build.id) self.report_status_to_all_for_chroot( state=CommitStatus.pending, description="Starting RPM build...", url=url, chroot=chroot, ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id,), countdown=120, # do the first check in 120s ) return HandlerResults(success=True, details={})
def get_build_logs_by_id(id_): log_service_versions() build = CoprBuildModel.get_by_id(id_) if build: return _get_build_logs_for_build(build) return f"We can't find any info about COPR build {id_}.\n"
def test_get_all_build_id(clean_before_and_after, multiple_copr_builds): builds_list = CoprBuildModel.get_all_by_build_id(str(123456)) assert len(list(builds_list)) == 2 # both should have the same project_name assert builds_list[1].project_name == builds_list[0].project_name assert builds_list[1].project_name == "SomeUser-hello-world-9"
def copr_build_info(id_): log_service_versions() build = CoprBuildModel.get_by_id(id_) if build: return _get_build_info(build, build_description="Copr build") return f"We can't find any info about Copr build {id_}.\n", 404
def test_get_logs(client): chroot = "foo-1-x86_64" state = "success" build_id = 2 project = GitProjectModel() project.namespace = "john-foo" project.repo_name = "bar" pr = PullRequestModel() pr.pr_id = 234 pr.project = project srpm = SRPMBuildModel() srpm.url = "https://some.random.copr.subdomain.org/my_srpm.srpm" c = CoprBuildModel() c.target = chroot c.build_id = str(build_id) c.srpm_build_id = 11 c.status = state c.srpm_build = srpm c.web_url = ( "https://copr.fedorainfracloud.org/coprs/john-foo-bar/john-foo-bar/build/2/" ) c.build_logs_url = "https://localhost:5000/build/2/foo-1-x86_64/logs" c.owner = "packit" c.project_name = "example_project" flexmock(CoprBuildModel).should_receive("get_by_id").and_return(c) flexmock(CoprBuildModel).should_receive("get_project").and_return(project) flexmock(CoprBuildModel).should_receive("job_trigger").and_return( flexmock(get_trigger_object=lambda: pr)) url = "/copr-build/1" logs_url = get_copr_build_info_url_from_flask(1) assert logs_url.endswith(url) resp = client.get(url).data.decode() assert f"srpm-build/{c.srpm_build_id}/logs" in resp assert c.web_url in resp assert c.build_logs_url in resp assert c.target in resp assert "Status: success" in resp assert "You can install" in resp assert "Download SRPM" in resp assert srpm.url in resp
def test_get_all_build_id(clean_before_and_after, multiple_copr_builds): builds_list = list(CoprBuildModel.get_all_by_build_id(str(123456))) assert len(builds_list) == 2 # both should have the same project_name assert builds_list[1].project_name == builds_list[0].project_name assert builds_list[1].project_name == "the-project-name"
class CoprBuildJobHelper(BaseBuildJobHelper): job_type_build = JobType.copr_build job_type_test = JobType.tests status_name_build: str = "rpm-build" status_name_test: str = "testing-farm" def __init__( self, service_config: ServiceConfig, package_config: PackageConfig, project: GitProject, metadata: EventData, db_trigger: AbstractTriggerDbType, job_config: JobConfig, targets_override: Optional[Set[str]] = None, pushgateway: Optional[Pushgateway] = None, ): super().__init__( service_config=service_config, package_config=package_config, project=project, metadata=metadata, db_trigger=db_trigger, job_config=job_config, targets_override=targets_override, pushgateway=pushgateway, ) self.msg_retrigger: str = MSG_RETRIGGER.format( job="build", command="copr-build" if self.job_build else "build", place="pull request", ) @property def default_project_name(self) -> str: """ Project name for copr. * use hostname prefix for non-github service * replace slash in namespace with dash * add `-stg` suffix for the stg app """ service_hostname = parse_git_repo( self.project.service.instance_url).hostname service_prefix = ("" if isinstance(self.project, GithubProject) else f"{service_hostname}-") namespace = self.project.namespace.replace("/", "-") stg = "-stg" if self.service_config.deployment == Deployment.stg else "" # We want to share project between all releases. # More details: https://github.com/packit/packit-service/issues/1044 identifier = "releases" if self.metadata.tag_name else self.metadata.identifier return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}" @property def job_project(self) -> Optional[str]: """ The job definition from the config file. """ if self.job_build and self.job_build.metadata.project: return self.job_build.metadata.project if self.job_tests and self.job_tests.metadata.project: return self.job_tests.metadata.project return self.default_project_name @property def job_owner(self) -> Optional[str]: """ Owner used for the copr build -- search the config or use the copr's config. """ if self.job_build and self.job_build.metadata.owner: return self.job_build.metadata.owner if self.job_tests and self.job_tests.metadata.owner: return self.job_tests.metadata.owner return self.api.copr_helper.copr_client.config.get("username") @property def preserve_project(self) -> Optional[bool]: """ If the project will be preserved or can be removed after 60 days. """ return self.job_build.metadata.preserve_project if self.job_build else None @property def list_on_homepage(self) -> Optional[bool]: """ If the project will be shown on the copr home page. """ return self.job_build.metadata.list_on_homepage if self.job_build else None @property def additional_repos(self) -> Optional[List[str]]: """ Additional repos that will be enable for copr build. """ return self.job_build.metadata.additional_repos if self.job_build else None @property def build_targets_all(self) -> Set[str]: """ Return all valid Copr build targets/chroots from config. """ return get_valid_build_targets(*self.configured_build_targets, default=None) @property def tests_targets_all(self) -> Set[str]: """ Return all valid test targets/chroots from config. """ return get_valid_build_targets(*self.configured_tests_targets, default=None) @property def available_chroots(self) -> Set[str]: """ Returns set of available COPR targets. """ return { *filter( lambda chroot: not chroot.startswith("_"), self.api.copr_helper.get_copr_client().mock_chroot_proxy. get_list().keys(), ) } def get_built_packages(self, build_id: int, chroot: str) -> List: return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages( build_id, chroot).packages def get_build(self, build_id: int): return self.api.copr_helper.copr_client.build_proxy.get(build_id) def monitor_not_submitted_copr_builds(self, number_of_builds: int, reason: str): """ Measure the time it took to set the failed status in case of event (e.g. failed SRPM) that prevents Copr build to be submitted. """ time = measure_time(end=datetime.now(timezone.utc), begin=self.metadata.task_accepted_time) for _ in range(number_of_builds): self.pushgateway.copr_build_not_submitted_time.labels( reason=reason).observe(time) def run_copr_build(self) -> TaskResults: self.report_status_to_all( description="Building SRPM ...", state=BaseCommitStatus.running, # pagure requires "valid url" url="", ) if results := self.create_srpm_if_needed(): return results if not self.srpm_model.success: msg = "SRPM build failed, check the logs for details." self.report_status_to_all( state=BaseCommitStatus.failure, description=msg, url=get_srpm_build_info_url(self.srpm_model.id), ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "srpm_failure") return TaskResults(success=False, details={"msg": msg}) try: build_id, web_url = self.run_build() except Exception as ex: sentry_integration.send_to_sentry(ex) # TODO: Where can we show more info about failure? # TODO: Retry self.report_status_to_all( state=BaseCommitStatus.error, description=f"Submit of the build failed: {ex}", ) self.monitor_not_submitted_copr_builds(len(self.build_targets), "submit_failure") return TaskResults( success=False, details={ "msg": "Submit of the Copr build failed.", "error": str(ex) }, ) unprocessed_chroots = [] for chroot in self.build_targets: if chroot not in self.available_chroots: self.report_status_to_all_for_chroot( state=BaseCommitStatus.error, description=f"Not supported target: {chroot}", url=get_srpm_build_info_url(self.srpm_model.id), chroot=chroot, ) self.monitor_not_submitted_copr_builds(1, "not_supported_target") unprocessed_chroots.append(chroot) continue copr_build = CoprBuildModel.create( build_id=str(build_id), commit_sha=self.metadata.commit_sha, project_name=self.job_project, owner=self.job_owner, web_url=web_url, target=chroot, status="pending", run_model=self.run_model, task_accepted_time=self.metadata.task_accepted_time, ) url = get_copr_build_info_url(id_=copr_build.id) self.report_status_to_all_for_chroot( state=BaseCommitStatus.running, description="Starting RPM build...", url=url, chroot=chroot, ) if unprocessed_chroots: unprocessed = "\n".join(sorted(unprocessed_chroots)) available = "\n".join(sorted(self.available_chroots)) self.status_reporter.comment( body="There are build targets that are not supported by COPR.\n" "<details>\n<summary>Unprocessed build targets</summary>\n\n" f"```\n{unprocessed}\n```\n</details>\n" "<details>\n<summary>Available build targets</summary>\n\n" f"```\n{available}\n```\n</details>", ) # release the hounds! celery_app.send_task( "task.babysit_copr_build", args=(build_id, ), countdown=120, # do the first check in 120s ) return TaskResults(success=True, details={})