def test_koji_build_failed(github_pr_event):
flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return(
flexmock(job_config_trigger_type=JobConfigTriggerType.release))
helper = build_helper(
event=github_pr_event,
metadata=JobMetadataConfig(targets=["bright-future"]),
)
flexmock(koji_build).should_receive("get_all_koji_targets").and_return(
["dark-past", "bright-future"]).once()
flexmock(StatusReporter).should_receive("set_status").with_args(
state=CommitStatus.pending,
description="Building SRPM ...",
check_name="packit-stg/production-build-bright-future",
url="",
).and_return()
srpm_build_url = get_srpm_log_url_from_flask(2)
flexmock(StatusReporter).should_receive("set_status").with_args(
state=CommitStatus.error,
description="Submit of the build failed: some error",
check_name="packit-stg/production-build-bright-future",
url=srpm_build_url,
).and_return()
flexmock(GitProject).should_receive(
"set_commit_status").and_return().never()
flexmock(SRPMBuildModel).should_receive("create").and_return(
SRPMBuildModel(id=2, success=True))
flexmock(KojiBuildModel).should_receive("get_or_create").and_return(
KojiBuildModel(id=1))
flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm")
# koji build
flexmock(sentry_integration).should_receive(
"send_to_sentry").and_return().once()
flexmock(Upstream).should_receive("koji_build").and_raise(
Exception, "some error")
result = helper.run_koji_build()
assert not result["success"]
assert result["details"]["errors"]
assert result["details"]["errors"]["bright-future"] == "some error"
def test_koji_build_failed_kerberos(github_pr_event):
flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return(
flexmock(job_config_trigger_type=JobConfigTriggerType.release))
helper = build_helper(
event=github_pr_event,
metadata=JobMetadataConfig(targets=["bright-future"]),
)
flexmock(koji_build).should_receive("get_all_koji_targets").and_return(
["dark-past", "bright-future"]).never()
flexmock(StatusReporter).should_receive("set_status").with_args(
state=CommitStatus.pending,
description="Building SRPM ...",
check_name="packit-stg/production-build-bright-future",
url="",
).and_return()
flexmock(StatusReporter).should_receive("set_status").with_args(
state=CommitStatus.error,
description=
"Kerberos authentication error: the bad authentication error",
check_name="packit-stg/production-build-bright-future",
url=get_srpm_log_url_from_flask(1),
).and_return()
flexmock(GitProject).should_receive(
"set_commit_status").and_return().never()
flexmock(SRPMBuildModel).should_receive("create").and_return(
SRPMBuildModel(id=1, success=True))
flexmock(KojiBuildModel).should_receive("get_or_create").and_return(
KojiBuildModel(id=1))
flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm")
flexmock(PackitAPI).should_receive("init_kerberos_ticket").and_raise(
PackitCommandFailedError,
"Command failed",
stdout_output="",
stderr_output="the bad authentication error",
)
response = helper.run_koji_build()
assert not response["success"]
assert ("Kerberos authentication error: the bad authentication error" ==
response["details"]["msg"])
def test_copr_and_koji_build_for_one_trigger(clean_before_and_after):
pr1 = PullRequestModel.get_or_create(
pr_id=1,
namespace="the-namespace",
repo_name="the-repo-name",
project_url="https://github.com/the-namespace/the-repo-name",
)
pr1_trigger = JobTriggerModel.get_or_create(
type=JobTriggerModelType.pull_request, trigger_id=pr1.id)
srpm_build = SRPMBuildModel.create("asd\nqwe\n",
success=True,
trigger_model=pr1)
copr_build = CoprBuildModel.get_or_create(
build_id="123456",
commit_sha="687abc76d67d",
project_name="SomeUser-hello-world-9",
owner="packit",
web_url="https://copr.something.somewhere/123456",
target=SampleValues.target,
status="pending",
srpm_build=srpm_build,
trigger_model=pr1,
)
koji_build = KojiBuildModel.get_or_create(
build_id="987654",
commit_sha="687abc76d67d",
web_url="https://copr.something.somewhere/123456",
target=SampleValues.target,
status="pending",
srpm_build=srpm_build,
trigger_model=pr1,
)
assert copr_build in pr1_trigger.copr_builds
assert koji_build in pr1_trigger.koji_builds
assert srpm_build in pr1_trigger.srpm_builds
assert srpm_build.job_trigger.get_trigger_object() == pr1
assert copr_build.job_trigger.get_trigger_object() == pr1
assert koji_build.job_trigger.get_trigger_object() == pr1
def test_koji_build_target_not_supported(github_pr_event):
trigger = flexmock(job_config_trigger_type=JobConfigTriggerType.release, id=123)
flexmock(AddPullRequestDbTrigger).should_receive("db_trigger").and_return(trigger)
helper = build_helper(
event=github_pr_event,
metadata=JobMetadataConfig(targets=["nonexisting-target"]),
db_trigger=trigger,
)
flexmock(koji_build).should_receive("get_all_koji_targets").and_return(
["dark-past", "bright-future"]
).once()
flexmock(StatusReporter).should_receive("set_status").with_args(
state=CommitStatus.pending,
description="Building SRPM ...",
check_name="packit-stg/production-build-nonexisting-target",
url="",
).and_return()
flexmock(StatusReporter).should_receive("set_status").with_args(
state=CommitStatus.error,
description="Target not supported: nonexisting-target",
check_name="packit-stg/production-build-nonexisting-target",
url=get_srpm_log_url_from_flask(1),
).and_return()
flexmock(GitProject).should_receive("set_commit_status").and_return().never()
flexmock(SRPMBuildModel).should_receive("create").and_return(
SRPMBuildModel(id=1, success=True)
)
flexmock(KojiBuildModel).should_receive("get_or_create").and_return(
KojiBuildModel(id=1)
)
flexmock(PackitAPI).should_receive("create_srpm").and_return("my.srpm")
response = helper.run_koji_build()
assert not response["success"]
assert (
"Target not supported: nonexisting-target"
== response["details"]["errors"]["nonexisting-target"]
)
def get(self, id):
"""A specific koji build details. From koji_build hash, filled by worker."""
builds_list = KojiBuildModel.get_all_by_build_id(str(id))
if not builds_list.first():
return response_maker(
{"error": "No info about build stored in DB"},
status=HTTPStatus.NOT_FOUND.value,
)
build = builds_list[0]
build_dict = {
"build_id": build.build_id,
"status": build.status,
"build_start_time": optional_time(build.build_start_time),
"build_finished_time": optional_time(build.build_finished_time),
"build_submitted_time": optional_time(build.build_submitted_time),
"chroot": build.target,
"web_url": build.web_url,
# from old data, sometimes build_logs_url is same and sometimes different to web_url
"build_logs_url": build.build_logs_url,
"pr_id": build.get_pr_id(),
"branch_name": build.get_branch_name(),
"ref": build.commit_sha,
"release": build.get_release_tag(),
}
project = build.get_project()
if project:
build_dict["project_url"] = project.project_url
build_dict["repo_namespace"] = project.namespace
build_dict["repo_name"] = project.repo_name
build_dict[
"srpm_logs"] = build.srpm_build.logs if build.srpm_build else None
return response_maker(build_dict)
def build_model(self) -> Optional[KojiBuildModel]:
if not self._build_model:
self._build_model = KojiBuildModel.get_by_build_id(
build_id=self.build_id)
return self._build_model
def koji_build_info(id_):
log_service_versions()
build = KojiBuildModel.get_by_id(id_)
if build:
return _get_build_info(build, build_description="Koji build")
return f"We can't find any info about Koji build {id_}.\n", 404
def build_model(self) -> Optional[KojiBuildModel]:
if not self._build_model_searched and not self._build_model:
self._build_model = KojiBuildModel.get_by_build_id(
build_id=self.build_id)
self._build_model_searched = True
return self._build_model
def run(self):
build = KojiBuildModel.get_by_build_id(
build_id=str(self.koji_event.build_id))
if not build:
msg = f"Koji build {self.koji_event.build_id} not found in the database."
logger.warning(msg)
return TaskResults(success=False, details={"msg": msg})
logger.debug(
f"Build on {build.target} in koji changed state "
f"from {self.koji_event.old_state} to {self.koji_event.state}.")
build.set_build_start_time(
datetime.utcfromtimestamp(self.koji_event.start_time) if self.
koji_event.start_time else None)
build.set_build_finished_time(
datetime.utcfromtimestamp(self.koji_event.completion_time) if self.
koji_event.completion_time else None)
url = get_koji_build_info_url_from_flask(build.id)
build_job_helper = KojiBuildJobHelper(
service_config=self.service_config,
package_config=self.package_config,
project=self.project,
metadata=self.data,
db_trigger=self.db_trigger,
job_config=self.job_config,
)
if self.koji_event.state == KojiBuildState.open:
build.set_status("pending")
build_job_helper.report_status_to_all_for_chroot(
description="RPM build is in progress...",
state=CommitStatus.pending,
url=url,
chroot=build.target,
)
elif self.koji_event.state == KojiBuildState.closed:
build.set_status("success")
build_job_helper.report_status_to_all_for_chroot(
description="RPMs were built successfully.",
state=CommitStatus.success,
url=url,
chroot=build.target,
)
elif self.koji_event.state == KojiBuildState.failed:
build.set_status("failed")
build_job_helper.report_status_to_all_for_chroot(
description="RPMs failed to be built.",
state=CommitStatus.failure,
url=url,
chroot=build.target,
)
elif self.koji_event.state == KojiBuildState.canceled:
build.set_status("error")
build_job_helper.report_status_to_all_for_chroot(
description="RPMs build was canceled.",
state=CommitStatus.error,
url=url,
chroot=build.target,
)
else:
logger.debug(
f"We don't react to this koji build state change: {self.koji_event.state}"
)
koji_build_logs = self.koji_event.get_koji_build_logs_url()
build.set_build_logs_url(koji_build_logs)
koji_rpm_task_web_url = self.koji_event.get_koji_build_logs_url()
build.set_web_url(koji_rpm_task_web_url)
msg = (f"Build on {build.target} in koji changed state "
f"from {self.koji_event.old_state} to {self.koji_event.state}.")
return TaskResults(success=True, details={"msg": msg})
def build(self) -> Optional[KojiBuildModel]:
if not self._build:
self._build = KojiBuildModel.get_by_build_id(
build_id=str(self.koji_event.build_id))
return self._build
def run_koji_build(self) -> TaskResults:
if not self.is_scratch:
msg = "Non-scratch builds not possible from upstream."
self.report_status_to_all(
description=msg,
state=CommitStatus.error,
url=KOJI_PRODUCTION_BUILDS_ISSUE,
)
return TaskResults(success=True, details={"msg": msg})
self.report_status_to_all(
description="Building SRPM ...", state=CommitStatus.pending
)
self.create_srpm_if_needed()
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
try:
# We need to do it manually
# because we don't use PackitAPI.build, but PackitAPI.up.koji_build
self.api.init_kerberos_ticket()
except PackitCommandFailedError as ex:
msg = f"Kerberos authentication error: {ex.stderr_output}"
logger.error(msg)
self.report_status_to_all(
state=CommitStatus.error,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
errors: Dict[str, str] = {}
for target in self.build_targets:
if target not in self.supported_koji_targets:
msg = f"Target not supported: {target}"
self.report_status_to_all_for_chroot(
state=CommitStatus.error,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
chroot=target,
)
errors[target] = msg
continue
try:
build_id, web_url = self.run_build(target=target)
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all_for_chroot(
state=CommitStatus.error,
description=f"Submit of the build failed: {ex}",
url=get_srpm_log_url_from_flask(self.srpm_model.id),
chroot=target,
)
errors[target] = str(ex)
continue
koji_build = KojiBuildModel.get_or_create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
web_url=web_url,
target=target,
status="pending",
srpm_build=self.srpm_model,
trigger_model=self.db_trigger,
)
url = get_koji_build_info_url_from_flask(id_=koji_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Building RPM ...",
url=url,
chroot=target,
)
if errors:
return TaskResults(
success=False,
details={
"msg": "Koji build submit was not successful for all chroots.",
"errors": errors,
},
)
# TODO: release the hounds!
"""
celery_app.send_task(
"task.babysit_koji_build",
args=(build_metadata.build_id,),
countdown=120, # do the first check in 120s
)
"""
return TaskResults(success=True, details={})
class KojiBuildJobHelper(BaseBuildJobHelper):
job_type_build = JobType.production_build
job_type_test = None
status_name_build: str = "production-build"
status_name_test: str = None
def __init__(
self,
service_config: ServiceConfig,
package_config: PackageConfig,
project: GitProject,
metadata: EventData,
db_trigger,
job_config: JobConfig,
targets_override: Optional[Set[str]] = None,
):
super().__init__(
service_config=service_config,
package_config=package_config,
project=project,
metadata=metadata,
db_trigger=db_trigger,
job_config=job_config,
targets_override=targets_override,
)
self.msg_retrigger: str = MSG_RETRIGGER.format(
job="build", command="production-build", place="pull request"
)
# Lazy properties
self._supported_koji_targets = None
@property
def is_scratch(self) -> bool:
return self.job_build and self.job_build.metadata.scratch
@property
def build_targets_all(self) -> Set[str]:
"""
Return all valid Koji targets/chroots from config.
"""
return get_koji_targets(*self.configured_build_targets)
@property
def tests_targets_all(self) -> Set[str]:
"""
[not used now]
Return all valid test targets/chroots from config.
"""
return get_koji_targets(*self.configured_tests_targets)
@property
def supported_koji_targets(self):
if self._supported_koji_targets is None:
self._supported_koji_targets = get_all_koji_targets()
return self._supported_koji_targets
def run_koji_build(self) -> TaskResults:
self.report_status_to_all(
description="Building SRPM ...", state=BaseCommitStatus.running
)
if results := self.create_srpm_if_needed():
return results
if self.srpm_model.status != PG_BUILD_STATUS_SUCCESS:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=BaseCommitStatus.failure,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
try:
# We need to do it manually
# because we don't use PackitAPI.build, but PackitAPI.up.koji_build
self.api.init_kerberos_ticket()
except PackitCommandFailedError as ex:
msg = f"Kerberos authentication error: {ex.stderr_output}"
logger.error(msg)
self.report_status_to_all(
state=BaseCommitStatus.error,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
errors: Dict[str, str] = {}
for target in self.build_targets:
if target not in self.supported_koji_targets:
msg = f"Target not supported: {target}"
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=target,
)
errors[target] = msg
continue
try:
build_id, web_url = self.run_build(target=target)
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=f"Submit of the build failed: {ex}",
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=target,
)
errors[target] = str(ex)
continue
koji_build = KojiBuildModel.create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
web_url=web_url,
target=target,
status="pending",
run_model=self.run_model,
)
url = get_koji_build_info_url(id_=koji_build.id)
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.running,
description="Building RPM ...",
url=url,
chroot=target,
)
if errors:
return TaskResults(
success=False,
details={
"msg": "Koji build submit was not successful for all chroots.",
"errors": errors,
},
)
# TODO: release the hounds!
"""
celery_app.send_task(
"task.babysit_koji_build",
args=(build_metadata.build_id,),
countdown=120, # do the first check in 120s
)
"""
return TaskResults(success=True, details={})
def test_koji_build_set_status(clean_before_and_after, a_koji_build):
assert a_koji_build.status == "pending"
a_koji_build.set_status("awesome")
assert a_koji_build.status == "awesome"
b = KojiBuildModel.get_by_build_id(a_koji_build.build_id, TARGET)
assert b.status == "awesome"
def run(self):
build = KojiBuildModel.get_by_build_id(
build_id=str(self.koji_task_event.build_id)
)
if not build:
msg = (
f"Koji build {self.koji_task_event.build_id} not found in the database."
)
logger.warning(msg)
return TaskResults(success=False, details={"msg": msg})
logger.debug(
f"Build on {build.target} in koji changed state "
f"from {self.koji_task_event.old_state} to {self.koji_task_event.state}."
)
build.set_build_start_time(
datetime.utcfromtimestamp(self.koji_task_event.start_time)
if self.koji_task_event.start_time
else None
)
build.set_build_finished_time(
datetime.utcfromtimestamp(self.koji_task_event.completion_time)
if self.koji_task_event.completion_time
else None
)
url = get_koji_build_info_url(build.id)
build_job_helper = KojiBuildJobHelper(
service_config=self.service_config,
package_config=self.package_config,
project=self.project,
metadata=self.data,
db_trigger=self.db_trigger,
job_config=self.job_config,
)
new_commit_status = {
KojiTaskState.free: BaseCommitStatus.pending,
KojiTaskState.open: BaseCommitStatus.running,
KojiTaskState.closed: BaseCommitStatus.success,
KojiTaskState.canceled: BaseCommitStatus.error,
KojiTaskState.assigned: None,
KojiTaskState.failed: BaseCommitStatus.failure,
}.get(self.koji_task_event.state)
description = {
KojiTaskState.free: "RPM build has been submitted...",
KojiTaskState.open: "RPM build is in progress...",
KojiTaskState.closed: "RPM build succeeded.",
KojiTaskState.canceled: "RPM build was canceled.",
KojiTaskState.assigned: None,
KojiTaskState.failed: "RPM build failed.",
}.get(self.koji_task_event.state)
if not (new_commit_status and description):
logger.debug(
f"We don't react to this koji build state change: {self.koji_task_event.state}"
)
else:
build.set_status(new_commit_status.value)
build_job_helper.report_status_to_all_for_chroot(
description=description,
state=new_commit_status,
url=url,
chroot=build.target,
)
koji_build_logs = KojiTaskEvent.get_koji_build_logs_url(
rpm_build_task_id=int(build.build_id),
koji_logs_url=self.service_config.koji_logs_url,
)
build.set_build_logs_url(koji_build_logs)
koji_rpm_task_web_url = KojiTaskEvent.get_koji_rpm_build_web_url(
rpm_build_task_id=int(build.build_id),
koji_web_url=self.service_config.koji_web_url,
)
build.set_web_url(koji_rpm_task_web_url)
msg = (
f"Build on {build.target} in koji changed state "
f"from {self.koji_task_event.old_state} to {self.koji_task_event.state}."
)
return TaskResults(success=True, details={"msg": msg})