def _create_srpm(self):
# we want to get packit logs from the SRPM creation process
# so we stuff them into a StringIO buffer
stream = StringIO()
handler = logging.StreamHandler(stream)
packit_logger = logging.getLogger("packit")
packit_logger.setLevel(logging.DEBUG)
packit_logger.addHandler(handler)
formatter = PackitFormatter(None, "%H:%M:%S")
handler.setFormatter(formatter)
srpm_success = True
exception: Optional[Exception] = None
extra_logs: str = ""
try:
self._srpm_path = Path(
self.api.create_srpm(srpm_dir=self.api.up.local_project.working_dir)
)
except SandcastleTimeoutReached as ex:
exception = ex
extra_logs = f"\nYou have reached 10-minute timeout while creating SRPM.\n"
except ApiException as ex:
exception = ex
# this is an internal error: let's not expose anything to public
extra_logs = (
"\nThere was a problem in the environment the packit-service is running in.\n"
"Please hang tight, the help is coming."
)
except Exception as ex:
exception = ex
# collect the logs now
packit_logger.removeHandler(handler)
stream.seek(0)
srpm_logs = stream.read()
if exception:
logger.info(f"exception while running SRPM build: {exception}")
logger.debug(f"{exception!r}")
srpm_success = False
# when do we NOT want to send stuff to sentry?
sentry_integration.send_to_sentry(exception)
# this needs to be done AFTER we gather logs
# so that extra logs are after actual logs
srpm_logs += extra_logs
if hasattr(exception, "output"):
output = getattr(exception, "output", "") # mypy
srpm_logs += f"\nOutput of the command in the sandbox:\n{output}\n"
srpm_logs += (
f"\nMessage: {exception}\nException: {exception!r}\n{self.msg_retrigger}"
"\nPlease join the freenode IRC channel #packit for the latest info.\n"
)
self._srpm_model = SRPMBuildModel.create(logs=srpm_logs, success=srpm_success)
def run(self) -> TaskResults:
"""
Sync the upstream release to dist-git as a pull request.
"""
self.local_project = LocalProject(
git_project=self.project,
working_dir=self.service_config.command_handler_work_dir,
)
self.api = PackitAPI(self.service_config, self.job_config,
self.local_project)
errors = {}
for branch in get_branches(*self.job_config.metadata.dist_git_branches,
default="master"):
try:
self.api.sync_release(dist_git_branch=branch,
version=self.data.tag_name)
except Exception as ex:
sentry_integration.send_to_sentry(ex)
errors[branch] = str(ex)
if errors:
branch_errors = ""
for branch, err in sorted(
errors.items(), key=lambda branch_error: branch_error[0]):
err_without_new_lines = err.replace("\n", " ")
branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n"
body_msg = (
f"Packit failed on creating pull-requests in dist-git:\n\n"
f"| dist-git branch | error |\n"
f"| --------------- | ----- |\n"
f"{branch_errors}\n\n"
"You can re-trigger the update by adding `/packit propose-update`"
" to the issue comment.\n")
self.project.create_issue(
title=
f"[packit] Propose update failed for release {self.data.tag_name}",
body=body_msg,
)
return TaskResults(
success=False,
details={
"msg": "Propose update failed.",
"errors": errors
},
)
return TaskResults(success=True, details={})
def handle_build_submit_error(self, ex):
"""
Handle errors when submitting Copr build.
"""
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=BaseCommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
self.monitor_not_submitted_copr_builds(
len(self.build_targets), "submit_failure"
)
def run(self) -> TaskResults:
"""
Sync the upstream release to dist-git as a pull request.
"""
self.local_project = LocalProject(
git_project=self.project,
working_dir=self.service_config.command_handler_work_dir,
)
self.api = PackitAPI(self.service_config, self.job_config,
self.local_project)
errors = {}
default_dg_branch = self.api.dg.local_project.git_project.default_branch
for branch in get_branches(*self.job_config.metadata.dist_git_branches,
default=default_dg_branch):
try:
self.api.sync_release(dist_git_branch=branch,
tag=self.data.tag_name)
except Exception as ex:
# the archive has not been uploaded to PyPI yet
if FILE_DOWNLOAD_FAILURE in str(ex):
# retry for the archive to become available
logger.info(
f"We were not able to download the archive: {ex}")
# when the task hits max_retries, it raises MaxRetriesExceededError
# and the error handling code would be never executed
retries = self.task.request.retries
if retries < int(
getenv("CELERY_RETRY_LIMIT", DEFAULT_RETRY_LIMIT)):
logger.info(f"Retrying for the {retries + 1}. time...")
# throw=False so that exception is not raised and task
# is not retried also automatically
self.task.retry(exc=ex,
countdown=15 * 2**retries,
throw=False)
return TaskResults(
success=False,
details={
"msg":
"Task was retried, we were not able to download the archive."
},
)
sentry_integration.send_to_sentry(ex)
errors[branch] = str(ex)
if errors:
branch_errors = ""
for branch, err in sorted(
errors.items(), key=lambda branch_error: branch_error[0]):
err_without_new_lines = err.replace("\n", " ")
branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n"
msg_retrigger = MSG_RETRIGGER.format(job="update",
command="propose-downstream",
place="issue")
body_msg = (
f"Packit failed on creating pull-requests in dist-git:\n\n"
f"| dist-git branch | error |\n"
f"| --------------- | ----- |\n"
f"{branch_errors}\n\n"
f"{msg_retrigger}\n")
self.project.create_issue(
title=
f"[packit] Propose downstream failed for release {self.data.tag_name}",
body=body_msg,
)
return TaskResults(
success=False,
details={
"msg": "Propose downstream failed.",
"errors": errors
},
)
return TaskResults(success=True, details={})
def run(self) -> TaskResults:
"""
Sync the upstream release to dist-git as a pull request.
"""
self.local_project = LocalProject(
git_project=self.project,
working_dir=self.service_config.command_handler_work_dir,
)
self.api = PackitAPI(self.service_config, self.job_config,
self.local_project)
errors = {}
for branch in get_branches(*self.job_config.metadata.dist_git_branches,
default="master"):
try:
self.api.sync_release(dist_git_branch=branch,
version=self.data.tag_name)
except Exception as ex:
# the archive has not been uploaded to PyPI yet
if FILE_DOWNLOAD_FAILURE in str(ex):
# retry for the archive to become available
logger.info(
f"We were not able to download the archive: {ex}")
# when the task hits max_retries, it raises MaxRetriesExceededError
# and the error handling code would be never executed
retries = self.task.request.retries
if retries < RETRY_LIMIT:
logger.info(f"Retrying for the {retries + 1}. time...")
self.task.retry(exc=ex, countdown=15 * 2**retries)
sentry_integration.send_to_sentry(ex)
errors[branch] = str(ex)
if errors:
branch_errors = ""
for branch, err in sorted(
errors.items(), key=lambda branch_error: branch_error[0]):
err_without_new_lines = err.replace("\n", " ")
branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n"
body_msg = (
f"Packit failed on creating pull-requests in dist-git:\n\n"
f"| dist-git branch | error |\n"
f"| --------------- | ----- |\n"
f"{branch_errors}\n\n"
"You can re-trigger the update by adding `/packit propose-update`"
" to the issue comment.\n")
self.project.create_issue(
title=
f"[packit] Propose update failed for release {self.data.tag_name}",
body=body_msg,
)
return TaskResults(
success=False,
details={
"msg": "Propose update failed.",
"errors": errors
},
)
return TaskResults(success=True, details={})
def run_testing_farm(self, build_id: int, chroot: str) -> TaskResults:
if chroot not in self.tests_targets:
# Leaving here just to be sure that we will discover this situation if it occurs.
# Currently not possible to trigger this situation.
msg = f"Target '{chroot}' not defined for tests but triggered."
logger.error(msg)
send_to_sentry(PackitConfigException(msg))
return TaskResults(
success=False,
details={"msg": msg},
)
if chroot not in self.build_targets:
self.report_missing_build_chroot(chroot)
return TaskResults(
success=False,
details={
"msg": f"Target '{chroot}' not defined for build. "
"Cannot run tests without build."
},
)
self.report_status_to_test_for_chroot(
state=CommitStatus.pending,
description="Build succeeded. Submitting the tests ...",
chroot=chroot,
)
logger.info("Sending testing farm request...")
if self.is_fmf_configured():
payload = self._payload(build_id, chroot)
else:
payload = self._payload_install_test(build_id, chroot)
endpoint = "requests"
logger.debug(f"POSTing {payload} to {self.tft_api_url}{endpoint}")
req = self.send_testing_farm_request(
endpoint=endpoint,
method="POST",
data=payload,
)
logger.debug(f"Request sent: {req}")
if not req:
msg = "Failed to post request to testing farm API."
logger.debug("Failed to post request to testing farm API.")
self.report_status_to_test_for_chroot(
state=CommitStatus.error,
description=msg,
chroot=chroot,
)
return TaskResults(success=False, details={"msg": msg})
# success set check on pending
if req.status_code != 200:
# something went wrong
if req.json() and "message" in req.json():
msg = req.json()["message"]
else:
msg = f"Failed to submit tests: {req.reason}"
logger.error(msg)
self.report_status_to_test_for_chroot(
state=CommitStatus.failure,
description=msg,
chroot=chroot,
)
return TaskResults(success=False, details={"msg": msg})
# Response: {"id": "9fa3cbd1-83f2-4326-a118-aad59f5", ...}
pipeline_id = req.json()["id"]
logger.debug(
f"Submitted ({req.status_code}) to testing farm as request {pipeline_id}"
)
TFTTestRunModel.create(
pipeline_id=pipeline_id,
commit_sha=self.metadata.commit_sha,
status=TestingFarmResult.new,
target=chroot,
web_url=None,
trigger_model=self.db_trigger,
# In _payload() we ask TF to test commit_sha of fork (PR's source).
# Store original url. If this proves to work, make it a separate column.
data={"base_project_url": self.project.get_web_url()},
)
self.report_status_to_test_for_chroot(
state=CommitStatus.pending,
description="Tests have been submitted ...",
url=f"{self.tft_api_url}requests/{pipeline_id}",
chroot=chroot,
)
return TaskResults(success=True, details={})
def run_testing_farm(self, chroot: str) -> TaskResults:
if chroot not in self.tests_targets:
# Leaving here just to be sure that we will discover this situation if it occurs.
# Currently not possible to trigger this situation.
msg = f"Target '{chroot}' not defined for tests but triggered."
logger.error(msg)
send_to_sentry(PackitConfigException(msg))
return TaskResults(
success=False,
details={"msg": msg},
)
if chroot not in self.build_targets:
self.report_missing_build_chroot(chroot)
return TaskResults(
success=False,
details={
"msg":
f"Target '{chroot}' not defined for build. "
"Cannot run tests without build."
},
)
self.report_status_to_test_for_chroot(
state=CommitStatus.pending,
description="Build succeeded. Submitting the tests ...",
chroot=chroot,
)
pipeline_id = str(uuid.uuid4())
logger.debug(f"Pipeline id: {pipeline_id}")
test_run_model = TFTTestRunModel.create(
pipeline_id=pipeline_id,
commit_sha=self.metadata.commit_sha,
status=TestingFarmResult.new,
target=chroot,
web_url=None,
trigger_model=self.db_trigger,
)
logger.debug("Sending testing farm request...")
payload = self._trigger_payload(pipeline_id, chroot)
logger.debug(f"Payload: {payload}")
req = self.send_testing_farm_request(TESTING_FARM_TRIGGER_URL, "POST",
{}, json.dumps(payload))
logger.debug(f"Request sent: {req}")
if not req:
msg = "Failed to post request to testing farm API."
logger.debug("Failed to post request to testing farm API.")
self.report_status_to_test_for_chroot(
state=CommitStatus.error,
description=msg,
chroot=chroot,
)
test_run_model.set_status(TestingFarmResult.error)
return TaskResults(success=False, details={"msg": msg})
else:
logger.debug(
f"Submitted to testing farm with return code: {req.status_code}"
)
"""
Response:
{
"id": "9fa3cbd1-83f2-4326-a118-aad59f5",
"success": true,
"url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<id>"
}
"""
# success set check on pending
if req.status_code != 200:
# something went wrong
if req.json() and "message" in req.json():
msg = req.json()["message"]
else:
msg = f"Failed to submit tests: {req.reason}"
logger.error(msg)
self.report_status_to_test_for_chroot(
state=CommitStatus.failure,
description=msg,
chroot=chroot,
)
test_run_model.set_status(TestingFarmResult.error)
return TaskResults(success=False, details={"msg": msg})
test_run_model.set_status(TestingFarmResult.running)
self.report_status_to_test_for_chroot(
state=CommitStatus.pending,
description="Tests are running ...",
url=req.json()["url"],
chroot=chroot,
)
return TaskResults(success=True, details={})
def _create_srpm(self):
"""
Create SRPM.
Returns:
Task results if job is done because of merge conflicts, `None`
otherwise.
"""
# we want to get packit logs from the SRPM creation process
# so we stuff them into a StringIO buffer
stream = StringIO()
handler = logging.StreamHandler(stream)
packit_logger = logging.getLogger("packit")
packit_logger.setLevel(logging.DEBUG)
packit_logger.addHandler(handler)
formatter = PackitFormatter()
handler.setFormatter(formatter)
srpm_success = True
exception: Optional[Exception] = None
extra_logs: str = ""
results: Optional[TaskResults] = None
try:
self._srpm_path = Path(
self.api.create_srpm(
srpm_dir=self.api.up.local_project.working_dir))
except SandcastleTimeoutReached as ex:
exception = ex
extra_logs = "\nYou have reached 10-minute timeout while creating SRPM.\n"
except ApiException as ex:
exception = ex
# this is an internal error: let's not expose anything to public
extra_logs = (
"\nThere was a problem in the environment the packit-service is running in.\n"
"Please hang tight, the help is coming.")
except PackitMergeException as ex:
exception = ex
results = TaskResults(
success=True,
details={
"msg": "Merge conflicts were detected, cannot build SRPM.",
"exception": str(ex),
},
)
except Exception as ex:
exception = ex
# collect the logs now
packit_logger.removeHandler(handler)
stream.seek(0)
srpm_logs = stream.read()
if exception:
logger.info(f"exception while running SRPM build: {exception}")
logger.debug(f"{exception!r}")
srpm_success = False
# when do we NOT want to send stuff to sentry?
if not isinstance(exception, PackitMergeException):
sentry_integration.send_to_sentry(exception)
# this needs to be done AFTER we gather logs
# so that extra logs are after actual logs
srpm_logs += extra_logs
if hasattr(exception, "output"):
output = getattr(exception, "output", "") # mypy
srpm_logs += f"\nOutput of the command in the sandbox:\n{output}\n"
srpm_logs += (
f"\nMessage: {exception}\nException: {exception!r}\n{self.msg_retrigger}"
"\nPlease join #packit on irc.libera.chat if you need help with the error above.\n"
)
self._srpm_model, self.run_model = SRPMBuildModel.create_with_new_run(
logs=srpm_logs,
success=srpm_success,
trigger_model=self.db_trigger,
)
return results
def _run_copr_build_and_save_output(self) -> BuildMetadata:
# we want to get packit logs from the SRPM creation process
# so we stuff them into a StringIO buffer
stream = StringIO()
handler = logging.StreamHandler(stream)
packit_logger = logging.getLogger("packit")
packit_logger.setLevel(logging.DEBUG)
packit_logger.addHandler(handler)
formatter = PackitFormatter(None, "%H:%M:%S")
handler.setFormatter(formatter)
c = BuildMetadata()
c.srpm_failed = False
ex: Optional[Exception] = None # shut up pycharm
extra_logs: str = ""
try:
c.copr_build_id, c.copr_web_url = self.api.run_copr_build(
project=self.job_project,
chroots=self.build_chroots,
owner=self.job_owner,
)
except SandcastleTimeoutReached as e:
ex = e
extra_logs = f"\nYou have reached 10-minute timeout while creating SRPM.\n"
except ApiException as e:
ex = e
# this is an internal error: let's not expose anything to public
extra_logs = (
"\nThere was a problem in the environment the packit-service is running in.\n"
"Please hang tight, the help is coming.")
except Exception as e:
ex = e # shut up mypy
# collect the logs now
packit_logger.removeHandler(handler)
stream.seek(0)
c.srpm_logs = stream.read()
if ex:
logger.info(f"exception while running a copr build: {ex}")
logger.debug(f"{ex!r}")
c.srpm_failed = True
# when do we NOT want to send stuff to sentry?
sentry_integration.send_to_sentry(ex)
# this needs to be done AFTER we gather logs
# so that extra logs are after actual logs
c.srpm_logs += extra_logs
if hasattr(ex, "output"):
output = getattr(ex, "output", "") # mypy
c.srpm_logs += f"\nOutput of the command in the sandbox:\n{output}\n"
c.srpm_logs += (
f"\nMessage: {ex}\nException: {ex!r}\n{self.msg_retrigger}"
"\nPlease join the freenode IRC channel #packit for the latest info.\n"
)
return c
def run_koji_build(self) -> TaskResults:
if not self.is_scratch:
msg = "Non-scratch builds not possible from upstream."
self.report_status_to_all(
description=msg,
state=CommitStatus.error,
url=KOJI_PRODUCTION_BUILDS_ISSUE,
)
return TaskResults(success=True, details={"msg": msg})
self.report_status_to_all(
description="Building SRPM ...", state=CommitStatus.pending
)
self.create_srpm_if_needed()
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
try:
# We need to do it manually
# because we don't use PackitAPI.build, but PackitAPI.up.koji_build
self.api.init_kerberos_ticket()
except PackitCommandFailedError as ex:
msg = f"Kerberos authentication error: {ex.stderr_output}"
logger.error(msg)
self.report_status_to_all(
state=CommitStatus.error,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
errors: Dict[str, str] = {}
for target in self.build_targets:
if target not in self.supported_koji_targets:
msg = f"Target not supported: {target}"
self.report_status_to_all_for_chroot(
state=CommitStatus.error,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
chroot=target,
)
errors[target] = msg
continue
try:
build_id, web_url = self.run_build(target=target)
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all_for_chroot(
state=CommitStatus.error,
description=f"Submit of the build failed: {ex}",
url=get_srpm_log_url_from_flask(self.srpm_model.id),
chroot=target,
)
errors[target] = str(ex)
continue
koji_build = KojiBuildModel.get_or_create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
web_url=web_url,
target=target,
status="pending",
srpm_build=self.srpm_model,
trigger_model=self.db_trigger,
)
url = get_koji_build_info_url_from_flask(id_=koji_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Building RPM ...",
url=url,
chroot=target,
)
if errors:
return TaskResults(
success=False,
details={
"msg": "Koji build submit was not successful for all chroots.",
"errors": errors,
},
)
# TODO: release the hounds!
"""
celery_app.send_task(
"task.babysit_koji_build",
args=(build_metadata.build_id,),
countdown=120, # do the first check in 120s
)
"""
return TaskResults(success=True, details={})
def run_copr_build(self) -> HandlerResults:
if not (self.job_build or self.job_tests):
msg = "No copr_build or tests job defined."
# we can't report it to end-user at this stage
return HandlerResults(success=False, details={"msg": msg})
self.report_status_to_all(
description="Building SRPM ...",
state=CommitStatus.pending,
# pagure requires "valid url"
url="",
)
self.create_srpm_if_needed()
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return HandlerResults(success=False, details={"msg": msg})
try:
build_id, web_url = self.run_build()
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=CommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
return HandlerResults(success=False, details={"error": str(ex)})
for chroot in self.build_targets:
copr_build = CoprBuildModel.get_or_create(
build_id=str(build_id),
commit_sha=self.event.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="pending",
srpm_build=self.srpm_model,
trigger_model=self.event.db_trigger,
)
url = get_copr_build_log_url_from_flask(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id,),
countdown=120, # do the first check in 120s
)
return HandlerResults(success=True, details={})
class KojiBuildJobHelper(BaseBuildJobHelper):
job_type_build = JobType.production_build
job_type_test = None
status_name_build: str = "production-build"
status_name_test: str = None
def __init__(
self,
service_config: ServiceConfig,
package_config: PackageConfig,
project: GitProject,
metadata: EventData,
db_trigger,
job_config: JobConfig,
targets_override: Optional[Set[str]] = None,
):
super().__init__(
service_config=service_config,
package_config=package_config,
project=project,
metadata=metadata,
db_trigger=db_trigger,
job_config=job_config,
targets_override=targets_override,
)
self.msg_retrigger: str = MSG_RETRIGGER.format(
job="build", command="production-build", place="pull request"
)
# Lazy properties
self._supported_koji_targets = None
@property
def is_scratch(self) -> bool:
return self.job_build and self.job_build.metadata.scratch
@property
def build_targets_all(self) -> Set[str]:
"""
Return all valid Koji targets/chroots from config.
"""
return get_koji_targets(*self.configured_build_targets)
@property
def tests_targets_all(self) -> Set[str]:
"""
[not used now]
Return all valid test targets/chroots from config.
"""
return get_koji_targets(*self.configured_tests_targets)
@property
def supported_koji_targets(self):
if self._supported_koji_targets is None:
self._supported_koji_targets = get_all_koji_targets()
return self._supported_koji_targets
def run_koji_build(self) -> TaskResults:
self.report_status_to_all(
description="Building SRPM ...", state=BaseCommitStatus.running
)
if results := self.create_srpm_if_needed():
return results
if self.srpm_model.status != PG_BUILD_STATUS_SUCCESS:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=BaseCommitStatus.failure,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
try:
# We need to do it manually
# because we don't use PackitAPI.build, but PackitAPI.up.koji_build
self.api.init_kerberos_ticket()
except PackitCommandFailedError as ex:
msg = f"Kerberos authentication error: {ex.stderr_output}"
logger.error(msg)
self.report_status_to_all(
state=BaseCommitStatus.error,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
errors: Dict[str, str] = {}
for target in self.build_targets:
if target not in self.supported_koji_targets:
msg = f"Target not supported: {target}"
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=target,
)
errors[target] = msg
continue
try:
build_id, web_url = self.run_build(target=target)
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=f"Submit of the build failed: {ex}",
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=target,
)
errors[target] = str(ex)
continue
koji_build = KojiBuildModel.create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
web_url=web_url,
target=target,
status="pending",
run_model=self.run_model,
)
url = get_koji_build_info_url(id_=koji_build.id)
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.running,
description="Building RPM ...",
url=url,
chroot=target,
)
if errors:
return TaskResults(
success=False,
details={
"msg": "Koji build submit was not successful for all chroots.",
"errors": errors,
},
)
# TODO: release the hounds!
"""
celery_app.send_task(
"task.babysit_koji_build",
args=(build_metadata.build_id,),
countdown=120, # do the first check in 120s
)
"""
return TaskResults(success=True, details={})
def run_testing_farm(self, chroot: str) -> HandlerResults:
if chroot not in self.tests_chroots:
# Leaving here just to be sure that we will discover this situation if it occurs.
# Currently not possible to trigger this situation.
msg = f"Target '{chroot}' not defined for tests but triggered."
logger.error(msg)
send_to_sentry(PackitConfigException(msg))
return HandlerResults(
success=False,
details={"msg": msg},
)
if chroot not in self.build_chroots:
self.report_missing_build_chroot(chroot)
return HandlerResults(
success=False,
details={
"msg":
f"Target '{chroot}' not defined for build. "
f"Cannot run tests without build."
},
)
self.report_status_to_test_for_chroot(
state=CommitStatus.pending,
description="Build succeeded. Submitting the tests ...",
chroot=chroot,
)
pipeline_id = str(uuid.uuid4())
logger.debug(f"Pipeline id: {pipeline_id}")
payload: dict = {
"pipeline": {
"id": pipeline_id
},
"api": {
"token": self.config.testing_farm_secret
},
"response-url": f"{self.api_url}/testing-farm/results",
"artifact": {
"repo-name":
self.project.repo,
"repo-namespace":
self.project.namespace,
"copr-repo-name":
f"{self.job_owner}/{self.default_project_name}",
"copr-chroot":
chroot,
"commit-sha":
self.event.commit_sha,
"git-url":
self.event.project_url,
"git-ref":
self.event.git_ref
if self.event.git_ref else self.event.commit_sha,
},
}
logger.debug("Sending testing farm request...")
logger.debug(payload)
req = self.send_testing_farm_request(TESTING_FARM_TRIGGER_URL, "POST",
{}, json.dumps(payload))
logger.debug(f"Request sent: {req}")
if not req:
msg = "Failed to post request to testing farm API."
logger.debug("Failed to post request to testing farm API.")
self.report_status_to_test_for_chroot(
state=CommitStatus.failure,
description=msg,
chroot=chroot,
)
return HandlerResults(success=False, details={"msg": msg})
else:
logger.debug(
f"Submitted to testing farm with return code: {req.status_code}"
)
"""
Response:
{
"id": "9fa3cbd1-83f2-4326-a118-aad59f5",
"success": true,
"url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<id>"
}
"""
# success set check on pending
if req.status_code != 200:
# something went wrong
if req.json() and "message" in req.json():
msg = req.json()["message"]
else:
msg = f"Failed to submit tests: {req.reason}"
logger.error(msg)
self.report_status_to_test_for_chroot(
state=CommitStatus.failure,
description=msg,
chroot=chroot,
)
return HandlerResults(success=False, details={"msg": msg})
self.report_status_to_test_for_chroot(
state=CommitStatus.pending,
description="Tests are running ...",
url=req.json()["url"],
chroot=chroot,
)
return HandlerResults(success=True, details={})
class CoprBuildJobHelper(BaseBuildJobHelper):
job_type_build = JobType.copr_build
job_type_test = JobType.tests
status_name_build: str = "rpm-build"
status_name_test: str = "testing-farm"
def __init__(
self,
service_config: ServiceConfig,
package_config: PackageConfig,
project: GitProject,
metadata: EventData,
db_trigger: AbstractTriggerDbType,
job_config: JobConfig,
targets_override: Optional[Set[str]] = None,
pushgateway: Optional[Pushgateway] = None,
):
super().__init__(
service_config=service_config,
package_config=package_config,
project=project,
metadata=metadata,
db_trigger=db_trigger,
job_config=job_config,
targets_override=targets_override,
pushgateway=pushgateway,
)
self.msg_retrigger: str = MSG_RETRIGGER.format(
job="build",
command="copr-build" if self.job_build else "build",
place="pull request",
)
@property
def default_project_name(self) -> str:
"""
Project name for copr.
* use hostname prefix for non-github service
* replace slash in namespace with dash
* add `-stg` suffix for the stg app
"""
service_hostname = parse_git_repo(
self.project.service.instance_url).hostname
service_prefix = ("" if isinstance(self.project, GithubProject) else
f"{service_hostname}-")
namespace = self.project.namespace.replace("/", "-")
stg = "-stg" if self.service_config.deployment == Deployment.stg else ""
# We want to share project between all releases.
# More details: https://github.com/packit/packit-service/issues/1044
identifier = "releases" if self.metadata.tag_name else self.metadata.identifier
return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}"
@property
def job_project(self) -> Optional[str]:
"""
The job definition from the config file.
"""
if self.job_build and self.job_build.metadata.project:
return self.job_build.metadata.project
if self.job_tests and self.job_tests.metadata.project:
return self.job_tests.metadata.project
return self.default_project_name
@property
def job_owner(self) -> Optional[str]:
"""
Owner used for the copr build -- search the config or use the copr's config.
"""
if self.job_build and self.job_build.metadata.owner:
return self.job_build.metadata.owner
if self.job_tests and self.job_tests.metadata.owner:
return self.job_tests.metadata.owner
return self.api.copr_helper.copr_client.config.get("username")
@property
def preserve_project(self) -> Optional[bool]:
"""
If the project will be preserved or can be removed after 60 days.
"""
return self.job_build.metadata.preserve_project if self.job_build else None
@property
def list_on_homepage(self) -> Optional[bool]:
"""
If the project will be shown on the copr home page.
"""
return self.job_build.metadata.list_on_homepage if self.job_build else None
@property
def additional_repos(self) -> Optional[List[str]]:
"""
Additional repos that will be enable for copr build.
"""
return self.job_build.metadata.additional_repos if self.job_build else None
@property
def build_targets_all(self) -> Set[str]:
"""
Return all valid Copr build targets/chroots from config.
"""
return get_valid_build_targets(*self.configured_build_targets,
default=None)
@property
def tests_targets_all(self) -> Set[str]:
"""
Return all valid test targets/chroots from config.
"""
return get_valid_build_targets(*self.configured_tests_targets,
default=None)
@property
def available_chroots(self) -> Set[str]:
"""
Returns set of available COPR targets.
"""
return {
*filter(
lambda chroot: not chroot.startswith("_"),
self.api.copr_helper.get_copr_client().mock_chroot_proxy.
get_list().keys(),
)
}
def get_built_packages(self, build_id: int, chroot: str) -> List:
return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages(
build_id, chroot).packages
def get_build(self, build_id: int):
return self.api.copr_helper.copr_client.build_proxy.get(build_id)
def monitor_not_submitted_copr_builds(self, number_of_builds: int,
reason: str):
"""
Measure the time it took to set the failed status in case of event (e.g. failed SRPM)
that prevents Copr build to be submitted.
"""
time = measure_time(end=datetime.now(timezone.utc),
begin=self.metadata.task_accepted_time)
for _ in range(number_of_builds):
self.pushgateway.copr_build_not_submitted_time.labels(
reason=reason).observe(time)
def run_copr_build(self) -> TaskResults:
self.report_status_to_all(
description="Building SRPM ...",
state=BaseCommitStatus.running,
# pagure requires "valid url"
url="",
)
if results := self.create_srpm_if_needed():
return results
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=BaseCommitStatus.failure,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
)
self.monitor_not_submitted_copr_builds(len(self.build_targets),
"srpm_failure")
return TaskResults(success=False, details={"msg": msg})
try:
build_id, web_url = self.run_build()
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=BaseCommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
self.monitor_not_submitted_copr_builds(len(self.build_targets),
"submit_failure")
return TaskResults(
success=False,
details={
"msg": "Submit of the Copr build failed.",
"error": str(ex)
},
)
unprocessed_chroots = []
for chroot in self.build_targets:
if chroot not in self.available_chroots:
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=f"Not supported target: {chroot}",
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=chroot,
)
self.monitor_not_submitted_copr_builds(1,
"not_supported_target")
unprocessed_chroots.append(chroot)
continue
copr_build = CoprBuildModel.create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="pending",
run_model=self.run_model,
task_accepted_time=self.metadata.task_accepted_time,
)
url = get_copr_build_info_url(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.running,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
if unprocessed_chroots:
unprocessed = "\n".join(sorted(unprocessed_chroots))
available = "\n".join(sorted(self.available_chroots))
self.status_reporter.comment(
body="There are build targets that are not supported by COPR.\n"
"<details>\n<summary>Unprocessed build targets</summary>\n\n"
f"```\n{unprocessed}\n```\n</details>\n"
"<details>\n<summary>Available build targets</summary>\n\n"
f"```\n{available}\n```\n</details>", )
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id, ),
countdown=120, # do the first check in 120s
)
return TaskResults(success=True, details={})
def run_testing_farm(self, chroot: str,
build: Optional["CoprBuildModel"]) -> TaskResults:
if chroot not in self.tests_targets:
# Leaving here just to be sure that we will discover this situation if it occurs.
# Currently not possible to trigger this situation.
msg = f"Target '{chroot}' not defined for tests but triggered."
logger.error(msg)
send_to_sentry(PackitConfigException(msg))
return TaskResults(
success=False,
details={"msg": msg},
)
if not self.skip_build and chroot not in self.build_targets:
self.report_missing_build_chroot(chroot)
return TaskResults(
success=False,
details={
"msg":
f"Target '{chroot}' not defined for build. "
"Cannot run tests without build."
},
)
if (self.job_config.metadata.use_internal_tf and
f"{self.project.service.hostname}/{self.project.full_repo_name}"
not in self.service_config.enabled_projects_for_internal_tf):
self.report_status_to_test_for_chroot(
state=BaseCommitStatus.neutral,
description=
"Internal TF not allowed for this project. Let us know.",
chroot=chroot,
url="https://packit.dev/#contact",
)
return TaskResults(
success=True,
details={"msg": "Project not allowed to use internal TF."},
)
self.report_status_to_test_for_chroot(
state=BaseCommitStatus.running,
description=f"{'Build succeeded. ' if not self.skip_build else ''}"
f"Submitting the tests ...",
chroot=chroot,
)
logger.info("Sending testing farm request...")
if self.is_fmf_configured():
artifact = (self._artifact(chroot, int(build.build_id),
build.built_packages)
if not self.skip_build else None)
payload = self._payload(chroot, artifact, build)
elif not self.is_fmf_configured() and not self.skip_build:
payload = self._payload_install_test(int(build.build_id), chroot)
else:
return TaskResults(
success=True,
details={"msg": "No actions for TestingFarmHandler."})
endpoint = "requests"
logger.debug(f"POSTing {payload} to {self.tft_api_url}{endpoint}")
req = self.send_testing_farm_request(
endpoint=endpoint,
method="POST",
data=payload,
)
logger.debug(f"Request sent: {req}")
if not req:
msg = "Failed to post request to testing farm API."
logger.debug("Failed to post request to testing farm API.")
self.report_status_to_test_for_chroot(
state=BaseCommitStatus.error,
description=msg,
chroot=chroot,
)
return TaskResults(success=False, details={"msg": msg})
# success set check on pending
if req.status_code != 200:
# something went wrong
if req.json() and "errors" in req.json():
msg = req.json()["errors"]
# specific case, unsupported arch
if nested_get(req.json(), "errors", "environments", "0",
"arch"):
msg = req.json()["errors"]["environments"]["0"]["arch"]
else:
msg = f"Failed to submit tests: {req.reason}"
logger.error(msg)
self.report_status_to_test_for_chroot(
state=BaseCommitStatus.failure,
description=msg,
chroot=chroot,
)
return TaskResults(success=False, details={"msg": msg})
# Response: {"id": "9fa3cbd1-83f2-4326-a118-aad59f5", ...}
pipeline_id = req.json()["id"]
logger.debug(
f"Submitted ({req.status_code}) to testing farm as request {pipeline_id}"
)
run_model = (RunModel.create(
type=self.db_trigger.job_trigger_model_type,
trigger_id=self.db_trigger.id,
) if self.skip_build else build.runs[-1])
created_model = TFTTestRunModel.create(
pipeline_id=pipeline_id,
commit_sha=self.metadata.commit_sha,
status=TestingFarmResult.new,
target=chroot,
web_url=None,
run_model=run_model,
# In _payload() we ask TF to test commit_sha of fork (PR's source).
# Store original url. If this proves to work, make it a separate column.
data={"base_project_url": self.project.get_web_url()},
)
self.report_status_to_test_for_chroot(
state=BaseCommitStatus.running,
description="Tests have been submitted ...",
url=get_testing_farm_info_url(created_model.id),
chroot=chroot,
)
return TaskResults(success=True, details={})
def run_copr_build(self) -> TaskResults:
if not (self.job_build or self.job_tests):
msg = "No copr_build or tests job defined."
# we can't report it to end-user at this stage
return TaskResults(success=False, details={"msg": msg})
self.report_status_to_all(
description="Building SRPM ...",
state=CommitStatus.pending,
# pagure requires "valid url"
url="",
)
self.create_srpm_if_needed()
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
try:
build_id, web_url = self.run_build()
Pushgateway().push_copr_build_created()
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=CommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
return TaskResults(
success=False,
details={"msg": "Submit of the Copr build failed.", "error": str(ex)},
)
unprocessed_chroots = []
for chroot in self.build_targets:
if chroot not in self.available_chroots:
self.report_status_to_all_for_chroot(
state=CommitStatus.error,
description=f"Not supported target: {chroot}",
url=get_srpm_log_url_from_flask(self.srpm_model.id),
chroot=chroot,
)
unprocessed_chroots.append(chroot)
continue
copr_build = CoprBuildModel.get_or_create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="pending",
srpm_build=self.srpm_model,
trigger_model=self.db_trigger,
)
url = get_copr_build_info_url_from_flask(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
if unprocessed_chroots:
unprocessed = "\n".join(sorted(unprocessed_chroots))
available = "\n".join(sorted(self.available_chroots))
self.project.pr_comment(
pr_id=self.metadata.pr_id,
body="There are build targets that are not supported by COPR.\n"
"<details>\n<summary>Unprocessed build targets</summary>\n\n"
f"```\n{unprocessed}\n```\n</details>\n"
"<details>\n<summary>Available build targets</summary>\n\n"
f"```\n{available}\n```\n</details>",
)
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id,),
countdown=120, # do the first check in 120s
)
return TaskResults(success=True, details={})
