async def _build_target_async(self, rpm: RPMMetadata, target: str):
""" Creates a Brew task to build the rpm against specific target
:param rpm: Metadata of the rpm
:param target: The target to build against
"""
dg: RPMDistGitRepo = rpm.distgit_repo()
logger = rpm.logger
logger.info("Building %s against target %s", rpm.name, target)
cmd = ["rhpkg", "build", "--nowait", "--target", target]
if self._scratch:
cmd.append("--skip-tag")
if not self._dry_run:
out, _ = await exectools.cmd_assert_async(cmd, cwd=dg.dg_path)
else:
logger.warning("DRY RUN - Would have created Brew task with %s",
cmd)
out = "Created task: 0\nTask info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=0\n"
# we should have a brew task we can monitor listed in the stdout.
out_lines = out.splitlines()
# Look for a line like: "Created task: 13949050" . Extract the identifier.
task_id = int(
next((line.split(":")[1]).strip() for line in out_lines
if line.startswith("Created task:")))
# Look for a line like: "Task info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=13948942"
task_url = next((line.split(":", 1)[1]).strip() for line in out_lines
if line.startswith("Task info:"))
logger.info("Build running: %s - %s - %s", rpm.rpm_name, target,
task_url)
return task_id, task_url
async def _rebase_rpm(runtime: Runtime, builder: RPMBuilder, rpm: RPMMetadata,
version, release):
logger = rpm.logger
action = "rebase_rpm"
record = {
"distgit_key": rpm.distgit_key,
"rpm": rpm.rpm_name,
"version": version,
"release": release,
"message": "Unknown failure",
"status": -1,
# Status defaults to failure until explicitly set by success. This handles raised exceptions.
}
try:
await builder.rebase(rpm, version, release)
record["version"] = rpm.version
record["release"] = rpm.release
record["specfile"] = rpm.specfile
record["private_fix"] = rpm.private_fix
record["source_head"] = rpm.source_head
record["source_commit"] = rpm.pre_init_sha or ""
record["dg_branch"] = rpm.distgit_repo().branch
record["status"] = 0
record["message"] = "Success"
logger.info("Successfully rebased rpm: %s", rpm.distgit_key)
except Exception:
tb = traceback.format_exc()
record["message"] = "Exception occurred:\n{}".format(tb)
logger.error("Exception occurred when rebasing %s:\n%s",
rpm.distgit_key, tb)
finally:
runtime.add_record(action, **record)
return record["status"]
async def build(self, rpm: RPMMetadata, retries: int = 3):
""" Builds rpm with the latest distgit commit
:param rpm: Metadata of the RPM
:param retries: The number of times to retry
"""
logger = rpm.logger
dg = rpm.distgit_repo()
if rpm.specfile is None:
rpm.specfile, nvr, rpm.pre_init_sha = await dg.resolve_specfile_async(
)
rpm.set_nvr(nvr[1], nvr[2])
if self._runtime.assembly and isolate_assembly_in_release(
rpm.release) != self._runtime.assembly:
# Assemblies should follow its naming convention
raise ValueError(
f"RPM {rpm.name} is not rebased with assembly '{self._runtime.assembly}'."
)
if rpm.private_fix is None:
rpm.private_fix = ".p1" in rpm.release
if rpm.private_fix:
logger.warning("This rpm build contains embargoed fixes.")
if len(
rpm.targets
) > 1: # for a multi target build, we need to ensure all buildroots have valid versions of golang compilers
logger.info("Checking whether this is a golang package...")
if await self._golang_required(rpm.specfile):
# assert buildroots contain the correct versions of golang
logger.info(
"This is a golang package. Checking whether buildroots contain consistent versions of golang compilers..."
)
await exectools.to_thread(rpm.assert_golang_versions)
# Submit build tasks
message = "Unknown error"
for attempt in range(retries):
# errors, task_ids, task_urls = await self._create_build_tasks(dg)
task_ids = []
task_urls = []
nvrs = []
logger.info("Creating Brew tasks...")
for task_id, task_url in await asyncio.gather(*[
self._build_target_async(rpm, target)
for target in rpm.targets
]):
task_ids.append(task_id)
task_urls.append(task_url)
# Wait for all tasks to complete
logger.info("Waiting for all tasks to complete")
errors = await self._watch_tasks_async(task_ids, logger)
# Gather brew-logs
logger.info("Gathering brew-logs")
for target, task_id in zip(rpm.targets, task_ids):
logs_dir = (Path(self._runtime.brew_logs_dir) / rpm.name /
f"{target}-{task_id}")
cmd = [
"brew", "download-logs", "--recurse", "-d", logs_dir,
task_id
]
if not self._dry_run:
logs_rc, _, logs_err = await exectools.cmd_gather_async(cmd
)
if logs_rc != exectools.SUCCESS:
logger.warning(
"Error downloading build logs from brew for task %s: %s"
% (task_id, logs_err))
else:
logger.warning(
"DRY RUN - Would have downloaded Brew logs with %s",
cmd)
failed_tasks = {
task_id
for task_id, error in errors.items() if error is not None
}
if not failed_tasks:
# All tasks complete.
with self._runtime.shared_koji_client_session() as koji_api:
if not koji_api.logged_in:
koji_api.gssapi_login()
with koji_api.multicall(strict=True) as m:
multicall_tasks = [
m.listBuilds(taskID=task_id, completeBefore=None)
for task_id in task_ids
] # this call should not be constrained by brew event
nvrs = [task.result[0]["nvr"] for task in multicall_tasks]
if self._runtime.hotfix:
# Tag rpms so they don't get garbage collected.
self._runtime.logger.info(
f'Tagging build(s) {nvrs} with {rpm.hotfix_brew_tag()} to prevent garbage collection'
)
with koji_api.multicall(strict=True) as m:
for nvr in nvrs:
m.tagBuild(rpm.hotfix_brew_tag(), nvr)
logger.info("Successfully built rpm: %s", rpm.rpm_name)
rpm.build_status = True
break
# An error occurred. We don't have a viable build.
message = ", ".join(f"Task {task_id} failed: {errors[task_id]}"
for task_id in failed_tasks)
logger.warning(
"Error building rpm %s [attempt #%s] in Brew: %s",
rpm.qualified_name,
attempt + 1,
message,
)
if attempt < retries - 1:
# Brew does not handle an immediate retry correctly, wait before trying another build
await asyncio.sleep(5 * 60)
if not rpm.build_status:
raise exectools.RetryException(
f"Giving up after {retries} failed attempt(s): {message}",
(task_ids, task_urls),
)
return task_ids, task_urls, nvrs