async def _build_target_async(self, rpm: RPMMetadata, target: str):
""" Creates a Brew task to build the rpm against specific target
:param rpm: Metadata of the rpm
:param target: The target to build against
"""
dg: RPMDistGitRepo = rpm.distgit_repo()
logger = rpm.logger
logger.info("Building %s against target %s", rpm.name, target)
cmd = ["rhpkg", "build", "--nowait", "--target", target]
if self._scratch:
cmd.append("--skip-tag")
if not self._dry_run:
out, _ = await exectools.cmd_assert_async(cmd, cwd=dg.dg_path)
else:
logger.warning("DRY RUN - Would have created Brew task with %s",
cmd)
out = "Created task: 0\nTask info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=0\n"
# we should have a brew task we can monitor listed in the stdout.
out_lines = out.splitlines()
# Look for a line like: "Created task: 13949050" . Extract the identifier.
task_id = int(
next((line.split(":")[1]).strip() for line in out_lines
if line.startswith("Created task:")))
# Look for a line like: "Task info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=13948942"
task_url = next((line.split(":", 1)[1]).strip() for line in out_lines
if line.startswith("Task info:"))
logger.info("Build running: %s - %s - %s", rpm.rpm_name, target,
task_url)
return task_id, task_url
async def _rebase_rpm(runtime: Runtime, builder: RPMBuilder, rpm: RPMMetadata,
version, release):
logger = rpm.logger
action = "rebase_rpm"
record = {
"distgit_key": rpm.distgit_key,
"rpm": rpm.rpm_name,
"version": version,
"release": release,
"message": "Unknown failure",
"status": -1,
# Status defaults to failure until explicitly set by success. This handles raised exceptions.
}
try:
await builder.rebase(rpm, version, release)
record["version"] = rpm.version
record["release"] = rpm.release
record["specfile"] = rpm.specfile
record["private_fix"] = rpm.private_fix
record["source_head"] = rpm.source_head
record["source_commit"] = rpm.pre_init_sha or ""
record["dg_branch"] = rpm.distgit_repo().branch
record["status"] = 0
record["message"] = "Success"
logger.info("Successfully rebased rpm: %s", rpm.distgit_key)
except Exception:
tb = traceback.format_exc()
record["message"] = "Exception occurred:\n{}".format(tb)
logger.error("Exception occurred when rebasing %s:\n%s",
rpm.distgit_key, tb)
finally:
runtime.add_record(action, **record)
return record["status"]
def test__build_rpm(self, MockDir, MockEntityLoggingAdapter):
runtime = mock.MagicMock(brew_logs_dir="/path/to/brew/logs")
koji_session = runtime.build_retrying_koji_client.return_value
data_obj = mock.MagicMock(key="foo",
filename="foo.yml",
path="/path/to/ocp-build-data/rpms/foo.yml",
data=yaml.safe_load(
TestRPMMetadata.FOO_RPM_CONFIG))
metadata = RPMMetadata(runtime, data_obj, clone_source=False)
metadata.source_path = "/path/to/sources/foo"
record = {}
terminate_event = mock.MagicMock()
with mock.patch("doozerlib.rpmcfg.exectools.cmd_gather") as mock_cmd_gather, \
mock.patch("doozerlib.rpmcfg.watch_tasks") as mock_watch_tasks:
def fake_cmd_gather(cmd, **kwargs):
if cmd == [
'tito', 'release', '--debug', '--yes', '--test', 'aos'
]:
return 0, "Created task: 1\nTask info: https://brewweb.example.com/brew/taskinfo?taskID=1", ""
if len(cmd) >= 2 and cmd[0] == "brew" and cmd[
1] == "download-logs" and "--recurse" in cmd:
return 0, "", ""
raise ValueError(f"Unexpected command: {cmd}")
mock_cmd_gather.side_effect = fake_cmd_gather
koji_session.getTaskRequest.return_value = (
"https://distgit.example.com/rpms/foo.git#abcdefg",
"rhaos-4.7-rhel-7-candidate", {})
koji_session.build.return_value = 2
mock_watch_tasks.return_value = {1: None, 2: None}
result = metadata._build_rpm(False, record, terminate_event)
self.assertTrue(result)
self.assertEqual(record["task_id"], 1)
self.assertListEqual(record["task_ids"], [1, 2])
mock_cmd_gather.assert_called()
koji_session.getTaskRequest.assert_called()
koji_session.build.assert_called()
mock_watch_tasks.assert_called()
def test_assert_golang_versions(self, MockDir, MockEntityLoggingAdapter):
runtime = mock.MagicMock(brew_logs_dir="/path/to/brew/logs")
koji_session = runtime.build_retrying_koji_client.return_value
data_obj = mock.MagicMock(key="foo",
filename="foo.yml",
path="/path/to/ocp-build-data/rpms/foo.yml",
data=yaml.safe_load(
TestRPMMetadata.FOO_RPM_CONFIG))
koji_session.multicall.return_value.__enter__.return_value.getBuildTarget.side_effect = lambda target: MagicMock(
result={"build_tag_name": target.replace("-candidate", "-build")})
metadata = RPMMetadata(runtime, data_obj, clone_source=False)
metadata.targets = [
"rhaos-4.7-rhel-7-candidate", "rhaos-4.7-rhel-8-candidate"
]
runtime.group_config.check_golang_versions = "exact"
with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds"
) as get_latest_builds:
def fake_get_latest_builds(tag_component_tuples, build_type, event,
session):
results = {
('rhaos-4.7-rhel-7-build', 'golang'): [{
"name": "golang",
"version": "1.2.3",
"release": "1.el7",
"epoch": None
}],
('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{
"name":
"golang-scl-shim",
"version":
"1.4.0",
"release":
"2.el7",
"epoch":
None
}],
('rhaos-4.7-rhel-8-build', 'golang'): [{
"name": "golang",
"version": "1.4.5",
"release": "3.el8",
"epoch": None
}],
('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [],
}
return [
results[tag_component]
for tag_component in tag_component_tuples
]
get_latest_builds.side_effect = fake_get_latest_builds
koji_session.getLatestBuilds.return_value = [{
"name": "go-toolset-1.4",
"version": "1.4.5",
"release": "4.el7",
"epoch": None
}]
metadata.assert_golang_versions()
koji_session.getLatestBuilds.assert_called_with(
'rhaos-4.7-rhel-7-build', package="go-toolset-1.4", type="rpm")
runtime.group_config.check_golang_versions = "exact"
RPMMetadata.target_golangs = {}
with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds"
) as get_latest_builds:
def fake_get_latest_builds(tag_component_tuples, build_type, event,
session):
results = {
('rhaos-4.7-rhel-7-build', 'golang'): [{
"name": "golang",
"version": "1.2.3",
"release": "1.el7",
"epoch": None
}],
('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{
"name":
"golang-scl-shim",
"version":
"1.4.6",
"release":
"2.el7",
"epoch":
None
}],
('rhaos-4.7-rhel-8-build', 'golang'): [{
"name": "golang",
"version": "1.4.6",
"release": "3.el8",
"epoch": None
}],
('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [],
}
return [
results[tag_component]
for tag_component in tag_component_tuples
]
get_latest_builds.side_effect = fake_get_latest_builds
koji_session.getLatestBuilds.return_value = [{
"name": "go-toolset-1.4",
"version": "1.4.5",
"release": "4.el7",
"epoch": None
}]
with self.assertRaises(DoozerFatalError):
metadata.assert_golang_versions()
runtime.group_config.check_golang_versions = "x.y"
RPMMetadata.target_golangs = {}
with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds"
) as get_latest_builds:
def fake_get_latest_builds(tag_component_tuples, build_type, event,
session):
results = {
('rhaos-4.7-rhel-7-build', 'golang'): [{
"name": "golang",
"version": "1.2.3",
"release": "1.el7",
"epoch": None
}],
('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{
"name":
"golang-scl-shim",
"version":
"1.4.5",
"release":
"2.el7",
"epoch":
None
}],
('rhaos-4.7-rhel-8-build', 'golang'): [{
"name": "golang",
"version": "1.4.6",
"release": "3.el8",
"epoch": None
}],
('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [],
}
return [
results[tag_component]
for tag_component in tag_component_tuples
]
get_latest_builds.side_effect = fake_get_latest_builds
koji_session.getLatestBuilds.return_value = [{
"name": "go-toolset-1.4",
"version": "1.4.5",
"release": "4.el7",
"epoch": None
}]
metadata.assert_golang_versions()
runtime.group_config.check_golang_versions = "x.y"
RPMMetadata.target_golangs = {}
with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds"
) as get_latest_builds:
def fake_get_latest_builds(tag_component_tuples, build_type, event,
session):
results = {
('rhaos-4.7-rhel-7-build', 'golang'): [{
"name": "golang",
"version": "1.2.3",
"release": "1.el7",
"epoch": None
}],
('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{
"name":
"golang-scl-shim",
"version":
"1.4.5",
"release":
"2.el7",
"epoch":
None
}],
('rhaos-4.7-rhel-8-build', 'golang'): [{
"name": "golang",
"version": "1.5.6",
"release": "3.el8",
"epoch": None
}],
('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [],
}
return [
results[tag_component]
for tag_component in tag_component_tuples
]
get_latest_builds.side_effect = fake_get_latest_builds
koji_session.getLatestBuilds.return_value = [{
"name": "go-toolset-1.4",
"version": "1.4.5",
"release": "4.el7",
"epoch": None
}]
with self.assertRaises(DoozerFatalError):
metadata.assert_golang_versions()
def check_group_rpm_package_consistency(
self, rpm_meta: RPMMetadata) -> List[AssemblyIssue]:
"""
Evaluate the current assembly builds of RPMs in the group and check whether they are consistent with
the assembly definition.
:param rpm_meta: The rpm metadata to evaluate
:return: Returns a (potentially empty) list of reasons the rpm should be rebuilt.
"""
self.runtime.logger.info(
f'Checking group RPM for consistency: {rpm_meta.distgit_key}...')
issues: List[AssemblyIssue] = []
for rpm_meta in self.runtime.rpm_metas():
dgk = rpm_meta.distgit_key
for el_ver in rpm_meta.determine_rhel_targets():
brew_build_dict = self.get_group_rpm_build_dicts(
el_ver=el_ver)[dgk]
if not brew_build_dict:
# Impermissible. The RPM should be built for each target.
issues.append(
AssemblyIssue(
f'Did not find rhel-{el_ver} build for {dgk}',
component=dgk))
continue
"""
Assess whether the image build has the upstream
source git repo and git commit that may have been declared/
overridden in an assembly definition.
"""
content_git_url = rpm_meta.config.content.source.git.url
if content_git_url:
# Make sure things are in https form so we can compare
# content_git_url = util.convert_remote_git_to_https(content_git_url)
# TODO: The commit in which this comment is introduced also introduces
# machine parsable yaml documents into distgit commits. Once this code
# has been running for our active 4.x releases for some time,
# we should check the distgit commit info against the git.url
# in our metadata.
try:
target_branch = rpm_meta.config.content.source.git.branch.target
if target_branch:
_ = int(target_branch,
16) # parse the name as a git commit
# if we reach here, a git commit hash was declared as the
# upstream source of the rpm package's content. We should verify
# it perfectly matches what we find in the assembly build.
# Each package build gets git commits encoded into the
# release field of the NVR. So the NVR should contain
# the desired commit.
build_nvr = brew_build_dict['nvr']
if target_branch[:7] not in build_nvr:
# Impermissible because the assembly definition can simply be changed.
issues.append(
AssemblyIssue(
f'{dgk} build for rhel-{el_ver} did not find git commit {target_branch[:7]} in package RPM NVR {build_nvr}',
component=dgk))
except ValueError:
# The meta's target branch a normal branch name
# and not a git commit. When this is the case,
# we don't try to assert anything about the build's
# git commit.
pass
return issues
async def rebase(self, rpm: RPMMetadata, version: str,
release: str) -> str:
""" Rebases and pushes the distgit repo for an rpm
:param rpm: Metadata of the rpm
:param version: Set rpm version
:param release: Set rpm release
:return: Hash of the new distgit commit
"""
logger = rpm.logger
# clone source and distgit
logger.info("Cloning source and distgit repos...")
dg: RPMDistGitRepo
_, dg = await asyncio.gather(
exectools.to_thread(rpm.clone_source),
exectools.to_thread(rpm.distgit_repo, autoclone=True),
)
# cleanup distgit dir
logger.info("Cleaning up distgit repo...")
await exectools.cmd_assert_async(
["git", "reset", "--hard", "origin/" + dg.branch],
cwd=dg.distgit_dir)
await exectools.cmd_assert_async(
["git", "rm", "--ignore-unmatch", "-rf", "."], cwd=dg.distgit_dir)
# set .p0/.p1 flag
if self._runtime.group_config.public_upstreams:
if not release.endswith(".p?"):
raise ValueError(
f"'release' must end with '.p?' for an rpm with a public upstream but its actual value is {release}"
)
if rpm.private_fix is None:
raise AssertionError(
"rpm.private_fix flag should already be set")
if rpm.private_fix:
logger.warning("Source contains embargoed fixes.")
pval = ".p1"
else:
pval = ".p0"
release = release[:-3] + pval
# include commit hash in release field
release += ".g" + rpm.pre_init_sha[:7]
if self._runtime.assembly:
release += f'.assembly.{self._runtime.assembly}'
rpm.set_nvr(version, release)
# generate new specfile
tarball_name = f"{rpm.config.name}-{rpm.version}-{rpm.release}.tar.gz"
logger.info("Creating rpm spec file...")
source_commit_url = '{}/commit/{}'.format(rpm.public_upstream_url,
rpm.pre_init_sha)
specfile = await self._populate_specfile_async(rpm, tarball_name,
source_commit_url)
dg_specfile_path = dg.dg_path / Path(rpm.specfile).name
async with aiofiles.open(dg_specfile_path, "w") as f:
await f.writelines(specfile)
if rpm.get_package_name_from_spec() != rpm.get_package_name():
raise IOError(
f'RPM package name in .spec file ({rpm.get_package_name_from_spec()}) does not match doozer metadata name {rpm.get_package_name()}'
)
rpm.specfile = str(dg_specfile_path)
# create tarball source as Source0
logger.info("Creating tarball source...")
tarball_path = dg.dg_path / tarball_name
await exectools.cmd_assert_async(
[
"tar",
"-czf",
tarball_path,
"--exclude=.git",
fr"--transform=s,^\./,{rpm.config.name}-{rpm.version}/,",
".",
],
cwd=rpm.source_path,
)
logger.info(
"Done creating tarball source. Uploading to distgit lookaside cache..."
)
if self._push:
if not self._dry_run:
await exectools.cmd_assert_async(
["rhpkg", "new-sources", tarball_name],
cwd=dg.dg_path,
retries=3)
else:
async with aiofiles.open(dg.dg_path / "sources", "w") as f:
f.write("SHA512 ({}) = {}\n".format(
tarball_name, "0" * 128))
if self._push:
logger.warning("DRY RUN - Would have uploaded %s",
tarball_name)
# copy Source1, Source2,... and Patch0, Patch1,...
logger.info("Determining additional sources and patches...")
out, _ = await exectools.cmd_assert_async(
["spectool", "--", dg_specfile_path], cwd=dg.dg_path)
for line in out.splitlines():
line_split = line.split(": ")
if len(line_split) < 2 or line_split[0] == "Source0":
continue
filename = line_split[1].strip()
src = Path(rpm.source_path, filename)
# For security, ensure the source file referenced by the specfile is contained in both source and distgit directories.
if not is_in_directory(src, rpm.source_path):
raise ValueError(
"STOP! Source file {} referenced in Specfile {} lives outside of the source directory {}"
.format(filename, dg_specfile_path, rpm.source_path))
dest = dg.dg_path / filename
if not is_in_directory(dest, dg.dg_path):
raise ValueError(
"STOP! Source file {} referenced in Specfile {} would be copied to a directory outside of distgit directory {}"
.format(filename, dg_specfile_path, dg.dg_path))
dest.parent.mkdir(parents=True, exist_ok=True)
logging.debug("Copying %s", filename)
shutil.copy(src, dest, follow_symlinks=False)
# run modifications
if rpm.config.content.source.modifications is not Missing:
logging.info("Running custom modifications...")
await exectools.to_thread(rpm._run_modifications, dg_specfile_path,
dg.dg_path)
# commit changes
logging.info("Committing distgit changes...")
await aiofiles.os.remove(tarball_path)
commit_hash = await exectools.to_thread(
dg.commit,
f"Automatic commit of package [{rpm.config.name}] release [{rpm.version}-{rpm.release}].",
commit_attributes={
'version': rpm.version,
'release': rpm.release,
'io.openshift.build.commit.id': rpm.pre_init_sha,
'io.openshift.build.source-location': rpm.public_upstream_url,
})
if self._push:
# push
if not self._dry_run:
await dg.push_async()
else:
logger.warning("Would have pushed %s", dg.name)
return commit_hash
async def build(self, rpm: RPMMetadata, retries: int = 3):
""" Builds rpm with the latest distgit commit
:param rpm: Metadata of the RPM
:param retries: The number of times to retry
"""
logger = rpm.logger
dg = rpm.distgit_repo()
if rpm.specfile is None:
rpm.specfile, nvr, rpm.pre_init_sha = await dg.resolve_specfile_async(
)
rpm.set_nvr(nvr[1], nvr[2])
if self._runtime.assembly and isolate_assembly_in_release(
rpm.release) != self._runtime.assembly:
# Assemblies should follow its naming convention
raise ValueError(
f"RPM {rpm.name} is not rebased with assembly '{self._runtime.assembly}'."
)
if rpm.private_fix is None:
rpm.private_fix = ".p1" in rpm.release
if rpm.private_fix:
logger.warning("This rpm build contains embargoed fixes.")
if len(
rpm.targets
) > 1: # for a multi target build, we need to ensure all buildroots have valid versions of golang compilers
logger.info("Checking whether this is a golang package...")
if await self._golang_required(rpm.specfile):
# assert buildroots contain the correct versions of golang
logger.info(
"This is a golang package. Checking whether buildroots contain consistent versions of golang compilers..."
)
await exectools.to_thread(rpm.assert_golang_versions)
# Submit build tasks
message = "Unknown error"
for attempt in range(retries):
# errors, task_ids, task_urls = await self._create_build_tasks(dg)
task_ids = []
task_urls = []
nvrs = []
logger.info("Creating Brew tasks...")
for task_id, task_url in await asyncio.gather(*[
self._build_target_async(rpm, target)
for target in rpm.targets
]):
task_ids.append(task_id)
task_urls.append(task_url)
# Wait for all tasks to complete
logger.info("Waiting for all tasks to complete")
errors = await self._watch_tasks_async(task_ids, logger)
# Gather brew-logs
logger.info("Gathering brew-logs")
for target, task_id in zip(rpm.targets, task_ids):
logs_dir = (Path(self._runtime.brew_logs_dir) / rpm.name /
f"{target}-{task_id}")
cmd = [
"brew", "download-logs", "--recurse", "-d", logs_dir,
task_id
]
if not self._dry_run:
logs_rc, _, logs_err = await exectools.cmd_gather_async(cmd
)
if logs_rc != exectools.SUCCESS:
logger.warning(
"Error downloading build logs from brew for task %s: %s"
% (task_id, logs_err))
else:
logger.warning(
"DRY RUN - Would have downloaded Brew logs with %s",
cmd)
failed_tasks = {
task_id
for task_id, error in errors.items() if error is not None
}
if not failed_tasks:
# All tasks complete.
with self._runtime.shared_koji_client_session() as koji_api:
if not koji_api.logged_in:
koji_api.gssapi_login()
with koji_api.multicall(strict=True) as m:
multicall_tasks = [
m.listBuilds(taskID=task_id, completeBefore=None)
for task_id in task_ids
] # this call should not be constrained by brew event
nvrs = [task.result[0]["nvr"] for task in multicall_tasks]
if self._runtime.hotfix:
# Tag rpms so they don't get garbage collected.
self._runtime.logger.info(
f'Tagging build(s) {nvrs} with {rpm.hotfix_brew_tag()} to prevent garbage collection'
)
with koji_api.multicall(strict=True) as m:
for nvr in nvrs:
m.tagBuild(rpm.hotfix_brew_tag(), nvr)
logger.info("Successfully built rpm: %s", rpm.rpm_name)
rpm.build_status = True
break
# An error occurred. We don't have a viable build.
message = ", ".join(f"Task {task_id} failed: {errors[task_id]}"
for task_id in failed_tasks)
logger.warning(
"Error building rpm %s [attempt #%s] in Brew: %s",
rpm.qualified_name,
attempt + 1,
message,
)
if attempt < retries - 1:
# Brew does not handle an immediate retry correctly, wait before trying another build
await asyncio.sleep(5 * 60)
if not rpm.build_status:
raise exectools.RetryException(
f"Giving up after {retries} failed attempt(s): {message}",
(task_ids, task_urls),
)
return task_ids, task_urls, nvrs