async def _build_target_async(self, rpm: RPMMetadata, target: str): """ Creates a Brew task to build the rpm against specific target :param rpm: Metadata of the rpm :param target: The target to build against """ dg: RPMDistGitRepo = rpm.distgit_repo() logger = rpm.logger logger.info("Building %s against target %s", rpm.name, target) cmd = ["rhpkg", "build", "--nowait", "--target", target] if self._scratch: cmd.append("--skip-tag") if not self._dry_run: out, _ = await exectools.cmd_assert_async(cmd, cwd=dg.dg_path) else: logger.warning("DRY RUN - Would have created Brew task with %s", cmd) out = "Created task: 0\nTask info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=0\n" # we should have a brew task we can monitor listed in the stdout. out_lines = out.splitlines() # Look for a line like: "Created task: 13949050" . Extract the identifier. task_id = int( next((line.split(":")[1]).strip() for line in out_lines if line.startswith("Created task:"))) # Look for a line like: "Task info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=13948942" task_url = next((line.split(":", 1)[1]).strip() for line in out_lines if line.startswith("Task info:")) logger.info("Build running: %s - %s - %s", rpm.rpm_name, target, task_url) return task_id, task_url
async def _rebase_rpm(runtime: Runtime, builder: RPMBuilder, rpm: RPMMetadata, version, release): logger = rpm.logger action = "rebase_rpm" record = { "distgit_key": rpm.distgit_key, "rpm": rpm.rpm_name, "version": version, "release": release, "message": "Unknown failure", "status": -1, # Status defaults to failure until explicitly set by success. This handles raised exceptions. } try: await builder.rebase(rpm, version, release) record["version"] = rpm.version record["release"] = rpm.release record["specfile"] = rpm.specfile record["private_fix"] = rpm.private_fix record["source_head"] = rpm.source_head record["source_commit"] = rpm.pre_init_sha or "" record["dg_branch"] = rpm.distgit_repo().branch record["status"] = 0 record["message"] = "Success" logger.info("Successfully rebased rpm: %s", rpm.distgit_key) except Exception: tb = traceback.format_exc() record["message"] = "Exception occurred:\n{}".format(tb) logger.error("Exception occurred when rebasing %s:\n%s", rpm.distgit_key, tb) finally: runtime.add_record(action, **record) return record["status"]
def test__build_rpm(self, MockDir, MockEntityLoggingAdapter): runtime = mock.MagicMock(brew_logs_dir="/path/to/brew/logs") koji_session = runtime.build_retrying_koji_client.return_value data_obj = mock.MagicMock(key="foo", filename="foo.yml", path="/path/to/ocp-build-data/rpms/foo.yml", data=yaml.safe_load( TestRPMMetadata.FOO_RPM_CONFIG)) metadata = RPMMetadata(runtime, data_obj, clone_source=False) metadata.source_path = "/path/to/sources/foo" record = {} terminate_event = mock.MagicMock() with mock.patch("doozerlib.rpmcfg.exectools.cmd_gather") as mock_cmd_gather, \ mock.patch("doozerlib.rpmcfg.watch_tasks") as mock_watch_tasks: def fake_cmd_gather(cmd, **kwargs): if cmd == [ 'tito', 'release', '--debug', '--yes', '--test', 'aos' ]: return 0, "Created task: 1\nTask info: https://brewweb.example.com/brew/taskinfo?taskID=1", "" if len(cmd) >= 2 and cmd[0] == "brew" and cmd[ 1] == "download-logs" and "--recurse" in cmd: return 0, "", "" raise ValueError(f"Unexpected command: {cmd}") mock_cmd_gather.side_effect = fake_cmd_gather koji_session.getTaskRequest.return_value = ( "https://distgit.example.com/rpms/foo.git#abcdefg", "rhaos-4.7-rhel-7-candidate", {}) koji_session.build.return_value = 2 mock_watch_tasks.return_value = {1: None, 2: None} result = metadata._build_rpm(False, record, terminate_event) self.assertTrue(result) self.assertEqual(record["task_id"], 1) self.assertListEqual(record["task_ids"], [1, 2]) mock_cmd_gather.assert_called() koji_session.getTaskRequest.assert_called() koji_session.build.assert_called() mock_watch_tasks.assert_called()
def test_assert_golang_versions(self, MockDir, MockEntityLoggingAdapter): runtime = mock.MagicMock(brew_logs_dir="/path/to/brew/logs") koji_session = runtime.build_retrying_koji_client.return_value data_obj = mock.MagicMock(key="foo", filename="foo.yml", path="/path/to/ocp-build-data/rpms/foo.yml", data=yaml.safe_load( TestRPMMetadata.FOO_RPM_CONFIG)) koji_session.multicall.return_value.__enter__.return_value.getBuildTarget.side_effect = lambda target: MagicMock( result={"build_tag_name": target.replace("-candidate", "-build")}) metadata = RPMMetadata(runtime, data_obj, clone_source=False) metadata.targets = [ "rhaos-4.7-rhel-7-candidate", "rhaos-4.7-rhel-8-candidate" ] runtime.group_config.check_golang_versions = "exact" with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds" ) as get_latest_builds: def fake_get_latest_builds(tag_component_tuples, build_type, event, session): results = { ('rhaos-4.7-rhel-7-build', 'golang'): [{ "name": "golang", "version": "1.2.3", "release": "1.el7", "epoch": None }], ('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{ "name": "golang-scl-shim", "version": "1.4.0", "release": "2.el7", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang'): [{ "name": "golang", "version": "1.4.5", "release": "3.el8", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [], } return [ results[tag_component] for tag_component in tag_component_tuples ] get_latest_builds.side_effect = fake_get_latest_builds koji_session.getLatestBuilds.return_value = [{ "name": "go-toolset-1.4", "version": "1.4.5", "release": "4.el7", "epoch": None }] metadata.assert_golang_versions() koji_session.getLatestBuilds.assert_called_with( 'rhaos-4.7-rhel-7-build', package="go-toolset-1.4", type="rpm") runtime.group_config.check_golang_versions = "exact" RPMMetadata.target_golangs = {} with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds" ) as get_latest_builds: def fake_get_latest_builds(tag_component_tuples, build_type, event, session): results = { ('rhaos-4.7-rhel-7-build', 'golang'): [{ "name": "golang", "version": "1.2.3", "release": "1.el7", "epoch": None }], ('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{ "name": "golang-scl-shim", "version": "1.4.6", "release": "2.el7", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang'): [{ "name": "golang", "version": "1.4.6", "release": "3.el8", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [], } return [ results[tag_component] for tag_component in tag_component_tuples ] get_latest_builds.side_effect = fake_get_latest_builds koji_session.getLatestBuilds.return_value = [{ "name": "go-toolset-1.4", "version": "1.4.5", "release": "4.el7", "epoch": None }] with self.assertRaises(DoozerFatalError): metadata.assert_golang_versions() runtime.group_config.check_golang_versions = "x.y" RPMMetadata.target_golangs = {} with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds" ) as get_latest_builds: def fake_get_latest_builds(tag_component_tuples, build_type, event, session): results = { ('rhaos-4.7-rhel-7-build', 'golang'): [{ "name": "golang", "version": "1.2.3", "release": "1.el7", "epoch": None }], ('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{ "name": "golang-scl-shim", "version": "1.4.5", "release": "2.el7", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang'): [{ "name": "golang", "version": "1.4.6", "release": "3.el8", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [], } return [ results[tag_component] for tag_component in tag_component_tuples ] get_latest_builds.side_effect = fake_get_latest_builds koji_session.getLatestBuilds.return_value = [{ "name": "go-toolset-1.4", "version": "1.4.5", "release": "4.el7", "epoch": None }] metadata.assert_golang_versions() runtime.group_config.check_golang_versions = "x.y" RPMMetadata.target_golangs = {} with mock.patch("doozerlib.rpmcfg.brew.get_latest_builds" ) as get_latest_builds: def fake_get_latest_builds(tag_component_tuples, build_type, event, session): results = { ('rhaos-4.7-rhel-7-build', 'golang'): [{ "name": "golang", "version": "1.2.3", "release": "1.el7", "epoch": None }], ('rhaos-4.7-rhel-7-build', 'golang-scl-shim'): [{ "name": "golang-scl-shim", "version": "1.4.5", "release": "2.el7", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang'): [{ "name": "golang", "version": "1.5.6", "release": "3.el8", "epoch": None }], ('rhaos-4.7-rhel-8-build', 'golang-scl-shim'): [], } return [ results[tag_component] for tag_component in tag_component_tuples ] get_latest_builds.side_effect = fake_get_latest_builds koji_session.getLatestBuilds.return_value = [{ "name": "go-toolset-1.4", "version": "1.4.5", "release": "4.el7", "epoch": None }] with self.assertRaises(DoozerFatalError): metadata.assert_golang_versions()
def check_group_rpm_package_consistency( self, rpm_meta: RPMMetadata) -> List[AssemblyIssue]: """ Evaluate the current assembly builds of RPMs in the group and check whether they are consistent with the assembly definition. :param rpm_meta: The rpm metadata to evaluate :return: Returns a (potentially empty) list of reasons the rpm should be rebuilt. """ self.runtime.logger.info( f'Checking group RPM for consistency: {rpm_meta.distgit_key}...') issues: List[AssemblyIssue] = [] for rpm_meta in self.runtime.rpm_metas(): dgk = rpm_meta.distgit_key for el_ver in rpm_meta.determine_rhel_targets(): brew_build_dict = self.get_group_rpm_build_dicts( el_ver=el_ver)[dgk] if not brew_build_dict: # Impermissible. The RPM should be built for each target. issues.append( AssemblyIssue( f'Did not find rhel-{el_ver} build for {dgk}', component=dgk)) continue """ Assess whether the image build has the upstream source git repo and git commit that may have been declared/ overridden in an assembly definition. """ content_git_url = rpm_meta.config.content.source.git.url if content_git_url: # Make sure things are in https form so we can compare # content_git_url = util.convert_remote_git_to_https(content_git_url) # TODO: The commit in which this comment is introduced also introduces # machine parsable yaml documents into distgit commits. Once this code # has been running for our active 4.x releases for some time, # we should check the distgit commit info against the git.url # in our metadata. try: target_branch = rpm_meta.config.content.source.git.branch.target if target_branch: _ = int(target_branch, 16) # parse the name as a git commit # if we reach here, a git commit hash was declared as the # upstream source of the rpm package's content. We should verify # it perfectly matches what we find in the assembly build. # Each package build gets git commits encoded into the # release field of the NVR. So the NVR should contain # the desired commit. build_nvr = brew_build_dict['nvr'] if target_branch[:7] not in build_nvr: # Impermissible because the assembly definition can simply be changed. issues.append( AssemblyIssue( f'{dgk} build for rhel-{el_ver} did not find git commit {target_branch[:7]} in package RPM NVR {build_nvr}', component=dgk)) except ValueError: # The meta's target branch a normal branch name # and not a git commit. When this is the case, # we don't try to assert anything about the build's # git commit. pass return issues
async def rebase(self, rpm: RPMMetadata, version: str, release: str) -> str: """ Rebases and pushes the distgit repo for an rpm :param rpm: Metadata of the rpm :param version: Set rpm version :param release: Set rpm release :return: Hash of the new distgit commit """ logger = rpm.logger # clone source and distgit logger.info("Cloning source and distgit repos...") dg: RPMDistGitRepo _, dg = await asyncio.gather( exectools.to_thread(rpm.clone_source), exectools.to_thread(rpm.distgit_repo, autoclone=True), ) # cleanup distgit dir logger.info("Cleaning up distgit repo...") await exectools.cmd_assert_async( ["git", "reset", "--hard", "origin/" + dg.branch], cwd=dg.distgit_dir) await exectools.cmd_assert_async( ["git", "rm", "--ignore-unmatch", "-rf", "."], cwd=dg.distgit_dir) # set .p0/.p1 flag if self._runtime.group_config.public_upstreams: if not release.endswith(".p?"): raise ValueError( f"'release' must end with '.p?' for an rpm with a public upstream but its actual value is {release}" ) if rpm.private_fix is None: raise AssertionError( "rpm.private_fix flag should already be set") if rpm.private_fix: logger.warning("Source contains embargoed fixes.") pval = ".p1" else: pval = ".p0" release = release[:-3] + pval # include commit hash in release field release += ".g" + rpm.pre_init_sha[:7] if self._runtime.assembly: release += f'.assembly.{self._runtime.assembly}' rpm.set_nvr(version, release) # generate new specfile tarball_name = f"{rpm.config.name}-{rpm.version}-{rpm.release}.tar.gz" logger.info("Creating rpm spec file...") source_commit_url = '{}/commit/{}'.format(rpm.public_upstream_url, rpm.pre_init_sha) specfile = await self._populate_specfile_async(rpm, tarball_name, source_commit_url) dg_specfile_path = dg.dg_path / Path(rpm.specfile).name async with aiofiles.open(dg_specfile_path, "w") as f: await f.writelines(specfile) if rpm.get_package_name_from_spec() != rpm.get_package_name(): raise IOError( f'RPM package name in .spec file ({rpm.get_package_name_from_spec()}) does not match doozer metadata name {rpm.get_package_name()}' ) rpm.specfile = str(dg_specfile_path) # create tarball source as Source0 logger.info("Creating tarball source...") tarball_path = dg.dg_path / tarball_name await exectools.cmd_assert_async( [ "tar", "-czf", tarball_path, "--exclude=.git", fr"--transform=s,^\./,{rpm.config.name}-{rpm.version}/,", ".", ], cwd=rpm.source_path, ) logger.info( "Done creating tarball source. Uploading to distgit lookaside cache..." ) if self._push: if not self._dry_run: await exectools.cmd_assert_async( ["rhpkg", "new-sources", tarball_name], cwd=dg.dg_path, retries=3) else: async with aiofiles.open(dg.dg_path / "sources", "w") as f: f.write("SHA512 ({}) = {}\n".format( tarball_name, "0" * 128)) if self._push: logger.warning("DRY RUN - Would have uploaded %s", tarball_name) # copy Source1, Source2,... and Patch0, Patch1,... logger.info("Determining additional sources and patches...") out, _ = await exectools.cmd_assert_async( ["spectool", "--", dg_specfile_path], cwd=dg.dg_path) for line in out.splitlines(): line_split = line.split(": ") if len(line_split) < 2 or line_split[0] == "Source0": continue filename = line_split[1].strip() src = Path(rpm.source_path, filename) # For security, ensure the source file referenced by the specfile is contained in both source and distgit directories. if not is_in_directory(src, rpm.source_path): raise ValueError( "STOP! Source file {} referenced in Specfile {} lives outside of the source directory {}" .format(filename, dg_specfile_path, rpm.source_path)) dest = dg.dg_path / filename if not is_in_directory(dest, dg.dg_path): raise ValueError( "STOP! Source file {} referenced in Specfile {} would be copied to a directory outside of distgit directory {}" .format(filename, dg_specfile_path, dg.dg_path)) dest.parent.mkdir(parents=True, exist_ok=True) logging.debug("Copying %s", filename) shutil.copy(src, dest, follow_symlinks=False) # run modifications if rpm.config.content.source.modifications is not Missing: logging.info("Running custom modifications...") await exectools.to_thread(rpm._run_modifications, dg_specfile_path, dg.dg_path) # commit changes logging.info("Committing distgit changes...") await aiofiles.os.remove(tarball_path) commit_hash = await exectools.to_thread( dg.commit, f"Automatic commit of package [{rpm.config.name}] release [{rpm.version}-{rpm.release}].", commit_attributes={ 'version': rpm.version, 'release': rpm.release, 'io.openshift.build.commit.id': rpm.pre_init_sha, 'io.openshift.build.source-location': rpm.public_upstream_url, }) if self._push: # push if not self._dry_run: await dg.push_async() else: logger.warning("Would have pushed %s", dg.name) return commit_hash
async def build(self, rpm: RPMMetadata, retries: int = 3): """ Builds rpm with the latest distgit commit :param rpm: Metadata of the RPM :param retries: The number of times to retry """ logger = rpm.logger dg = rpm.distgit_repo() if rpm.specfile is None: rpm.specfile, nvr, rpm.pre_init_sha = await dg.resolve_specfile_async( ) rpm.set_nvr(nvr[1], nvr[2]) if self._runtime.assembly and isolate_assembly_in_release( rpm.release) != self._runtime.assembly: # Assemblies should follow its naming convention raise ValueError( f"RPM {rpm.name} is not rebased with assembly '{self._runtime.assembly}'." ) if rpm.private_fix is None: rpm.private_fix = ".p1" in rpm.release if rpm.private_fix: logger.warning("This rpm build contains embargoed fixes.") if len( rpm.targets ) > 1: # for a multi target build, we need to ensure all buildroots have valid versions of golang compilers logger.info("Checking whether this is a golang package...") if await self._golang_required(rpm.specfile): # assert buildroots contain the correct versions of golang logger.info( "This is a golang package. Checking whether buildroots contain consistent versions of golang compilers..." ) await exectools.to_thread(rpm.assert_golang_versions) # Submit build tasks message = "Unknown error" for attempt in range(retries): # errors, task_ids, task_urls = await self._create_build_tasks(dg) task_ids = [] task_urls = [] nvrs = [] logger.info("Creating Brew tasks...") for task_id, task_url in await asyncio.gather(*[ self._build_target_async(rpm, target) for target in rpm.targets ]): task_ids.append(task_id) task_urls.append(task_url) # Wait for all tasks to complete logger.info("Waiting for all tasks to complete") errors = await self._watch_tasks_async(task_ids, logger) # Gather brew-logs logger.info("Gathering brew-logs") for target, task_id in zip(rpm.targets, task_ids): logs_dir = (Path(self._runtime.brew_logs_dir) / rpm.name / f"{target}-{task_id}") cmd = [ "brew", "download-logs", "--recurse", "-d", logs_dir, task_id ] if not self._dry_run: logs_rc, _, logs_err = await exectools.cmd_gather_async(cmd ) if logs_rc != exectools.SUCCESS: logger.warning( "Error downloading build logs from brew for task %s: %s" % (task_id, logs_err)) else: logger.warning( "DRY RUN - Would have downloaded Brew logs with %s", cmd) failed_tasks = { task_id for task_id, error in errors.items() if error is not None } if not failed_tasks: # All tasks complete. with self._runtime.shared_koji_client_session() as koji_api: if not koji_api.logged_in: koji_api.gssapi_login() with koji_api.multicall(strict=True) as m: multicall_tasks = [ m.listBuilds(taskID=task_id, completeBefore=None) for task_id in task_ids ] # this call should not be constrained by brew event nvrs = [task.result[0]["nvr"] for task in multicall_tasks] if self._runtime.hotfix: # Tag rpms so they don't get garbage collected. self._runtime.logger.info( f'Tagging build(s) {nvrs} with {rpm.hotfix_brew_tag()} to prevent garbage collection' ) with koji_api.multicall(strict=True) as m: for nvr in nvrs: m.tagBuild(rpm.hotfix_brew_tag(), nvr) logger.info("Successfully built rpm: %s", rpm.rpm_name) rpm.build_status = True break # An error occurred. We don't have a viable build. message = ", ".join(f"Task {task_id} failed: {errors[task_id]}" for task_id in failed_tasks) logger.warning( "Error building rpm %s [attempt #%s] in Brew: %s", rpm.qualified_name, attempt + 1, message, ) if attempt < retries - 1: # Brew does not handle an immediate retry correctly, wait before trying another build await asyncio.sleep(5 * 60) if not rpm.build_status: raise exectools.RetryException( f"Giving up after {retries} failed attempt(s): {message}", (task_ids, task_urls), ) return task_ids, task_urls, nvrs