def act(self, *args, **kwargs):
""" Run the modification action
:param ceiling_dir: If not None, prevent from writing to a directory that is out of ceiling_dir.
:param session: If not None, a requests.Session object for HTTP requests
"""
LOGGER.debug("Running 'add' modification action...")
context = kwargs["context"]
distgit_path = context['distgit_path']
source = urlparse(self.source)
if source.scheme not in self.SUPPORTED_URL_SCHEMES:
raise ValueError(
"Unsupported URL scheme {} used in 'add' action.".format(source.scheme))
source_url = source.geturl() # normalized URL
path = str(distgit_path.joinpath(self.path))
ceiling_dir = kwargs.get("ceiling_dir")
session = kwargs.get("session") or requests.session()
if ceiling_dir and not is_in_directory(path, ceiling_dir):
raise ValueError("Writing to a file out of {} is not allowed.".format(ceiling_dir))
# NOTE: `overwriting` is checked before writing.
# Data race might happen but it should suffice for prevent from accidently overwriting in-tree sources.
if not self.overwriting and os.path.exists(path):
raise IOError(
"Destination path {} exists. Use 'overwriting: true' to overwrite.".format(self.path))
LOGGER.debug("Getting out-of-tree source {}...".format(source_url))
response = session.get(source_url)
response.raise_for_status()
mkdirs(os.path.dirname(path))
with io.open(path, "wb") as dest_file:
dest_file.write(response.content)
LOGGER.debug("Out-of-tree source saved: {} -> {}".format(source_url, path))
def act(self, *args, **kwargs):
""" Run the command
:param context: A context dict. `context.set_env` is a `dict` of env vars to set for command (overriding existing).
"""
context = kwargs["context"]
component = context["component_name"]
distgit_path = Path(context['distgit_path'])
ceiling_dir = Path(kwargs["ceiling_dir"])
LOGGER.info("Distgit repo %s: Running 'remove' modification action...",
component)
removed = []
for path in distgit_path.rglob(self.glob):
if not is_in_directory(path, ceiling_dir):
raise PermissionError(
"Removing a file from a location outside of directory {} is not allowed."
.format(ceiling_dir))
relative_path = str(path.relative_to(distgit_path))
LOGGER.info("Distgit repo %s: Removing %s", component,
relative_path)
path.unlink()
removed.append(relative_path)
if len(removed):
LOGGER.info("Distgit repo %s: %s files have been removed:\n%s",
component, len(removed), "\n".join(removed))
else:
LOGGER.warning("Distgit repo %s: No files matching glob `%s`",
component, self.glob)
async def rebase(self, rpm: RPMMetadata, version: str,
release: str) -> str:
""" Rebases and pushes the distgit repo for an rpm
:param rpm: Metadata of the rpm
:param version: Set rpm version
:param release: Set rpm release
:return: Hash of the new distgit commit
"""
logger = rpm.logger
# clone source and distgit
logger.info("Cloning source and distgit repos...")
dg: RPMDistGitRepo
_, dg = await asyncio.gather(
exectools.to_thread(rpm.clone_source),
exectools.to_thread(rpm.distgit_repo, autoclone=True),
)
# cleanup distgit dir
logger.info("Cleaning up distgit repo...")
await exectools.cmd_assert_async(
["git", "reset", "--hard", "origin/" + dg.branch],
cwd=dg.distgit_dir)
await exectools.cmd_assert_async(
["git", "rm", "--ignore-unmatch", "-rf", "."], cwd=dg.distgit_dir)
# set .p0/.p1 flag
if self._runtime.group_config.public_upstreams:
if not release.endswith(".p?"):
raise ValueError(
f"'release' must end with '.p?' for an rpm with a public upstream but its actual value is {release}"
)
if rpm.private_fix is None:
raise AssertionError(
"rpm.private_fix flag should already be set")
if rpm.private_fix:
logger.warning("Source contains embargoed fixes.")
pval = ".p1"
else:
pval = ".p0"
release = release[:-3] + pval
# include commit hash in release field
release += ".g" + rpm.pre_init_sha[:7]
if self._runtime.assembly:
release += f'.assembly.{self._runtime.assembly}'
rpm.set_nvr(version, release)
# generate new specfile
tarball_name = f"{rpm.config.name}-{rpm.version}-{rpm.release}.tar.gz"
logger.info("Creating rpm spec file...")
source_commit_url = '{}/commit/{}'.format(rpm.public_upstream_url,
rpm.pre_init_sha)
specfile = await self._populate_specfile_async(rpm, tarball_name,
source_commit_url)
dg_specfile_path = dg.dg_path / Path(rpm.specfile).name
async with aiofiles.open(dg_specfile_path, "w") as f:
await f.writelines(specfile)
if rpm.get_package_name_from_spec() != rpm.get_package_name():
raise IOError(
f'RPM package name in .spec file ({rpm.get_package_name_from_spec()}) does not match doozer metadata name {rpm.get_package_name()}'
)
rpm.specfile = str(dg_specfile_path)
# create tarball source as Source0
logger.info("Creating tarball source...")
tarball_path = dg.dg_path / tarball_name
await exectools.cmd_assert_async(
[
"tar",
"-czf",
tarball_path,
"--exclude=.git",
fr"--transform=s,^\./,{rpm.config.name}-{rpm.version}/,",
".",
],
cwd=rpm.source_path,
)
logger.info(
"Done creating tarball source. Uploading to distgit lookaside cache..."
)
if self._push:
if not self._dry_run:
await exectools.cmd_assert_async(
["rhpkg", "new-sources", tarball_name],
cwd=dg.dg_path,
retries=3)
else:
async with aiofiles.open(dg.dg_path / "sources", "w") as f:
f.write("SHA512 ({}) = {}\n".format(
tarball_name, "0" * 128))
if self._push:
logger.warning("DRY RUN - Would have uploaded %s",
tarball_name)
# copy Source1, Source2,... and Patch0, Patch1,...
logger.info("Determining additional sources and patches...")
out, _ = await exectools.cmd_assert_async(
["spectool", "--", dg_specfile_path], cwd=dg.dg_path)
for line in out.splitlines():
line_split = line.split(": ")
if len(line_split) < 2 or line_split[0] == "Source0":
continue
filename = line_split[1].strip()
src = Path(rpm.source_path, filename)
# For security, ensure the source file referenced by the specfile is contained in both source and distgit directories.
if not is_in_directory(src, rpm.source_path):
raise ValueError(
"STOP! Source file {} referenced in Specfile {} lives outside of the source directory {}"
.format(filename, dg_specfile_path, rpm.source_path))
dest = dg.dg_path / filename
if not is_in_directory(dest, dg.dg_path):
raise ValueError(
"STOP! Source file {} referenced in Specfile {} would be copied to a directory outside of distgit directory {}"
.format(filename, dg_specfile_path, dg.dg_path))
dest.parent.mkdir(parents=True, exist_ok=True)
logging.debug("Copying %s", filename)
shutil.copy(src, dest, follow_symlinks=False)
# run modifications
if rpm.config.content.source.modifications is not Missing:
logging.info("Running custom modifications...")
await exectools.to_thread(rpm._run_modifications, dg_specfile_path,
dg.dg_path)
# commit changes
logging.info("Committing distgit changes...")
await aiofiles.os.remove(tarball_path)
commit_hash = await exectools.to_thread(
dg.commit,
f"Automatic commit of package [{rpm.config.name}] release [{rpm.version}-{rpm.release}].",
commit_attributes={
'version': rpm.version,
'release': rpm.release,
'io.openshift.build.commit.id': rpm.pre_init_sha,
'io.openshift.build.source-location': rpm.public_upstream_url,
})
if self._push:
# push
if not self._dry_run:
await dg.push_async()
else:
logger.warning("Would have pushed %s", dg.name)
return commit_hash