def _populate_distro_dir(self):
"""Copy files used in the distro to package and test the software to .distro.
Raises:
PackitException, if the dist-git repository is not pristine, that is,
there are changed or untracked files in it.
"""
dist_git_is_pristine = (not self.dist_git.git.diff()
and not self.dist_git.git.clean("-xdn"))
if not dist_git_is_pristine:
raise PackitException(
"Cannot initialize a source-git repo. "
"The corresponding dist-git repository at "
f"{self.dist_git.working_dir!r} is not pristine."
"Use 'git reset --hard HEAD' to reset changed files and "
"'git clean -xdff' to delete untracked files and directories.")
command = ["rsync", "--archive", "--delete"]
for exclude in ["*.patch", "sources", ".git*"]:
command += ["--filter", f"exclude {exclude}"]
command += [
str(self.dist_git.working_dir) + "/",
str(self.distro_dir),
]
self.distro_dir.mkdir(parents=True)
run_command(command)
def bump_spec(
self,
version: str = None,
changelog_entry: str = None,
bump_release: bool = False,
):
"""
Run rpmdev-bumpspec on the upstream spec file: it enables
changing version and adding a changelog entry
:param version: new version which should be present in the spec
:param changelog_entry: new changelog entry (just the comment)
:param bump_release: "bump trailing .<DIGIT> component if found, append .1 if not"
from the rpmdev-bumpspec --help
"""
cmd = ["rpmdev-bumpspec"]
if version:
# 1.2.3-4 means, version = 1.2.3, release = 4
cmd += ["--new", version]
if changelog_entry:
cmd += ["--comment", changelog_entry]
if bump_release:
cmd += ["-r"]
cmd.append(str(self.absolute_specfile_path))
run_command(cmd)
def with_action(self, action_name: str) -> bool:
"""
If the action is defined in the self.package_config.actions,
we run it and return False (so we can skip the if block)
If the action is not defined, return True.
Usage:
> if self._with_action(action_name="patch"):
> # Run default implementation
>
> # Custom command was run if defined in the config
Context manager is currently not possible without ugly hacks:
https://stackoverflow.com/questions/12594148/skipping-execution-of-with-block
https://www.python.org/dev/peps/pep-0377/ (rejected)
:param action_name: str (Name of the action that can be overwritten
in the package_config.actions)
:return: True, if the action is not overwritten, False when custom command was run
"""
logger.debug(f"Running {action_name}.")
if action_name in self.actions:
command = self.actions[action_name]
logger.info(f"Using user-defined script for {action_name}: {command}")
run_command(cmd=command)
return False
logger.debug(f"Running default implementation for {action_name}.")
return True
def create_archive(self):
"""
Create archive, using `git archive` by default, from the content of the upstream
repository, only committed changes are present in the archive
"""
if self.with_action(action=ActionName.create_archive):
if self.package_config.upstream_project_name:
dir_name = (f"{self.package_config.upstream_project_name}"
f"-{self.get_current_version()}")
else:
dir_name = f"{self.package_name}-{self.get_current_version()}"
logger.debug("name + version = %s", dir_name)
# We don't care about the name of the archive, really
# we just require for the archive to be placed in the cwd
if self.package_config.create_tarball_command:
archive_cmd = self.package_config.create_tarball_command
else:
# FIXME: .tar.gz is naive
archive_name = f"{dir_name}.tar.gz"
archive_cmd = [
"git",
"archive",
"-o",
archive_name,
"--prefix",
f"{dir_name}/",
"HEAD",
]
run_command(archive_cmd)
def _run_prep(self):
"""
run `rpmbuild -bp` in the dist-git repo to get a git-repo
in the %prep phase so we can pick the commits in the source-git repo
"""
_packitpatch_path = shutil.which("_packitpatch")
if not _packitpatch_path:
raise PackitException(
"We are trying to unpack a dist-git archive and lay patches on top "
'by running `rpmbuild -bp` but we cannot find "_packitpatch" command on PATH: '
"please install packit as an RPM.")
logger.info(f"expanding %prep section in {self.dist_git.working_dir}")
rpmbuild_args = [
"rpmbuild",
"--nodeps",
"--define",
f"_topdir {str(self.dist_git.working_dir)}",
"-bp",
"--define",
f"_specdir {self.dist_git.working_dir}",
"--define",
f"_sourcedir {self.dist_git.working_dir}",
]
rpmbuild_args += RPM_MACROS_FOR_PREP
if logger.level <= logging.DEBUG: # -vv can be super-duper verbose
rpmbuild_args.append("-v")
rpmbuild_args.append(str(self.dist_git_specfile.path))
run_command(
rpmbuild_args,
cwd=self.dist_git.working_dir,
print_live=True,
)
def _populate_distro_dir(self):
"""Copy files used in the distro to package and test the software to .distro.
Raises:
PackitException, if the dist-git repository is not pristine, that is,
there are changed or untracked files in it.
"""
if not is_the_repo_pristine(self.dist_git):
raise PackitException(
"Cannot initialize a source-git repo. "
"The corresponding dist-git repository at "
f"{self.dist_git.working_dir!r} is not pristine. "
f"{REPO_NOT_PRISTINE_HINT}")
command = ["rsync", "--archive", "--delete"]
for exclude in ["*.patch", "sources", ".git*"]:
command += ["--filter", f"exclude {exclude}"]
command += [
str(self.dist_git.working_dir) + "/",
str(self.distro_dir),
]
self.distro_dir.mkdir(parents=True)
run_command(command)
def sync_files(synced_files: Sequence[SyncFilesItem]):
"""
Copy files b/w upstream and downstream repo.
"""
for item in synced_files:
command = item.command()
logger.debug(f"Running {command!r} ...")
run_command(command, print_live=True)
def test_run_cmd_unicode(tmp_path):
# don't ask me what this is, I took it directly from systemd's test suite
# that's what packit was UnicodeDecodeError-ing on
cancer = (
b"\x06\xd0\xf1\t\x01\xa1\x01\t "
b"\x15\x00&\xff\x00u\x08\x95@\x81\x02\t!\x15\x00&\xff\x00u\x08\x95@\x91\x02\xc0"
)
t = tmp_path / "the-cancer"
t.write_bytes(cancer)
command = ["cat", str(t)]
assert cancer == run_command(command, decode=False, output=True)
with pytest.raises(UnicodeDecodeError):
run_command(command, decode=True, output=True)
def linearize_history(git_ref: str):
r"""
Transform complex git history into a linear one starting from a selected git ref.
Change this:
* | | | | | | | | ea500ac513 (tag: v245) Merge pull request #15...
|\ \ \ \ \ \ \ \ \
| * | | | | | | | | 0d5aef3eb5 hwdb: update for v245
| | |_|_|_|_|_|/ /
| |/| | | | | | |
* | | | | | | | | 03985d069b NEWS: final contributor update for v245
Into this:
* 0d5aef3eb5 hwdb: update for v245
* 03985d069b NEWS: final contributor update for v245
"""
logger.info(
"When git history is too complex with merge commits having parents \n"
"across a wide range, git is known to produce patches which cannot be applied. \n"
"Therefore we are going to make the history linear on a dedicated branch \n"
"to make sure the patches will be able to be applied.")
current_time = datetime.datetime.now().strftime(DATETIME_FORMAT)
target_branch = f"packit-patches-{current_time}"
logger.info(f"Switch branch to {target_branch!r}.")
run_command(["git", "checkout", "-B", target_branch])
target = f"{git_ref}..HEAD"
logger.debug(f"Linearize history {target}.")
# https://stackoverflow.com/a/17994534/909579
# With this command we will rewrite git history of our newly created branch
# by dropping the merge commits and setting parent commits to those from target branch
# this means we will drop the reference from which we are merging
# filter branch passes these to cut:
# ` -p 61f3e897f13101f29fb8027e8839498a469ad58e`
# ` -p b7cf4b4ef5d0336443f21809b1506bc4a8aa75a9 -p 257188f80ce1a083e3a88b679b898a7...`
# so we will keep the first parent and drop all the others
run_command(
[
"git",
"filter-branch",
"-f",
"--parent-filter",
'cut -f 2,3 -d " "',
target,
],
# git prints nasty warning when filter-branch is used that it's dangerous
# this env var prevents it from prints
env={"FILTER_BRANCH_SQUELCH_WARNING": "1"},
)
def fix_spec(self, archive: str, version: str, commit: str):
"""
In order to create a SRPM from current git checkout, we need to have the spec reference
the tarball and unpack it. This method updates the spec so it's possible.
:param archive: relative path to the archive: used as Source0
:param version: version to set in the spec
:param commit: commit to set in the changelog
"""
self._fix_spec_source(archive)
self._fix_spec_prep(version)
# we only care about the first number in the release
# so that we can re-run `packit srpm`
git_des_command = [
"git",
"describe",
"--tags",
"--long",
"--match",
"*",
]
try:
git_des_out = run_command(git_des_command, output=True).strip()
except PackitCommandFailedError as ex:
logger.info(f"Exception while describing the repository: {ex}")
# probably no tags in the git repo
git_desc_suffix = ""
else:
# git adds various info in the output separated by -
# so let's just drop version and reuse everything else
g_desc_raw = git_des_out.rsplit("-", 2)[1:]
# release components are meant to be separated by ".", not "-"
git_desc_suffix = "." + ".".join(g_desc_raw)
try:
all_pr_list = self.local_project.git_service.get_project(
).get_pr_list()
current_branch = self._local_project.git_repo.active_branch
pr_id = [
p.id for p in all_pr_list if p.source_branch == current_branch
]
except Exception as pr_id_error:
logger.info(
f"Exception while detecting pull request if for current brand: {pr_id_error}"
)
pr_id = ""
original_release_number = self.specfile.get_release_number().split(
".", 1)[0]
current_time = datetime.datetime.now().strftime(DATETIME_FORMAT)
release = f"{original_release_number}.{current_time}{git_desc_suffix}{pr_id}"
msg = f"- Development snapshot ({commit})"
logger.debug(f"Setting Release in spec to {release!r}")
# instead of changing version, we change Release field
# upstream projects should take care of versions
self.specfile.set_spec_version(
version=version,
release=release,
changelog_entry=msg,
)
def create_patches(
self, upstream: str = None, destination: str = None
) -> List[Tuple[str, str]]:
"""
Create patches from downstream commits.
:param destination: str
:param upstream: str -- git branch or tag
:return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg)
"""
upstream = upstream or self.get_specfile_version()
commits = self.get_commits_to_upstream(upstream, add_upstream_head_commit=True)
destination = destination or self.local_project.working_dir
patches_to_create = []
for i, commit in enumerate(commits[1:]):
parent = commits[i]
git_diff_cmd = [
"git",
"diff",
"--patch",
parent.hexsha,
commit.hexsha,
"--",
".",
] + [
f":(exclude){sync_file.src}"
for sync_file in self.package_config.synced_files.get_raw_files_to_sync(
Path(self.local_project.working_dir),
Path(
# this is not important, we only care about src
destination
),
)
]
diff = run_command(
cmd=git_diff_cmd, cwd=self.local_project.working_dir, output=True
)
if not diff:
logger.info(f"No patch for commit: {commit.summary} ({commit.hexsha})")
continue
patches_to_create.append((commit, diff))
patch_list = []
for i, (commit, diff) in enumerate(patches_to_create):
patch_name = f"{i + 1:04d}-{commit.hexsha}.patch"
patch_path = os.path.join(destination, patch_name)
patch_msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>"
logger.debug(f"Saving patch: {patch_name}\n{patch_msg}")
with open(patch_path, mode="w") as patch_file:
patch_file.write(diff)
patch_list.append((patch_name, patch_msg))
return patch_list
def create_patches(self,
upstream: str = None,
destination: str = None) -> List[Tuple[str, str]]:
"""
Create patches from downstream commits.
:param destination: str
:param upstream: str -- git branch or tag
:return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg)
"""
upstream = upstream or self.get_specfile_version()
destination = destination or self.local_project.working_dir
commits = self.get_commits_to_upstream(upstream,
add_upstream_head_commit=True)
sync_files_to_ignore = [
str(sf.src.relative_to(self.local_project.working_dir)) for sf in
self.package_config.get_all_files_to_sync().get_raw_files_to_sync(
Path(self.local_project.working_dir),
Path(
# dest (downstream) is not important, we only care about src (upstream)
destination),
)
]
files_to_ignore = (self.package_config.patch_generation_ignore_paths +
sync_files_to_ignore)
patch_list = []
# first value is upstream ref, i.e parent for the first commit we want to create patch from
for i, commit in enumerate(commits[1:]):
parent_commit = commits[i]
git_diff_cmd = [
"git",
"format-patch",
"--output-directory",
f"{destination}",
f"{parent_commit.hexsha}..{commit.hexsha}",
"--",
".",
] + [
f":(exclude){file_to_ignore}"
for file_to_ignore in files_to_ignore
]
patch_file = run_command(
cmd=git_diff_cmd,
cwd=self.local_project.working_dir,
output=True,
decode=True,
)
if patch_file:
msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>"
patch_list.append((os.path.basename(patch_file.strip()), msg))
else:
logger.info(
f"No patch for commit: {commit.summary} ({commit.hexsha})")
return patch_list
def create_srpm(self) -> str:
logger.debug("Start creating of the SRPM.")
archive = self.create_archive()
logger.debug(f"Using archive: {archive}")
output = run_command(
cmd=[
"rpmbuild",
"-bs",
f"{self.dist_specfile_path}",
"--define",
f"_sourcedir {self.distgit.working_dir}",
"--define",
f"_specdir {self.distgit.working_dir}",
"--define",
f"_buildir {self.distgit.working_dir}",
"--define",
f"_srcrpmdir {self.distgit.working_dir}",
"--define",
f"_rpmdir {self.distgit.working_dir}",
],
fail=True,
output=True,
)
specfile_name = output.split(':')[1].rstrip()
logger.info(f"Specfile created: {specfile_name}")
return specfile_name
def bump_spec(self, version: str = None, changelog_entry: str = None):
"""
Run rpmdev-bumpspec on the upstream spec file: it enables
changing version and adding a changelog entry
:param version: new version which should be present in the spec
:param changelog_entry: new changelog entry (just the comment)
"""
cmd = ["rpmdev-bumpspec"]
if version:
# 1.2.3-4 means, version = 1.2.3, release = 4
cmd += ["--new", version]
if changelog_entry:
cmd += ["--comment", changelog_entry]
cmd.append(str(self.absolute_specfile_path))
run_command(cmd)
def clone(self,
package_name: str,
target_path: str,
anonymous: bool = False):
"""
clone a dist-git repo; this has to be done in current env
b/c we don't have the keytab in sandbox
"""
cmd = [self.fedpkg_exec]
if self.fas_username:
cmd += ["--user", self.fas_username]
cmd += ["-q", "clone"]
if anonymous:
cmd += ["-a"]
cmd += [package_name, target_path]
utils.run_command(cmd=cmd)
def get_output_from_action(self, action: ActionName):
"""
Run action if specified in the self.actions and return output
else return None
"""
if action in self.package_config.actions:
command = self.package_config.actions[action]
logger.info(f"Using user-defined script for {action}: {command}")
return run_command(cmd=command, output=True)
return None
def new_sources(self, sources="", fail=True):
if not Path(self.directory).is_dir():
raise Exception("Cannot access fedpkg repository:")
return run_command(
cmd=[self.fedpkg_exec, "new-sources", sources],
cwd=self.directory,
error_message=f"Adding new sources failed:",
fail=fail,
)
def clone(self, package_name: str, target_path: str, anonymous: bool = False):
"""
clone a dist-git repo; this has to be done in current env
b/c we don't have the keytab in sandbox
"""
cmd = [self.fedpkg_exec]
if self.fas_username:
cmd += ["--user", self.fas_username]
cmd += ["-q", "clone"]
if anonymous:
cmd += ["-a"]
cmd += [package_name, target_path]
error_msg = (
f"Packit failed to clone the repository {package_name}; please make sure that you"
f"authorized to clone repositories from fedora dist-git - this may require"
f"SSH keys set up or Kerberos ticket being active."
)
utils.run_command(cmd=cmd, error_message=error_msg)
def run_command(self, command: List[str], return_output=True):
"""
exec a command
:param command: the command
:param return_output: return output from this method if True
"""
return utils.run_command(cmd=command,
cwd=self.local_project.working_dir,
output=return_output)
def build(
self,
scratch: bool = False,
nowait: bool = False,
koji_target: Optional[str] = None,
):
cmd = [self.fedpkg_exec, "build"]
if scratch:
cmd.append("--scratch")
if nowait:
cmd.append("--nowait")
if koji_target:
cmd += ["--target", koji_target]
utils.run_command(
cmd=cmd,
cwd=self.directory,
error_message="Submission of build to koji failed.",
fail=True,
)
def get_output_from_action(self, action_name: str):
"""
Run action if specified in the self.actions and return output
else return None
"""
if action_name in self.actions:
command = self.actions[action_name]
logger.info(f"Using user-defined script for {action_name}: {command}")
return run_command(cmd=command, output=True)
return None
def build(self, scratch: bool = False):
cmd = [self.fedpkg_exec, "build", "--nowait"]
if scratch:
cmd.append("--scratch")
out = run_command(
cmd=cmd,
cwd=self.directory,
error_message="Submission of build to koji failed.",
fail=True,
output=True,
)
logger.info("%s", out)
def get_last_tag(self) -> Optional[str]:
""" get last git-tag from the repo """
try:
last_tag = run_command(
["git", "describe", "--tags", "--abbrev=0"],
output=True,
cwd=self.local_project.working_dir,
).strip()
except PackitCommandFailedError as ex:
logger.debug(f"{ex!r}")
logger.info("Can't describe this repository, are there any git tags?")
# no tags in the git repo
return None
return last_tag
def create_srpm(self, srpm_path: str = None) -> Path:
"""
Create SRPM from the actual content of the repo
:param srpm_path: path to the srpm
:return: path to the srpm
"""
cwd = self.local_project.working_dir
cmd = [
"rpmbuild",
"-bs",
"--define",
f"_sourcedir {cwd}",
"--define",
f"_specdir {cwd}",
"--define",
f"_srcrpmdir {os.getcwd()}",
# no idea about this one, but tests were failing in tox w/o it
"--define",
f"_topdir {cwd}",
# we also need these 3 so that rpmbuild won't create them
"--define",
f"_builddir {cwd}",
"--define",
f"_rpmdir {cwd}",
"--define",
f"_buildrootdir {cwd}",
self.specfile_path,
]
present_srpms = set(Path.cwd().glob("*.src.rpm"))
logger.debug("present srpms = %s", present_srpms)
out = run_command(
cmd,
output=True,
error_message="SRPM could not be created. Is the archive present?",
cwd=self.local_project.working_dir,
).strip()
logger.debug(f"{out}")
# not doing 'Wrote: (.+)' since people can have different locales; hi Franto!
reg = r": (.+\.src\.rpm)$"
try:
the_srpm = re.findall(reg, out)[0]
except IndexError:
raise PackitException("SRPM cannot be found, something is wrong.")
if srpm_path:
shutil.move(the_srpm, srpm_path)
return Path(srpm_path)
return Path(the_srpm)
def create_patches(self,
upstream: str = None,
destination: str = None) -> List[Tuple[str, str]]:
"""
Create patches from downstream commits.
:param destination: str
:param upstream: str -- git branch or tag
:return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg)
"""
upstream = upstream or self.get_specfile_version()
commits = self.get_commits_to_upstream(upstream,
add_usptream_head_commit=True)
patch_list = []
destination = destination or self.local_project.working_dir
for i, commit in enumerate(commits[1:]):
parent = commits[i]
patch_name = f"{i + 1:04d}-{commit.hexsha}.patch"
patch_path = os.path.join(destination, patch_name)
patch_msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>"
logger.debug(f"PATCH: {patch_name}\n{patch_msg}")
git_diff_cmd = [
"git",
"diff",
"--patch",
parent.hexsha,
commit.hexsha,
"--",
".",
] + [
f":(exclude){sync_file.src}" for sync_file in
self.package_config.synced_files.raw_files_to_sync
]
diff = run_command(cmd=git_diff_cmd,
cwd=self.local_project.working_dir,
output=True)
with open(patch_path, mode="w") as patch_file:
patch_file.write(diff)
patch_list.append((patch_name, patch_msg))
return patch_list
def run_command(self,
command: List[str],
return_output: bool = True,
env: Optional[Dict] = None):
"""
exec a command
:param command: the command
:param return_output: return output from this method if True
:param env: dict with env vars to set for the command
"""
return utils.run_command(
cmd=command,
cwd=self.local_project.working_dir,
output=return_output,
env=env,
)
def version_from_specfile(self) -> str:
"""
Version extracted from the specfile.
"""
version_raw = run_command(
cmd=[
"rpmspec",
"-q",
"--qf",
"'%{version}\\n'",
"--srpm",
self.source_specfile_path,
],
output=True,
fail=True,
)
version = version_raw.strip("'\\\n")
return version
def get_current_version(self) -> str:
"""
Get version of the project in current state (hint `git describe`)
:return: e.g. 0.1.1.dev86+ga17a559.d20190315 or 0.6.1.1.gce4d84e
"""
action_output = self.get_output_from_action(
action=ActionName.get_current_version)
if action_output:
return action_output
ver = run_command(self.package_config.current_version_command,
output=True).strip()
logger.debug("version = %s", ver)
# FIXME: this might not work when users expect the dashes
# but! RPM refuses dashes in version/release
ver = ver.replace("-", ".")
logger.debug("sanitized version = %s", ver)
return ver
def init_ticket(self, keytab: str = None):
# TODO: this method has nothing to do with fedpkg, pull it out
if not keytab:
logger.info("won't be doing kinit, no credentials provided")
return
if keytab and Path(keytab).is_file():
cmd = [
"kinit",
f"{self.fas_username}@FEDORAPROJECT.ORG",
"-k",
"-t",
keytab,
]
else:
# there is no keytab, but user still might have active ticket - try to renew it
cmd = ["kinit", "-R", f"{self.fas_username}@FEDORAPROJECT.ORG"]
return run_command(cmd=cmd,
error_message="Failed to init kerberos ticket:",
fail=True)
def create_patches(self, upstream: str = None) -> List[Tuple[str, str]]:
"""
Create patches from downstream commits.
:param upstream: str -- git branch or tag
:return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg)
"""
upstream = upstream or self.version_from_specfile
commits = self.get_commits_to_upstream(upstream, add_usptream_head_commit=True)
patch_list = []
for i, commit in enumerate(commits[1:]):
parent = commits[i]
patch_name = f"{i + 1:04d}-{commit.hexsha}.patch"
patch_path = os.path.join(self.distgit.working_dir, patch_name)
patch_msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>"
logger.debug(f"PATCH: {patch_name}\n{patch_msg}")
diff = run_command(
cmd=[
"git",
"diff",
"--patch",
parent.hexsha,
commit.hexsha,
"--",
".",
'":(exclude)redhat"',
],
cwd=self.sourcegit.working_dir,
output=True,
)
with open(patch_path, mode="w") as patch_file:
patch_file.write(diff)
patch_list.append((patch_name, patch_msg))
return patch_list
