def _populate_distro_dir(self): """Copy files used in the distro to package and test the software to .distro. Raises: PackitException, if the dist-git repository is not pristine, that is, there are changed or untracked files in it. """ dist_git_is_pristine = (not self.dist_git.git.diff() and not self.dist_git.git.clean("-xdn")) if not dist_git_is_pristine: raise PackitException( "Cannot initialize a source-git repo. " "The corresponding dist-git repository at " f"{self.dist_git.working_dir!r} is not pristine." "Use 'git reset --hard HEAD' to reset changed files and " "'git clean -xdff' to delete untracked files and directories.") command = ["rsync", "--archive", "--delete"] for exclude in ["*.patch", "sources", ".git*"]: command += ["--filter", f"exclude {exclude}"] command += [ str(self.dist_git.working_dir) + "/", str(self.distro_dir), ] self.distro_dir.mkdir(parents=True) run_command(command)
def bump_spec( self, version: str = None, changelog_entry: str = None, bump_release: bool = False, ): """ Run rpmdev-bumpspec on the upstream spec file: it enables changing version and adding a changelog entry :param version: new version which should be present in the spec :param changelog_entry: new changelog entry (just the comment) :param bump_release: "bump trailing .<DIGIT> component if found, append .1 if not" from the rpmdev-bumpspec --help """ cmd = ["rpmdev-bumpspec"] if version: # 1.2.3-4 means, version = 1.2.3, release = 4 cmd += ["--new", version] if changelog_entry: cmd += ["--comment", changelog_entry] if bump_release: cmd += ["-r"] cmd.append(str(self.absolute_specfile_path)) run_command(cmd)
def with_action(self, action_name: str) -> bool: """ If the action is defined in the self.package_config.actions, we run it and return False (so we can skip the if block) If the action is not defined, return True. Usage: > if self._with_action(action_name="patch"): > # Run default implementation > > # Custom command was run if defined in the config Context manager is currently not possible without ugly hacks: https://stackoverflow.com/questions/12594148/skipping-execution-of-with-block https://www.python.org/dev/peps/pep-0377/ (rejected) :param action_name: str (Name of the action that can be overwritten in the package_config.actions) :return: True, if the action is not overwritten, False when custom command was run """ logger.debug(f"Running {action_name}.") if action_name in self.actions: command = self.actions[action_name] logger.info(f"Using user-defined script for {action_name}: {command}") run_command(cmd=command) return False logger.debug(f"Running default implementation for {action_name}.") return True
def create_archive(self): """ Create archive, using `git archive` by default, from the content of the upstream repository, only committed changes are present in the archive """ if self.with_action(action=ActionName.create_archive): if self.package_config.upstream_project_name: dir_name = (f"{self.package_config.upstream_project_name}" f"-{self.get_current_version()}") else: dir_name = f"{self.package_name}-{self.get_current_version()}" logger.debug("name + version = %s", dir_name) # We don't care about the name of the archive, really # we just require for the archive to be placed in the cwd if self.package_config.create_tarball_command: archive_cmd = self.package_config.create_tarball_command else: # FIXME: .tar.gz is naive archive_name = f"{dir_name}.tar.gz" archive_cmd = [ "git", "archive", "-o", archive_name, "--prefix", f"{dir_name}/", "HEAD", ] run_command(archive_cmd)
def _run_prep(self): """ run `rpmbuild -bp` in the dist-git repo to get a git-repo in the %prep phase so we can pick the commits in the source-git repo """ _packitpatch_path = shutil.which("_packitpatch") if not _packitpatch_path: raise PackitException( "We are trying to unpack a dist-git archive and lay patches on top " 'by running `rpmbuild -bp` but we cannot find "_packitpatch" command on PATH: ' "please install packit as an RPM.") logger.info(f"expanding %prep section in {self.dist_git.working_dir}") rpmbuild_args = [ "rpmbuild", "--nodeps", "--define", f"_topdir {str(self.dist_git.working_dir)}", "-bp", "--define", f"_specdir {self.dist_git.working_dir}", "--define", f"_sourcedir {self.dist_git.working_dir}", ] rpmbuild_args += RPM_MACROS_FOR_PREP if logger.level <= logging.DEBUG: # -vv can be super-duper verbose rpmbuild_args.append("-v") rpmbuild_args.append(str(self.dist_git_specfile.path)) run_command( rpmbuild_args, cwd=self.dist_git.working_dir, print_live=True, )
def _populate_distro_dir(self): """Copy files used in the distro to package and test the software to .distro. Raises: PackitException, if the dist-git repository is not pristine, that is, there are changed or untracked files in it. """ if not is_the_repo_pristine(self.dist_git): raise PackitException( "Cannot initialize a source-git repo. " "The corresponding dist-git repository at " f"{self.dist_git.working_dir!r} is not pristine. " f"{REPO_NOT_PRISTINE_HINT}") command = ["rsync", "--archive", "--delete"] for exclude in ["*.patch", "sources", ".git*"]: command += ["--filter", f"exclude {exclude}"] command += [ str(self.dist_git.working_dir) + "/", str(self.distro_dir), ] self.distro_dir.mkdir(parents=True) run_command(command)
def sync_files(synced_files: Sequence[SyncFilesItem]): """ Copy files b/w upstream and downstream repo. """ for item in synced_files: command = item.command() logger.debug(f"Running {command!r} ...") run_command(command, print_live=True)
def test_run_cmd_unicode(tmp_path): # don't ask me what this is, I took it directly from systemd's test suite # that's what packit was UnicodeDecodeError-ing on cancer = ( b"\x06\xd0\xf1\t\x01\xa1\x01\t " b"\x15\x00&\xff\x00u\x08\x95@\x81\x02\t!\x15\x00&\xff\x00u\x08\x95@\x91\x02\xc0" ) t = tmp_path / "the-cancer" t.write_bytes(cancer) command = ["cat", str(t)] assert cancer == run_command(command, decode=False, output=True) with pytest.raises(UnicodeDecodeError): run_command(command, decode=True, output=True)
def linearize_history(git_ref: str): r""" Transform complex git history into a linear one starting from a selected git ref. Change this: * | | | | | | | | ea500ac513 (tag: v245) Merge pull request #15... |\ \ \ \ \ \ \ \ \ | * | | | | | | | | 0d5aef3eb5 hwdb: update for v245 | | |_|_|_|_|_|/ / | |/| | | | | | | * | | | | | | | | 03985d069b NEWS: final contributor update for v245 Into this: * 0d5aef3eb5 hwdb: update for v245 * 03985d069b NEWS: final contributor update for v245 """ logger.info( "When git history is too complex with merge commits having parents \n" "across a wide range, git is known to produce patches which cannot be applied. \n" "Therefore we are going to make the history linear on a dedicated branch \n" "to make sure the patches will be able to be applied.") current_time = datetime.datetime.now().strftime(DATETIME_FORMAT) target_branch = f"packit-patches-{current_time}" logger.info(f"Switch branch to {target_branch!r}.") run_command(["git", "checkout", "-B", target_branch]) target = f"{git_ref}..HEAD" logger.debug(f"Linearize history {target}.") # https://stackoverflow.com/a/17994534/909579 # With this command we will rewrite git history of our newly created branch # by dropping the merge commits and setting parent commits to those from target branch # this means we will drop the reference from which we are merging # filter branch passes these to cut: # ` -p 61f3e897f13101f29fb8027e8839498a469ad58e` # ` -p b7cf4b4ef5d0336443f21809b1506bc4a8aa75a9 -p 257188f80ce1a083e3a88b679b898a7...` # so we will keep the first parent and drop all the others run_command( [ "git", "filter-branch", "-f", "--parent-filter", 'cut -f 2,3 -d " "', target, ], # git prints nasty warning when filter-branch is used that it's dangerous # this env var prevents it from prints env={"FILTER_BRANCH_SQUELCH_WARNING": "1"}, )
def fix_spec(self, archive: str, version: str, commit: str): """ In order to create a SRPM from current git checkout, we need to have the spec reference the tarball and unpack it. This method updates the spec so it's possible. :param archive: relative path to the archive: used as Source0 :param version: version to set in the spec :param commit: commit to set in the changelog """ self._fix_spec_source(archive) self._fix_spec_prep(version) # we only care about the first number in the release # so that we can re-run `packit srpm` git_des_command = [ "git", "describe", "--tags", "--long", "--match", "*", ] try: git_des_out = run_command(git_des_command, output=True).strip() except PackitCommandFailedError as ex: logger.info(f"Exception while describing the repository: {ex}") # probably no tags in the git repo git_desc_suffix = "" else: # git adds various info in the output separated by - # so let's just drop version and reuse everything else g_desc_raw = git_des_out.rsplit("-", 2)[1:] # release components are meant to be separated by ".", not "-" git_desc_suffix = "." + ".".join(g_desc_raw) try: all_pr_list = self.local_project.git_service.get_project( ).get_pr_list() current_branch = self._local_project.git_repo.active_branch pr_id = [ p.id for p in all_pr_list if p.source_branch == current_branch ] except Exception as pr_id_error: logger.info( f"Exception while detecting pull request if for current brand: {pr_id_error}" ) pr_id = "" original_release_number = self.specfile.get_release_number().split( ".", 1)[0] current_time = datetime.datetime.now().strftime(DATETIME_FORMAT) release = f"{original_release_number}.{current_time}{git_desc_suffix}{pr_id}" msg = f"- Development snapshot ({commit})" logger.debug(f"Setting Release in spec to {release!r}") # instead of changing version, we change Release field # upstream projects should take care of versions self.specfile.set_spec_version( version=version, release=release, changelog_entry=msg, )
def create_patches( self, upstream: str = None, destination: str = None ) -> List[Tuple[str, str]]: """ Create patches from downstream commits. :param destination: str :param upstream: str -- git branch or tag :return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg) """ upstream = upstream or self.get_specfile_version() commits = self.get_commits_to_upstream(upstream, add_upstream_head_commit=True) destination = destination or self.local_project.working_dir patches_to_create = [] for i, commit in enumerate(commits[1:]): parent = commits[i] git_diff_cmd = [ "git", "diff", "--patch", parent.hexsha, commit.hexsha, "--", ".", ] + [ f":(exclude){sync_file.src}" for sync_file in self.package_config.synced_files.get_raw_files_to_sync( Path(self.local_project.working_dir), Path( # this is not important, we only care about src destination ), ) ] diff = run_command( cmd=git_diff_cmd, cwd=self.local_project.working_dir, output=True ) if not diff: logger.info(f"No patch for commit: {commit.summary} ({commit.hexsha})") continue patches_to_create.append((commit, diff)) patch_list = [] for i, (commit, diff) in enumerate(patches_to_create): patch_name = f"{i + 1:04d}-{commit.hexsha}.patch" patch_path = os.path.join(destination, patch_name) patch_msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>" logger.debug(f"Saving patch: {patch_name}\n{patch_msg}") with open(patch_path, mode="w") as patch_file: patch_file.write(diff) patch_list.append((patch_name, patch_msg)) return patch_list
def create_patches(self, upstream: str = None, destination: str = None) -> List[Tuple[str, str]]: """ Create patches from downstream commits. :param destination: str :param upstream: str -- git branch or tag :return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg) """ upstream = upstream or self.get_specfile_version() destination = destination or self.local_project.working_dir commits = self.get_commits_to_upstream(upstream, add_upstream_head_commit=True) sync_files_to_ignore = [ str(sf.src.relative_to(self.local_project.working_dir)) for sf in self.package_config.get_all_files_to_sync().get_raw_files_to_sync( Path(self.local_project.working_dir), Path( # dest (downstream) is not important, we only care about src (upstream) destination), ) ] files_to_ignore = (self.package_config.patch_generation_ignore_paths + sync_files_to_ignore) patch_list = [] # first value is upstream ref, i.e parent for the first commit we want to create patch from for i, commit in enumerate(commits[1:]): parent_commit = commits[i] git_diff_cmd = [ "git", "format-patch", "--output-directory", f"{destination}", f"{parent_commit.hexsha}..{commit.hexsha}", "--", ".", ] + [ f":(exclude){file_to_ignore}" for file_to_ignore in files_to_ignore ] patch_file = run_command( cmd=git_diff_cmd, cwd=self.local_project.working_dir, output=True, decode=True, ) if patch_file: msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>" patch_list.append((os.path.basename(patch_file.strip()), msg)) else: logger.info( f"No patch for commit: {commit.summary} ({commit.hexsha})") return patch_list
def create_srpm(self) -> str: logger.debug("Start creating of the SRPM.") archive = self.create_archive() logger.debug(f"Using archive: {archive}") output = run_command( cmd=[ "rpmbuild", "-bs", f"{self.dist_specfile_path}", "--define", f"_sourcedir {self.distgit.working_dir}", "--define", f"_specdir {self.distgit.working_dir}", "--define", f"_buildir {self.distgit.working_dir}", "--define", f"_srcrpmdir {self.distgit.working_dir}", "--define", f"_rpmdir {self.distgit.working_dir}", ], fail=True, output=True, ) specfile_name = output.split(':')[1].rstrip() logger.info(f"Specfile created: {specfile_name}") return specfile_name
def bump_spec(self, version: str = None, changelog_entry: str = None): """ Run rpmdev-bumpspec on the upstream spec file: it enables changing version and adding a changelog entry :param version: new version which should be present in the spec :param changelog_entry: new changelog entry (just the comment) """ cmd = ["rpmdev-bumpspec"] if version: # 1.2.3-4 means, version = 1.2.3, release = 4 cmd += ["--new", version] if changelog_entry: cmd += ["--comment", changelog_entry] cmd.append(str(self.absolute_specfile_path)) run_command(cmd)
def clone(self, package_name: str, target_path: str, anonymous: bool = False): """ clone a dist-git repo; this has to be done in current env b/c we don't have the keytab in sandbox """ cmd = [self.fedpkg_exec] if self.fas_username: cmd += ["--user", self.fas_username] cmd += ["-q", "clone"] if anonymous: cmd += ["-a"] cmd += [package_name, target_path] utils.run_command(cmd=cmd)
def get_output_from_action(self, action: ActionName): """ Run action if specified in the self.actions and return output else return None """ if action in self.package_config.actions: command = self.package_config.actions[action] logger.info(f"Using user-defined script for {action}: {command}") return run_command(cmd=command, output=True) return None
def new_sources(self, sources="", fail=True): if not Path(self.directory).is_dir(): raise Exception("Cannot access fedpkg repository:") return run_command( cmd=[self.fedpkg_exec, "new-sources", sources], cwd=self.directory, error_message=f"Adding new sources failed:", fail=fail, )
def clone(self, package_name: str, target_path: str, anonymous: bool = False): """ clone a dist-git repo; this has to be done in current env b/c we don't have the keytab in sandbox """ cmd = [self.fedpkg_exec] if self.fas_username: cmd += ["--user", self.fas_username] cmd += ["-q", "clone"] if anonymous: cmd += ["-a"] cmd += [package_name, target_path] error_msg = ( f"Packit failed to clone the repository {package_name}; please make sure that you" f"authorized to clone repositories from fedora dist-git - this may require" f"SSH keys set up or Kerberos ticket being active." ) utils.run_command(cmd=cmd, error_message=error_msg)
def run_command(self, command: List[str], return_output=True): """ exec a command :param command: the command :param return_output: return output from this method if True """ return utils.run_command(cmd=command, cwd=self.local_project.working_dir, output=return_output)
def build( self, scratch: bool = False, nowait: bool = False, koji_target: Optional[str] = None, ): cmd = [self.fedpkg_exec, "build"] if scratch: cmd.append("--scratch") if nowait: cmd.append("--nowait") if koji_target: cmd += ["--target", koji_target] utils.run_command( cmd=cmd, cwd=self.directory, error_message="Submission of build to koji failed.", fail=True, )
def get_output_from_action(self, action_name: str): """ Run action if specified in the self.actions and return output else return None """ if action_name in self.actions: command = self.actions[action_name] logger.info(f"Using user-defined script for {action_name}: {command}") return run_command(cmd=command, output=True) return None
def build(self, scratch: bool = False): cmd = [self.fedpkg_exec, "build", "--nowait"] if scratch: cmd.append("--scratch") out = run_command( cmd=cmd, cwd=self.directory, error_message="Submission of build to koji failed.", fail=True, output=True, ) logger.info("%s", out)
def get_last_tag(self) -> Optional[str]: """ get last git-tag from the repo """ try: last_tag = run_command( ["git", "describe", "--tags", "--abbrev=0"], output=True, cwd=self.local_project.working_dir, ).strip() except PackitCommandFailedError as ex: logger.debug(f"{ex!r}") logger.info("Can't describe this repository, are there any git tags?") # no tags in the git repo return None return last_tag
def create_srpm(self, srpm_path: str = None) -> Path: """ Create SRPM from the actual content of the repo :param srpm_path: path to the srpm :return: path to the srpm """ cwd = self.local_project.working_dir cmd = [ "rpmbuild", "-bs", "--define", f"_sourcedir {cwd}", "--define", f"_specdir {cwd}", "--define", f"_srcrpmdir {os.getcwd()}", # no idea about this one, but tests were failing in tox w/o it "--define", f"_topdir {cwd}", # we also need these 3 so that rpmbuild won't create them "--define", f"_builddir {cwd}", "--define", f"_rpmdir {cwd}", "--define", f"_buildrootdir {cwd}", self.specfile_path, ] present_srpms = set(Path.cwd().glob("*.src.rpm")) logger.debug("present srpms = %s", present_srpms) out = run_command( cmd, output=True, error_message="SRPM could not be created. Is the archive present?", cwd=self.local_project.working_dir, ).strip() logger.debug(f"{out}") # not doing 'Wrote: (.+)' since people can have different locales; hi Franto! reg = r": (.+\.src\.rpm)$" try: the_srpm = re.findall(reg, out)[0] except IndexError: raise PackitException("SRPM cannot be found, something is wrong.") if srpm_path: shutil.move(the_srpm, srpm_path) return Path(srpm_path) return Path(the_srpm)
def create_patches(self, upstream: str = None, destination: str = None) -> List[Tuple[str, str]]: """ Create patches from downstream commits. :param destination: str :param upstream: str -- git branch or tag :return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg) """ upstream = upstream or self.get_specfile_version() commits = self.get_commits_to_upstream(upstream, add_usptream_head_commit=True) patch_list = [] destination = destination or self.local_project.working_dir for i, commit in enumerate(commits[1:]): parent = commits[i] patch_name = f"{i + 1:04d}-{commit.hexsha}.patch" patch_path = os.path.join(destination, patch_name) patch_msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>" logger.debug(f"PATCH: {patch_name}\n{patch_msg}") git_diff_cmd = [ "git", "diff", "--patch", parent.hexsha, commit.hexsha, "--", ".", ] + [ f":(exclude){sync_file.src}" for sync_file in self.package_config.synced_files.raw_files_to_sync ] diff = run_command(cmd=git_diff_cmd, cwd=self.local_project.working_dir, output=True) with open(patch_path, mode="w") as patch_file: patch_file.write(diff) patch_list.append((patch_name, patch_msg)) return patch_list
def run_command(self, command: List[str], return_output: bool = True, env: Optional[Dict] = None): """ exec a command :param command: the command :param return_output: return output from this method if True :param env: dict with env vars to set for the command """ return utils.run_command( cmd=command, cwd=self.local_project.working_dir, output=return_output, env=env, )
def version_from_specfile(self) -> str: """ Version extracted from the specfile. """ version_raw = run_command( cmd=[ "rpmspec", "-q", "--qf", "'%{version}\\n'", "--srpm", self.source_specfile_path, ], output=True, fail=True, ) version = version_raw.strip("'\\\n") return version
def get_current_version(self) -> str: """ Get version of the project in current state (hint `git describe`) :return: e.g. 0.1.1.dev86+ga17a559.d20190315 or 0.6.1.1.gce4d84e """ action_output = self.get_output_from_action( action=ActionName.get_current_version) if action_output: return action_output ver = run_command(self.package_config.current_version_command, output=True).strip() logger.debug("version = %s", ver) # FIXME: this might not work when users expect the dashes # but! RPM refuses dashes in version/release ver = ver.replace("-", ".") logger.debug("sanitized version = %s", ver) return ver
def init_ticket(self, keytab: str = None): # TODO: this method has nothing to do with fedpkg, pull it out if not keytab: logger.info("won't be doing kinit, no credentials provided") return if keytab and Path(keytab).is_file(): cmd = [ "kinit", f"{self.fas_username}@FEDORAPROJECT.ORG", "-k", "-t", keytab, ] else: # there is no keytab, but user still might have active ticket - try to renew it cmd = ["kinit", "-R", f"{self.fas_username}@FEDORAPROJECT.ORG"] return run_command(cmd=cmd, error_message="Failed to init kerberos ticket:", fail=True)
def create_patches(self, upstream: str = None) -> List[Tuple[str, str]]: """ Create patches from downstream commits. :param upstream: str -- git branch or tag :return: [(patch_name, msg)] list of created patches (tuple of the file name and commit msg) """ upstream = upstream or self.version_from_specfile commits = self.get_commits_to_upstream(upstream, add_usptream_head_commit=True) patch_list = [] for i, commit in enumerate(commits[1:]): parent = commits[i] patch_name = f"{i + 1:04d}-{commit.hexsha}.patch" patch_path = os.path.join(self.distgit.working_dir, patch_name) patch_msg = f"{commit.summary}\nAuthor: {commit.author.name} <{commit.author.email}>" logger.debug(f"PATCH: {patch_name}\n{patch_msg}") diff = run_command( cmd=[ "git", "diff", "--patch", parent.hexsha, commit.hexsha, "--", ".", '":(exclude)redhat"', ], cwd=self.sourcegit.working_dir, output=True, ) with open(patch_path, mode="w") as patch_file: patch_file.write(diff) patch_list.append((patch_name, patch_msg)) return patch_list