def clone( self, package_name: str, target_path: Union[Path, str], branch: Optional[str] = None, anonymous: bool = False, ): """ clone a dist-git repo; this has to be done in current env b/c we don't have the keytab in sandbox """ cmd = [self.tool] if self.fas_username: cmd += ["--user", self.fas_username] cmd += ["-q", "clone"] if branch: cmd += ["--branch", branch] if anonymous: cmd += ["--anonymous"] cmd += [package_name, str(target_path)] error_msg = ( f"{self.tool} failed to clone repository {package_name}; " "please make sure that you are authorized to clone repositories " "from Fedora dist-git - this may require SSH keys set up or " "Kerberos ticket being active.") commands.run_command(cmd=cmd, error_message=error_msg)
def dump_to(file: Path): """Dump 'packit' database into a file. To restore db from this file, run: psql -d packit < database_packit.sql Args: file: File where to put the dump. Raises: PackitCommandFailedError: When pg_dump fails. """ # We have to specify libpq connection string to be able to pass the # password to the pg_dump. Luckily get_pg_url() does almost what we need. pg_connection = get_pg_url().replace("+psycopg2", "") cmd = ["pg_dump", f"--file={file}", f"--dbname={pg_connection}"] packit_logger = getLogger("packit") was_debug = packit_logger.level == DEBUG logger.info(f"Running pg_dump to create '{DB_NAME}' database backup") try: if was_debug: # Temporarily increase log level to avoid password leaking into logs packit_logger.setLevel(INFO) run_command(cmd=cmd) finally: if was_debug: packit_logger.setLevel(DEBUG)
def create_patches( self, git_ref: str, destination: str, files_to_ignore: Optional[List[str]] = None, ) -> List[PatchMetadata]: """ Create patches from git commits. :param git_ref: start processing commits from this till HEAD :param destination: place the patch files here :param files_to_ignore: list of files to ignore when creating patches :return: [PatchMetadata, ...] list of patches """ contained = self.are_child_commits_contained(git_ref) if not contained: self.linearize_history(git_ref) patch_list: List[PatchMetadata] = [] try: commits = self.get_commits_since_ref( git_ref, add_upstream_head_commit=False) git_f_p_cmd = [ "git", "format-patch", "--output-directory", f"{destination}", git_ref, "--", ".", ] + [ f":(exclude){file_to_ignore}" for file_to_ignore in files_to_ignore ] git_format_patch_out = run_command( cmd=git_f_p_cmd, cwd=self.lp.working_dir, output=True, decode=True, ).strip() if git_format_patch_out: patches: Dict[str, bytes] = { # we need to read bytes since we cannot decode whatever is inside patches patch_name: Path(patch_name).read_bytes() for patch_name in git_format_patch_out.split("\n") } patch_list = self.process_patches(patches, commits) patch_list = self.process_git_am_style_patches(patch_list) else: logger.warning( f"No patches between {git_ref!r} and {self.lp.ref!r}") return patch_list finally: if not contained: # check out the previous branch run_command(["git", "checkout", "-", "--"])
def test_run_cmd_unicode(tmp_path): # don't ask me what this is, I took it directly from systemd's test suite # that's what packit was UnicodeDecodeError-ing on cancer = ( b"\x06\xd0\xf1\t\x01\xa1\x01\t " b"\x15\x00&\xff\x00u\x08\x95@\x81\x02\t!\x15\x00&\xff\x00u\x08\x95@\x91\x02\xc0" ) t = tmp_path / "the-cancer" t.write_bytes(cancer) command = ["cat", str(t)] assert cancer == run_command(command, decode=False, output=True) with pytest.raises(UnicodeDecodeError): run_command(command, decode=True, output=True)
def linearize_history(git_ref: str): r""" Transform complex git history into a linear one starting from a selected git ref. Change this: * | | | | | | | | ea500ac513 (tag: v245) Merge pull request #15... |\ \ \ \ \ \ \ \ \ | * | | | | | | | | 0d5aef3eb5 hwdb: update for v245 | | |_|_|_|_|_|/ / | |/| | | | | | | * | | | | | | | | 03985d069b NEWS: final contributor update for v245 Into this: * 0d5aef3eb5 hwdb: update for v245 * 03985d069b NEWS: final contributor update for v245 """ logger.info( "When git history is too complex with merge commits having parents \n" "across a wide range, git is known to produce patches which cannot be applied. \n" "Therefore we are going to make the history linear on a dedicated branch \n" "to make sure the patches will be able to be applied.") current_time = datetime.datetime.now().strftime(DATETIME_FORMAT) target_branch = f"packit-patches-{current_time}" logger.info(f"Switch branch to {target_branch!r}.") run_command(["git", "checkout", "-B", target_branch]) target = f"{git_ref}..HEAD" logger.debug(f"Linearize history {target}.") # https://stackoverflow.com/a/17994534/909579 # With this command we will rewrite git history of our newly created branch # by dropping the merge commits and setting parent commits to those from target branch # this means we will drop the reference from which we are merging # filter branch passes these to cut: # ` -p 61f3e897f13101f29fb8027e8839498a469ad58e` # ` -p b7cf4b4ef5d0336443f21809b1506bc4a8aa75a9 -p 257188f80ce1a083e3a88b679b898a7...` # so we will keep the first parent and drop all the others run_command( [ "git", "filter-branch", "-f", "--parent-filter", 'cut -f 2,3 -d " "', target, ], # git prints nasty warning when filter-branch is used that it's dangerous # this env var prevents it from prints env={"FILTER_BRANCH_SQUELCH_WARNING": "1"}, )
def get_last_tag(self, before: str = None) -> Optional[str]: """ Get last git-tag from the repo. :param before: get last tag before the given tag """ logger.debug(f"We're about to `git-describe` the upstream repository " f"{self.local_project.working_dir}.") try: cmd = ["git", "describe", "--tags", "--abbrev=0"] if before: cmd += [f"{before}^"] last_tag = run_command( cmd, output=True, cwd=self.local_project.working_dir, ).strip() except PackitCommandFailedError as ex: logger.debug(f"{ex!r}") logger.info( "Can't describe this repository, are there any git tags?") # no tags in the git repo return None return last_tag
def get_spec_release( self, bump_version: bool = True, release_suffix: Optional[str] = None ) -> Optional[str]: """Assemble pieces of the spec file %release field we intend to set within the default fix-spec-file action The format is: {original_release_number}.{current_time}.{sanitized_current_branch}{git_desc_suffix} Example: 1.20210913173257793557.packit.experiment.24.g8b618e91 Returns: string which is meant to be put into a spec file %release field by packit """ original_release_number = self.specfile.expanded_release.split(".", 1)[0] if release_suffix is not None: return ( f"{original_release_number}.{release_suffix}" if release_suffix else None ) if not bump_version: return None # we only care about the first number in the release # so that we can re-run `packit srpm` git_des_command = [ "git", "describe", "--tags", "--long", "--match", "*", ] try: git_des_out = run_command( git_des_command, output=True, cwd=self.local_project.working_dir ).stdout.strip() except PackitCommandFailedError as ex: # probably no tags in the git repo logger.info(f"Exception while describing the repository: {ex!r}") git_desc_suffix = "" else: # git adds various info in the output separated by - # so let's just drop version and reuse everything else g_desc_raw = git_des_out.rsplit("-", 2)[1:] # release components are meant to be separated by ".", not "-" git_desc_suffix = "." + ".".join(g_desc_raw) # the leading dot is put here b/c git_desc_suffix can be empty # and we could have two subsequent dots - rpm errors out in such a case current_branch = self.local_project.ref sanitized_current_branch = sanitize_branch_name_for_rpm(current_branch) current_time = datetime.datetime.now().strftime(DATETIME_FORMAT) return ( f"{original_release_number}.{current_time}." f"{sanitized_current_branch}{git_desc_suffix}" )
def create_patches( self, git_ref: str, destination: str, files_to_ignore: Optional[List[str]] = None, ) -> List[PatchMetadata]: """ Create patches from git commits. :param git_ref: start processing commits from this till HEAD :param destination: place the patch files here :param files_to_ignore: list of files to ignore when creating patches :return: [PatchMetadata, ...] list of patches """ files_to_ignore = files_to_ignore or [] contained = self.are_child_commits_contained(git_ref) if not contained: self.linearize_history(git_ref) patch_list: List[PatchMetadata] = [] try: commits = self.get_commits_since_ref( git_ref, add_upstream_head_commit=False) # this is a string, separated by new-lines, with the names of patch files git_format_patch_out = self.run_git_format_patch( destination, files_to_ignore, git_ref) if git_format_patch_out: patches: Dict[str, bytes] = { # we need to read bytes since we cannot decode whatever is inside patches patch_name: Path(patch_name).read_bytes() for patch_name in git_format_patch_out.split("\n") } patch_list = self.process_patches(patches, commits, destination, files_to_ignore) patch_list = self.process_git_am_style_patches(patch_list) else: logger.info( f"No patches between {git_ref!r} and {self.lp.ref!r}") return patch_list finally: if not contained: # check out the previous branch run_command(["git", "checkout", "-", "--"])
def clone_fedora_package( package_name: str, dist_git_path: Path, branch: str = "c8s", namespace: str = "rpms", stg: bool = False, ): """ clone selected package from Fedora's src.fedoraproject.org """ run_command([ "git", "clone", "-b", branch, f"https://src{'.stg' if stg else ''}.fedoraproject.org/{namespace}/{package_name}.git", str(dist_git_path), ])
def clone_centos_package( package_name: str, dist_git_path: Path, branch: str = "c8s", namespace: str = "rpms", stg: bool = False, ): """ clone selected package from git.[stg.]centos.org """ run_command([ "git", "clone", "-b", branch, f"https://git{'.stg' if stg else ''}.centos.org/{namespace}/{package_name}.git", str(dist_git_path), ])
def clone_centos_9_package( package_name: str, dist_git_path: Path, branch: str = "c9s", namespace: str = "rpms", stg: bool = None, ): """ clone selected package from git.[stg.]centos.org """ if stg: logger.warning( "There is no staging instance for CentOS Stream 9 dist-git.") run_command([ "git", "clone", "-b", branch, f"https://{CENTOS_STREAM_GITLAB}/{namespace}/{package_name}.git", str(dist_git_path), ])
def clone(self, package_name: str, target_path: str, anonymous: bool = False): """ clone a dist-git repo; this has to be done in current env b/c we don't have the keytab in sandbox """ cmd = [self.fedpkg_exec] if self.fas_username: cmd += ["--user", self.fas_username] cmd += ["-q", "clone"] if anonymous: cmd += ["-a"] cmd += [package_name, target_path] error_msg = ( f"Packit failed to clone the repository {package_name}; " "please make sure that you are authorized to clone repositories " "from Fedora dist-git - this may require SSH keys set up or " "Kerberos ticket being active.") commands.run_command(cmd=cmd, error_message=error_msg)
def git_interpret_trailers(patch: str) -> str: """Run 'git interpret-trailers' on 'patch' and return the output. We don't use --parse/--unfold to unfold multiline values per RFC822 because we use YAML block scalars for multiline trailer values. Args: patch: Path of the patch. Returns: The output of the command. """ cmd = ["git", "interpret-trailers", "--only-input", "--only-trailers", patch] return run_command(cmd=cmd, cwd=Path.cwd(), output=True).stdout
def _run_prep(self): """ run `rpmbuild -bp` in the dist-git repo to get a git-repo in the %prep phase so we can pick the commits in the source-git repo """ _packitpatch_path = shutil.which("_packitpatch") if not _packitpatch_path: raise PackitException( "We are trying to unpack a dist-git archive and lay patches on top " 'by running `rpmbuild -bp` but we cannot find "_packitpatch" command on PATH: ' "please install packit as an RPM." ) logger.info( f"expanding %prep section in {self.dist_git.local_project.working_dir}" ) rpmbuild_args = [ "rpmbuild", "--nodeps", "--define", f"_topdir {str(self.dist_git.local_project.working_dir)}", "-bp", "--define", f"_specdir {str(self.dist_git.absolute_specfile_dir)}", "--define", f"_sourcedir {str(self.dist_git.absolute_source_dir)}", ] rpmbuild_args += RPM_MACROS_FOR_PREP if logger.level <= logging.DEBUG: # -vv can be super-duper verbose rpmbuild_args.append("-v") rpmbuild_args.append(str(self.dist_git.absolute_specfile_path)) run_command( rpmbuild_args, cwd=self.dist_git.local_project.working_dir, print_live=True, )
def run_command( self, command: List[str], return_output: bool = True, env: Optional[Dict] = None, cwd: Union[str, Path, None] = None, print_live: bool = False, ) -> commands.CommandResult: return commands.run_command( cmd=command, cwd=cwd or self.local_project.working_dir, output=return_output, env=env, print_live=print_live, )
def get_last_tag(self) -> Optional[str]: """ get last git-tag from the repo """ try: last_tag = run_command( ["git", "describe", "--tags", "--abbrev=0"], output=True, cwd=self.local_project.working_dir, ).strip() except PackitCommandFailedError as ex: logger.debug(f"{ex!r}") logger.info( "Can't describe this repository, are there any git tags?") # no tags in the git repo return None return last_tag
def run_prep_for_srpm(srpm_path: Path): cmd = [ "rpmbuild", "-rp", "--define", f"_sourcedir {srpm_path.parent}", "--define", f"_srcdir {srpm_path.parent}", "--define", f"_specdir {srpm_path.parent}", "--define", f"_srcrpmdir {srpm_path.parent}", "--define", f"_topdir {srpm_path.parent}", # we also need these 3 so that rpmbuild won't create them "--define", f"_builddir {srpm_path.parent}", "--define", f"_rpmdir {srpm_path.parent}", "--define", f"_buildrootdir {srpm_path.parent}", str(srpm_path), ] run_command(cmd)
def run_command( self, command: List[str], return_output: bool = True, env: Optional[Dict] = None, cwd: Union[str, Path] = None, ): """ exec a command :param command: the command :param return_output: return output from this method if True :param env: dict with env vars to set for the command :param cwd: working directory to run command in """ return commands.run_command( cmd=command, cwd=cwd or self.local_project.working_dir, output=return_output, env=env, )
def run_git_format_patch( self, destination: str, files_to_ignore: List[str], ref_or_range: str, no_prefix: bool = False, ): """ run `git format-patch $ref_or_range` in self.local_project.working_dir :param destination: place the patches here :param files_to_ignore: ignore changes in these files :param ref_or_range: [ <since> | <revision range> ]: 1. A single commit, <since>, specifies that the commits leading to the tip of the current branch that are not in the history that leads to the <since> to be output. 2. Generic <revision range> expression (see "SPECIFYING REVISIONS" section in gitrevisions(7)) means the commits in the specified range. :param no_prefix: prefix is the leading a/ and b/ - format-patch does this by default :return: str, git format-patch output: new-line separated list of patch names """ git_f_p_cmd = [ "git", "format-patch", "--output-directory", f"{destination}" ] if no_prefix: git_f_p_cmd.append("--no-prefix") git_f_p_cmd += [ ref_or_range, "--", ".", ] + [ f":(exclude){file_to_ignore}" for file_to_ignore in files_to_ignore ] return run_command( cmd=git_f_p_cmd, cwd=self.lp.working_dir, output=True, decode=True, ).strip()
def get_commit_messages( self, after: Optional[str] = None, before: str = "HEAD" ) -> str: """ :param after: get commit messages after this revision, if None, all commit messages before 'before' will be returned :param before: get commit messages before this revision :return: commit messages """ # let's print changes b/w the last 2 revisions; # ambiguous argument '0.1.0..HEAD': unknown revision or path not in the working tree. # Use '--' to separate paths from revisions, like this commits_range = f"{after}..{before}" if after else before if not before: raise PackitException( "Unable to get a list of commit messages in range " f"{commits_range} because the upper bound is not " f"defined ({before!r})." ) cmd = [ "git", "log", "--no-merges", "--pretty=format:- %s (%an)", commits_range, "--", ] try: return run_command( cmd, output=True, cwd=self.local_project.working_dir ).stdout.strip() except PackitCommandFailedError as ex: logger.error(f"We couldn't get commit messages for %changelog\n{ex}") logger.info(f"Does the git ref {after} exist in the git repo?") logger.info( "If the ref is a git tag, " 'you should consider setting "upstream_tag_template":\n ' "https://packit.dev/docs/configuration/#upstream_tag_template" ) raise
def get_commit_messages(self, after: str = None, before: str = "HEAD") -> str: """ :param after: get commit messages after this revision, if None, all commit messages before 'before' will be returned :param before: get commit messages before this revision :return: commit messages """ # let's print changes b/w the last 2 revisions; # ambiguous argument '0.1.0..HEAD': unknown revision or path not in the working tree. # Use '--' to separate paths from revisions, like this commits_range = f"{after}..{before}" if after else before cmd = [ "git", "log", "--no-merges", "--pretty=format:- %s (%an)", commits_range, "--", ] return run_command(cmd, output=True, cwd=self.local_project.working_dir).strip()
def linearize_history(self, git_ref: str) -> str: r""" Transform complex git history into a linear one starting from a selected git ref. Returns the name of the linearized branch. Change this: * | | | | | | | | ea500ac513 (tag: v245) Merge pull request #15... |\ \ \ \ \ \ \ \ \ | * | | | | | | | | 0d5aef3eb5 hwdb: update for v245 | | |_|_|_|_|_|/ / | |/| | | | | | | * | | | | | | | | 03985d069b NEWS: final contributor update for v245 Into this: * 0d5aef3eb5 hwdb: update for v245 * 03985d069b NEWS: final contributor update for v245 """ logger.info( "When git history is too complex with merge commits having parents \n" "across a wide range, git is known to produce patches which cannot be applied. \n" "Therefore we are going to make the history linear on a dedicated branch \n" "to make sure the patches will be able to be applied." ) if self.lp.git_repo.is_dirty(): raise PackitGitException( "The source-git repo is dirty which means we won't be able to do a linear history. " "Please commit the changes to resolve the issue. If you are changing the content " "of the repository in an action, you can commit those as well." ) current_time = datetime.datetime.now().strftime(DATETIME_FORMAT) initial_branch = self.lp.ref target_branch = f"packit-patches-{current_time}" logger.info(f"Switch branch to {target_branch!r}.") ref = self.lp.create_branch(target_branch) ref.checkout() target = f"{git_ref}..HEAD" logger.debug(f"Linearize history {target}.") # https://stackoverflow.com/a/17994534/909579 # With this command we will rewrite git history of our newly created branch # by dropping the merge commits and setting parent commits to those from target branch # this means we will drop the reference from which we are merging # filter branch passes these to cut: # ` -p 61f3e897f13101f29fb8027e8839498a469ad58e` # ` -p b7cf4b4ef5d0336443f21809b1506bc4a8aa75a9 -p 257188f80ce1a083e3a88b679b898a7...` # so we will keep the first parent and drop all the others try: run_command( [ "git", "filter-branch", "-f", "--parent-filter", 'cut -f 2,3 -d " "', target, ], # git prints nasty warning when filter-branch is used that it's dangerous # this env var prevents it from printing env={"FILTER_BRANCH_SQUELCH_WARNING": "1"}, print_live=True, cwd=self.lp.working_dir, ) finally: # check out the former branch self.lp.checkout_ref(initial_branch) # we could also delete the newly created branch, # but let's not do that so that user can inspect it return target_branch
def create_patches( self, git_ref: str, destination: str, files_to_ignore: Optional[List[str]] = None, ) -> List[PatchMetadata]: """ Create patches from git commits. :param git_ref: start processing commits from this till HEAD :param destination: place the patch files here :param files_to_ignore: list of files to ignore when creating patches :return: [(patch_path, msg)] list of created patches (tuple of the file path and commit msg) """ contained = self.are_child_commits_contained(git_ref) if not contained: self.linearize_history(git_ref) patch_list: List[PatchMetadata] = [] try: commits = self.get_commits_since_ref( git_ref, add_upstream_head_commit=False) git_f_p_cmd = [ "git", "format-patch", "--output-directory", f"{destination}", git_ref, "--", ".", ] + [ f":(exclude){file_to_ignore}" for file_to_ignore in files_to_ignore ] git_format_patch_out = run_command( cmd=git_f_p_cmd, cwd=self.lp.working_dir, output=True, decode=True, ).strip() if git_format_patch_out: patches = { patch_name: Path(patch_name).read_text() for patch_name in git_format_patch_out.split("\n") } for commit in commits: for patch_name, patch_content in patches.items(): # `git format-patch` usually creates one patch for a merge commit, # so some commits won't be covered by a dedicated patch file if commit.hexsha in patch_content: path = Path(patch_name) patch_metadata = PatchMetadata.from_commit( commit=commit, patch_path=path) if patch_metadata.ignore: logger.debug( f"[IGNORED: {patch_metadata.name}] {commit.summary}" ) else: logger.debug( f"[{patch_metadata.name}] {commit.summary}" ) patch_list.append(patch_metadata) break else: logger.warning( f"No patches between {git_ref!r} and {self.lp.ref!r}") return patch_list finally: if not contained: # check out the previous branch run_command(["git", "checkout", "-", "--"])
def build_srpm(path: Path): run_command(["rpmbuild", "--rebuild", str(path)], output=True)
def get_all_koji_targets() -> List[str]: return run_command(["koji", "list-targets", "--quiet"], output=True).split()
def test_run_command_w_env(): run_command(["bash", "-c", "env | grep PATH"], env={"X": "Y"})
def fix_spec(self, archive: str, version: str, commit: str): """ In order to create a SRPM from current git checkout, we need to have the spec reference the tarball and unpack it. This method updates the spec so it's possible. :param archive: relative path to the archive: used as Source0 :param version: version to set in the spec :param commit: commit to set in the changelog """ self._fix_spec_source(archive) self._fix_spec_prep(archive) # we only care about the first number in the release # so that we can re-run `packit srpm` git_des_command = [ "git", "describe", "--tags", "--long", "--match", "*", ] try: git_des_out = run_command( git_des_command, output=True, cwd=self.local_project.working_dir).strip() except PackitCommandFailedError as ex: # probably no tags in the git repo logger.info(f"Exception while describing the repository: {ex!r}") git_desc_suffix = "" else: # git adds various info in the output separated by - # so let's just drop version and reuse everything else g_desc_raw = git_des_out.rsplit("-", 2)[1:] # release components are meant to be separated by ".", not "-" git_desc_suffix = "." + ".".join(g_desc_raw) # the leading dot is put here b/c git_desc_suffix can be empty # and we could have two subsequent dots - rpm errors in such a case current_branch = self.local_project.ref sanitized_current_branch = sanitize_branch_name_for_rpm(current_branch) original_release_number = self.specfile.get_release_number().split( ".", 1)[0] current_time = datetime.datetime.now().strftime(DATETIME_FORMAT) release = (f"{original_release_number}.{current_time}." f"{sanitized_current_branch}{git_desc_suffix}") last_tag = self.get_last_tag() msg = "" if last_tag: msg = self.get_commit_messages(after=last_tag) if not msg: # no describe, no tag - just a boilerplate message w/ commit hash # or, there were no changes b/w HEAD and last_tag, which implies last_tag == HEAD msg = f"- Development snapshot ({commit})" logger.debug(f"Setting Release in spec to {release!r}.") # instead of changing version, we change Release field # upstream projects should take care of versions self.specfile.set_spec_version( version=version, release=release, changelog_entry=msg, )
def remove_gpg_key_pair(gpg_binary: str, fingerprint: str): run_command( [gpg_binary, "--batch", "--yes", "--delete-secret-and-public-key", fingerprint] )