def get_commit_count(tag, commit_id): """ Return the number of commits between the tag and commit_id""" # git describe returns either a tag-commitcount-gSHA1 OR # just the tag. # # so we need to pass in the tag as well. # output = run_command("git describe --match=%s %s" % (tag, commit_id)) # if tag == output: # return 0 # else: # parse the count from the output (status, output) = getstatusoutput("git describe --match=%s %s" % (tag, commit_id)) debug("tag - %s" % tag) debug("output - %s" % output) if status != 0: debug("git describe of tag %s failed (%d)" % (tag, status)) debug("going to use number of commits from initial commit") (status, output) = getstatusoutput("git rev-list --max-parents=0 HEAD") if status == 0: # output is now inital commit (status, output) = getstatusoutput("git rev-list %s..%s --count" % (output, commit_id)) if status == 0: return output return 0 if tag != output: # tag-commitcount-gSHA1, we want the penultimate value cnt = output.split("-")[-2] return cnt return 0
def is_git_state_clean(): """ Determines if the state of the current git repository is clean or not. """ (status, _) = getstatusoutput("git diff-index --quiet HEAD") if status != 0: return False (status, output) = getstatusoutput("git ls-files --exclude-standard --others") if len(output) > 0 or status > 0: return False return True
def _build(self, branch): """ Submit a Fedora build from current directory. """ target_param = "" scratch_param = "" build_target = self._get_build_target_for_branch(branch) if build_target: target_param = "--target %s" % build_target if self.scratch: scratch_param = "--scratch" build_cmd = "%s build --nowait %s %s" % (self.cli_tool, scratch_param, target_param) if self.dry_run: self.print_dry_run_warning(build_cmd) return info_out("Submitting build: %s" % build_cmd) (status, output) = getstatusoutput(build_cmd) if status > 0: if "already been built" in output: warn_out("Build has been submitted previously, continuing...") else: error_out([ "Unable to submit build." " Status code: %s\n" % status, " Output: %s\n" % output, ]) # Print the task ID and URL: for line in extract_task_info(output): print(line)
def get_commit_count(tag, commit_id): """ Return the number of commits between the tag and commit_id""" # git describe returns either a tag-commitcount-gSHA1 OR # just the tag. # # so we need to pass in the tag as well. # output = run_command("git describe --match=%s %s" % (tag, commit_id)) # if tag == output: # return 0 # else: # parse the count from the output (status, output) = getstatusoutput("git describe --match=%s %s" % (tag, commit_id)) debug("tag - %s" % tag) debug("output - %s" % output) if status != 0: debug("git describe of tag %s failed (%d)" % (tag, status)) return 0 if tag != output: # tag-commitcount-gSHA1, we want the penultimate value cnt = output.split("-")[-2] return cnt return 0
def patch_upstream(self): """ Create one patch per each release """ ch_dir = self.git_root if self.relative_project_dir != "/": ch_dir = os.path.join(self.git_root, self.relative_project_dir) os.chdir(ch_dir) debug("Running /usr/bin/generate-patches.pl -d %s %s %s-1 %s %s" % (self.rpmbuild_gitcopy, self.project_name, self.upstream_version, self.build_version, self.git_commit_id)) output = run_command("/usr/bin/generate-patches.pl -d %s %s %s-1 %s %s" % (self.rpmbuild_gitcopy, self.project_name, self.upstream_version, self.build_version, self.git_commit_id)) self.patch_files = output.split("\n") for p_file in self.patch_files: (status, output) = getstatusoutput( "grep 'Binary files .* differ' %s/%s " % (self.rpmbuild_gitcopy, p_file)) if status == 0 and output != "": error_out("You are doomed. Diff contains binary files. You can not use this builder") run_command("cp %s/%s %s" % (self.rpmbuild_gitcopy, p_file, self.rpmbuild_sourcedir)) (patch_number, patch_insert_index, patch_apply_index, lines) = self._patch_upstream() for patch in self.patch_files: lines.insert(patch_insert_index, "Patch%s: %s\n" % (patch_number, patch)) lines.insert(patch_apply_index, "%%patch%s -p1\n" % (patch_number)) patch_number += 1 patch_insert_index += 1 patch_apply_index += 2 self._write_spec(lines)
def _setup_sources(self): super(GitAnnexBuilder, self)._setup_sources() self.old_cwd = os.getcwd() os.chdir(os.path.join(self.old_cwd, self.relative_project_dir)) # NOTE: 'which' may not be installed... (docker containers) (status, output) = getstatusoutput("which git-annex") if status != 0: msg = "Please run '%s' as root." % self.package_manager.install(["git-annex"]) error_out('%s' % msg) run_command("git-annex lock") annexed_files = run_command("git-annex find --include='*'").splitlines() run_command("git-annex get") run_command("git-annex unlock") debug(" Annex files: %s" % annexed_files) for annex in annexed_files: debug("Copying unlocked file %s" % annex) os.remove(os.path.join(self.rpmbuild_gitcopy, annex)) shutil.copy(annex, self.rpmbuild_gitcopy) self._lock() os.chdir(self.old_cwd)
def _build(self, branch): """ Submit a Fedora build from current directory. """ target_param = "" build_target = self._get_build_target_for_branch(branch) if build_target: target_param = "--target %s" % build_target build_cmd = "%s build --nowait %s" % (self.cli_tool, target_param) if self.dry_run: self.print_dry_run_warning(build_cmd) return print("Submitting build: %s" % build_cmd) (status, output) = getstatusoutput(build_cmd) if status > 0: if "already been built" in output: print("Build has been submitted previously, continuing...") else: sys.stderr.write("ERROR: Unable to submit build.\n") sys.stderr.write(" Status code: %s\n" % status) sys.stderr.write(" Output: %s\n" % output) sys.exit(1) # Print the task ID and URL: for line in extract_task_info(output): print(line)
def get_commit_count(tag, commit_id): """ Return the number of commits between the tag and commit_id""" # git describe returns either a tag-commitcount-gSHA1 OR # just the tag. # # so we need to pass in the tag as well. # output = run_command("git describe --match=%s %s" % (tag, commit_id)) # if tag == output: # return 0 # else: # parse the count from the output (status, output) = getstatusoutput( "git describe --match=%s %s" % (tag, commit_id)) debug("tag - %s" % tag) debug("output - %s" % output) if status != 0: debug("git describe of tag %s failed (%d)" % (tag, status)) return 0 if tag != output: # tag-commitcount-gSHA1, we want the penultimate value cnt = output.split("-")[-2] return cnt return 0
def _setup_sources(self): super(GitAnnexBuilder, self)._setup_sources() old_cwd = os.getcwd() os.chdir(os.path.join(old_cwd, self.relative_project_dir)) # NOTE: 'which' may not be installed... (docker containers) (status, output) = getstatusoutput("which git-annex") if status != 0: msg = "Please run '%s install git-annex' as root." % package_manager() error_out('%s' % msg) run_command("git-annex lock") annexed_files = run_command("git-annex find --include='*'").splitlines() run_command("git-annex get") run_command("git-annex unlock") debug(" Annex files: %s" % annexed_files) for annex in annexed_files: debug("Copying unlocked file %s" % annex) os.remove(os.path.join(self.rpmbuild_gitcopy, annex)) shutil.copy(annex, self.rpmbuild_gitcopy) self._lock() os.chdir(old_cwd)
def _obs_user_confirm_commit(self, project_checkout): """ Prompt user if they wish to proceed with commit. """ print("") text = "Running '%s diff' in: %s" % (self.cli_tool, project_checkout) print("#" * len(text)) print(text) print("#" * len(text)) print("") os.chdir(project_checkout) (status, diff_output) = getstatusoutput("%s diff" % self.cli_tool) if diff_output.strip() == "": print("No changes in main branch, skipping commit.") else: print(diff_output) print("") print("##### Please review the above diff #####") if not self._ask_yes_no( "Do you wish to proceed with commit? [y/n] "): print("Fine, you're on your own!") self.cleanup() sys.exit(1) print("Proceeding with commit.") commit_msg_file = self._confirm_commit_msg(diff_output) cmd = '%s commit -F %s' % (self.cli_tool, commit_msg_file) debug("obs commit command: %s" % cmd) print if self.dry_run: self.print_dry_run_warning(cmd) else: print("Proceeding with commit.") os.chdir(self.package_workdir) print(run_command(cmd)) os.unlink(commit_msg_file) if self.no_build: getstatusoutput( "%s abortbuild %s %s" % (self.cli_tool, self.obs_project_name, self.obs_package_name)) print( "Aborting automatic rebuild because --no-build has been specified." )
def get_relative_project_dir(project_name, commit): """ Return the project's sub-directory relative to the git root. This could be a different directory than where the project currently resides, so we export a copy of the project's metadata from .tito/packages/ at the point in time of the tag we are building. """ cmd = "git show %s:%s/packages/%s" % (commit, tito_config_dir(), project_name) try: (status, pkg_metadata) = getstatusoutput(cmd) except: cmd = "git show %s:%s/packages/%s" % (commit, "rel-eng", project_name) (status, pkg_metadata) = getstatusoutput(cmd) tokens = pkg_metadata.strip().split(" ") debug("Got package metadata: %s" % tokens) if status != 0: return None return tokens[1]
def _obs_user_confirm_commit(self, project_checkout): """ Prompt user if they wish to proceed with commit. """ print("") text = "Running '%s diff' in: %s" % (self.cli_tool, project_checkout) print("#" * len(text)) print(text) print("#" * len(text)) print("") os.chdir(project_checkout) (status, diff_output) = getstatusoutput("%s diff" % self.cli_tool) if diff_output.strip() == "": print("No changes in main branch, skipping commit.") else: print(diff_output) print("") print("##### Please review the above diff #####") if not self._ask_yes_no("Do you wish to proceed with commit? [y/n] "): print("Fine, you're on your own!") self.cleanup() sys.exit(1) print("Proceeding with commit.") commit_msg_file = self._confirm_commit_msg(diff_output) cmd = '%s commit -F %s' % (self.cli_tool, commit_msg_file) debug("obs commit command: %s" % cmd) print if self.dry_run: self.print_dry_run_warning(cmd) else: print("Proceeding with commit.") os.chdir(self.package_workdir) print(run_command(cmd)) os.unlink(commit_msg_file) if self.no_build: getstatusoutput("%s abortbuild %s %s" % ( self.cli_tool, self.obs_project_name, self.obs_package_name)) print("Aborting automatic rebuild because --no-build has been specified.")
def find_git_root(): """ Find the top-level directory for this git repository. Returned as a full path. """ (status, cdup) = getstatusoutput("git rev-parse --show-cdup") if status > 0: error_out(["%s does not appear to be within a git checkout." % os.getcwd()]) if cdup.strip() == "": cdup = "./" return os.path.abspath(cdup)
def find_git_root(): """ Find the top-level directory for this git repository. Returned as a full path. """ (status, cdup) = getstatusoutput("git rev-parse --show-cdup") if status > 0: error_out( ["%s does not appear to be within a git checkout." % os.getcwd()]) if cdup.strip() == "": cdup = "./" return os.path.abspath(cdup)
def get_commit_count(tag, commit_id): """ Return the number of commits between the tag and commit_id""" # git describe returns either a tag-commitcount-gSHA1 OR # just the tag. # # so we need to pass in the tag as well. # output = run_command("git describe --match=%s %s" % (tag, commit_id)) # if tag == output: # return 0 # else: # parse the count from the output (status, output) = getstatusoutput( "git describe --match=%s %s" % (tag, commit_id)) debug("tag - %s" % tag) debug("output - %s" % output) if status != 0: debug("git describe of tag %s failed (%d)" % (tag, status)) debug("going to use number of commits from initial commit") (status, output) = getstatusoutput( "git rev-list --max-parents=0 HEAD") if status == 0: # output is now inital commit (status, output) = getstatusoutput( "git rev-list %s..%s --count" % (output, commit_id)) if status == 0: return output return 0 if tag != output: # tag-commitcount-gSHA1, we want the penultimate value cnt = output.split("-")[-2] return cnt return 0
def run_command(command, print_on_success=False): """ Run command. If command fails, print status code and command output. """ (status, output) = getstatusoutput(command) if status > 0: msgs = ["Error running command: %s\n" % command, "Status code: %s\n" % status, "Command output: %s\n" % output] error_out(msgs, die=False) raise RunCommandException(command, status, output) elif print_on_success: print("Command: %s\n" % command) print("Status code: %s\n" % status) print("Command output: %s\n" % output) return output
def run_command(command, print_on_success=False): """ Run command. If command fails, print status code and command output. """ (status, output) = getstatusoutput(command) if status > 0: sys.stderr.write("\n########## ERROR ############\n") sys.stderr.write("Error running command: %s\n" % command) sys.stderr.write("Status code: %s\n" % status) sys.stderr.write("Command output: %s\n" % output) raise RunCommandException(command, status, output) elif print_on_success: print("Command: %s\n" % command) print("Status code: %s\n" % status) print("Command output: %s\n" % output) return output
def run_command(command, print_on_success=False): """ Run command. If command fails, print status code and command output. """ (status, output) = getstatusoutput(command) if status > 0: msgs = [ "Error running command: %s\n" % command, "Status code: %s\n" % status, "Command output: %s\n" % output, ] error_out(msgs, die=False) raise RunCommandException(command, status, output) elif print_on_success: print("Command: %s\n" % command) print("Status code: %s\n" % status) print("Command output: %s\n" % output) return output
def patch_upstream(self): """ Generate patches for any differences between our tag and the upstream tag, and apply them into an exported copy of the spec file. """ patch_filename = "%s-to-%s-%s.patch" % (self.upstream_tag, self.project_name, self.build_version) patch_file = os.path.join(self.rpmbuild_gitcopy, patch_filename) patch_dir = self.git_root if self.relative_project_dir != "/": patch_dir = os.path.join(self.git_root, self.relative_project_dir) os.chdir(patch_dir) debug("patch dir = %s" % patch_dir) print("Generating patch [%s]" % patch_filename) debug("Patch: %s" % patch_file) patch_command = "git diff --relative %s..%s > %s" % \ (self.upstream_tag, self.git_commit_id, patch_file) debug("Generating patch with: %s" % patch_command) output = run_command(patch_command) print(output) (status, output) = getstatusoutput( "grep 'Binary files .* differ' %s " % patch_file) if status == 0 and output != "": error_out("You are doomed. Diff contains binary files. You can not use this builder") # Creating two copies of the patch here in the temp build directories # just out of laziness. Some builders need sources in SOURCES and # others need them in the git copy. Being lazy here avoids one-off # hacks and both copies get cleaned up anyhow. run_command("cp %s %s" % (patch_file, self.rpmbuild_sourcedir)) (patch_number, patch_insert_index, patch_apply_index, lines) = self._patch_upstream() lines.insert(patch_insert_index, "Patch%s: %s\n" % (patch_number, patch_filename)) if patch_apply_index > 0: lines.insert(patch_apply_index, "%%patch%s -p1\n" % (patch_number)) self._write_spec(lines)
def _get_upstream_version(self): """ Get the upstream version. Checks for "upstreamversion" in the spec file and uses it if found. Otherwise assumes the upstream version is equal to the version we're building. i.e. satellite-java-0.4.15 will be built on spacewalk-java-0.4.15 with just the package release being incremented on rebuilds. """ # Use upstreamversion if defined in the spec file: (status, output) = getstatusoutput( "cat %s | grep 'define upstreamversion' | " "awk '{ print $3 ; exit }'" % self.spec_file) if status == 0 and output != "": return output if self.test: return self.build_version.split("-")[0] # Otherwise, assume we use our version: else: return self.display_version
def _build(self, branch): """ Submit a Mead build from current directory. """ target_param = "" build_target = self._get_build_target_for_branch(branch) if build_target: target_param = "--target=%s" % build_target build_cmd = [self.cli_tool, "maven-chain", "--nowait"] if self.brew_target: build_cmd.append("--target=%s" % self.brew_target) build_cmd.append("--ini=%s" % (os.path.join(self.package_workdir, "mead.chain"))) build_cmd.append(target_param) if self.scratch: build_cmd.append("--scratch") build_cmd = " ".join(build_cmd) if self.dry_run: self.print_dry_run_warning(build_cmd) return info_out("Submitting build: %s" % build_cmd) (status, output) = getstatusoutput(build_cmd) if status > 0: if "already been built" in output: warn_out("Build has been submitted previously, continuing...") else: error_out([ "Unable to submit build.", " Status code: %s\n" % status, " Output: %s\n" % output, ]) # Print the task ID and URL: for line in extract_task_info(output): print(line)
def _setup_sources(self): super(GitAnnexBuilder, self)._setup_sources() old_cwd = os.getcwd() os.chdir(os.path.join(old_cwd, self.relative_project_dir)) (status, output) = getstatusoutput("which git-annex") if status != 0: msg = "Please run 'yum install git-annex' as root." error_out('%s' % msg) run_command("git-annex lock") annexed_files = run_command( "git-annex find --include='*'").splitlines() run_command("git-annex get") run_command("git-annex unlock") debug(" Annex files: %s" % annexed_files) for annex in annexed_files: debug("Copying unlocked file %s" % annex) os.remove(os.path.join(self.rpmbuild_gitcopy, annex)) shutil.copy(annex, self.rpmbuild_gitcopy) os.chdir(old_cwd)
def _read_project_config(self): """ Read project specific tito config if it exists. If no tag is specified we use tito.props from the current HEAD. If a tag is specified, we try to load a tito.props from that tag. """ debug("Determined package name to be: %s" % self.package_name) # Use the properties file in the current project directory, if it # exists: current_props_file = os.path.join(os.getcwd(), TITO_PROPS) if (os.path.exists(current_props_file)): self.config.read(current_props_file) print("Loaded package specific tito.props overrides") # Check for a tito.props back when this tag was created and use it # instead. (if it exists) if self.tag: relative_dir = get_relative_project_dir(self.package_name, self.tag) debug("Relative project dir: %s" % relative_dir) cmd = "git show %s:%s%s" % (self.tag, relative_dir, TITO_PROPS) debug(cmd) (status, output) = getstatusoutput(cmd) if status == 0: faux_config_file = FauxConfigFile(output) self.config.read_fp(faux_config_file) print("Loaded package specific tito.props overrides from %s" % self.tag) return debug("Unable to locate package specific config for this package.")
def _git_user_confirm_commit(self, project_checkout): """ Prompt user if they wish to proceed with commit. """ print("") text = "Running 'git diff' in: %s" % project_checkout print("#" * len(text)) print(text) print("#" * len(text)) print("") main_branch = self.git_branches[0] os.chdir(project_checkout) # Newer versions of git don't seem to want --cached here? Try both: (unused, diff_output) = getstatusoutput("git diff --cached") if diff_output.strip() == "": debug("git diff --cached returned nothing, falling back to git diff.") (unused, diff_output) = getstatusoutput("git diff") if diff_output.strip() == "": print("No changes in main branch, skipping commit for: %s" % main_branch) else: print(diff_output) print("") print("##### Please review the above diff #####") if not self._ask_yes_no("Do you wish to proceed with commit? [y/n] "): print("Fine, you're on your own!") self.cleanup() sys.exit(1) print("Proceeding with commit.") commit_msg_file = self._confirm_commit_msg(diff_output) cmd = '%s commit -F %s' % (self.cli_tool, commit_msg_file) debug("git commit command: %s" % cmd) print if self.dry_run: self.print_dry_run_warning(cmd) else: print("Proceeding with commit.") os.chdir(self.package_workdir) run_command(cmd) os.unlink(commit_msg_file) cmd = "%s push" % self.cli_tool if self.dry_run: self.print_dry_run_warning(cmd) else: # Push print(cmd) try: run_command(cmd) except RunCommandException as e: error_out("`%s` failed with: %s" % (cmd, e.output)) if not self.no_build: self._build(main_branch) for branch in self.git_branches[1:]: info_out("Merging branch: '%s' -> '%s'" % (main_branch, branch)) run_command("%s switch-branch %s" % (self.cli_tool, branch)) self._merge(main_branch) cmd = "git push origin %s:%s" % (branch, branch) if self.dry_run: self.print_dry_run_warning(cmd) else: print(cmd) try: run_command(cmd) except RunCommandException as e: error_out("`%s` failed with: %s" % (cmd, e.output)) if not self.no_build: self._build(branch) print
def tag_exists_locally(tag): (status, output) = getstatusoutput("git tag | grep %s" % tag) if status > 0: return False else: return True
def _check_tag_does_not_exist(self, new_tag): status, output = getstatusoutput( 'git tag -l %s|grep ""' % new_tag) if status == 0: raise Exception("Tag %s already exists!" % new_tag)
def _setup_sources(self): """ Create a copy of the git source for the project at the point in time our build tag was created. Created in the temporary rpmbuild SOURCES directory. """ self._create_build_dirs() working_path = os.path.join(os.getcwd(), self.relative_project_dir) debug('SETUP SOURCES') if self.relative_project_dir in os.path.join(os.getcwd(), ''): working_path = os.getcwd() debug("working_path: %s" % working_path) for directory, _, filenames in os.walk(working_path): debug('WALK') dir_artifacts_with_path = [ os.path.join(directory, f) for f in filenames ] debug(dir_artifacts_with_path) for artifact in dir_artifacts_with_path: debug(" Copying source file %s" % artifact) if os.path.isfile(artifact): if not os.path.exists("/".join( [self.rpmbuild_gitcopy, os.path.basename(artifact)])): shutil.copy(artifact, self.rpmbuild_gitcopy) if not os.path.exists("/".join( [self.rpmbuild_sourcedir, os.path.basename(artifact)])): shutil.copy(artifact, self.rpmbuild_sourcedir) # NOTE: The spec file we actually use is the one exported by git # archive into the temp build directory. This is done so we can # modify the version/release on the fly when building test rpms # that use a git SHA1 for their version. self.spec_file_name = os.path.basename( find_spec_like_file(self.rpmbuild_sourcedir)) self.spec_file = os.path.join(self.rpmbuild_sourcedir, self.spec_file_name) self.old_cwd = os.getcwd() # pylint: disable=W0201 if self.relative_project_dir not in os.path.join(os.getcwd(), ''): os.chdir(os.path.join(self.old_cwd, self.relative_project_dir)) # NOTE: 'which' may not be installed... (docker containers) status = getstatusoutput("which git-annex")[0] if status != 0: msg = "Please run '%s' as root." % self.package_manager.install( ["git-annex"]) error_out('%s' % msg) run_command("git-annex lock") annexed_files = run_command( "git-annex find --include='*'").splitlines() run_command("git-annex get") run_command("git-annex unlock") debug(" Annex files: %s" % annexed_files) for annex in annexed_files: debug("Copying unlocked file %s" % annex) if os.path.isfile(os.path.join(self.rpmbuild_gitcopy, annex)): os.remove(os.path.join(self.rpmbuild_gitcopy, annex)) shutil.copy(annex, self.rpmbuild_gitcopy) self._lock() os.chdir(self.old_cwd)
def _git_user_confirm_commit(self, project_checkout): """ Prompt user if they wish to proceed with commit. """ print("") text = "Running 'git diff' in: %s" % project_checkout print("#" * len(text)) print(text) print("#" * len(text)) print("") main_branch = self.git_branches[0] os.chdir(project_checkout) # Newer versions of git don't seem to want --cached here? Try both: (unused, diff_output) = getstatusoutput("git diff --cached") if diff_output.strip() == "": debug("git diff --cached returned nothing, falling back to git diff.") (unused, diff_output) = getstatusoutput("git diff") if diff_output.strip() == "": print("No changes in main branch, skipping commit for: %s" % main_branch) else: print(diff_output) print("") print("##### Please review the above diff #####") if not self._ask_yes_no("Do you wish to proceed with commit? [y/n] "): print("Fine, you're on your own!") self.cleanup() sys.exit(1) print("Proceeding with commit.") commit_msg_file = self._confirm_commit_msg(diff_output) cmd = '%s commit -F %s' % (self.cli_tool, commit_msg_file) debug("git commit command: %s" % cmd) print if self.dry_run: self.print_dry_run_warning(cmd) else: print("Proceeding with commit.") os.chdir(self.package_workdir) run_command(cmd) os.unlink(commit_msg_file) cmd = self._push_command() if self.dry_run: self.print_dry_run_warning(cmd) else: # Push print(cmd) try: run_command(cmd) except RunCommandException as e: error_out("`%s` failed with: %s" % (cmd, e.output)) if not self.no_build: self._build(main_branch) for branch in self.git_branches[1:]: info_out("Merging branch: '%s' -> '%s'" % (main_branch, branch)) run_command("%s switch-branch %s" % (self.cli_tool, branch)) self._merge(main_branch) cmd = "git push origin %s:%s" % (branch, branch) if self.dry_run: self.print_dry_run_warning(cmd) else: print(cmd) try: run_command(cmd) except RunCommandException as e: error_out("`%s` failed with: %s" % (cmd, e.output)) if not self.no_build: self._build(branch) print