def verify_package_dir(package_dir): """Check if package_dir points to a valid package dir (i.e. contains at least an osg/ dir or an upstream/ dir) and is in a git repo. """ top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "rev-parse", "--show-toplevel" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git top-level directory of %s. Output:\n%s" % (err, package_dir, out)) if top_dir != out.strip(): raise GitError("Specified package directory (%s) is not a top-level directory in the git repo (%s)." % \ (package_dir, top_dir)) command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "ls-files", "osg", "upstream" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git subdirectories of %s. Output:\n%s" % (err, package_dir, out)) for line in out.split("\n"): if line.startswith('osg/') or line.startswith('upstream/'): return True return False
def is_outdated(package_dir): """Return True if the package has been changed since the revision in the local git repo. """ remote = get_current_branch_remote(package_dir) branch = get_branch(package_dir) branch_ref = "refs/heads/%s" % branch branch_hash = '' top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "show-ref" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git references for directory %s. Output:\n%s", (err, package_dir, out)) for line in out.splitlines(): info = line.strip().split() if len(info) != 2: continue if info[1] == branch_ref: branch_hash = info[0] break if not branch_hash: raise GitError("Unable to determine local branch's hash.") out, err = utils.sbacktick([ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "ls-remote", "--heads", remote ]) if err: raise GitError( "Exit code %d getting remote git status for directory %s. Output:\n%s" % (err, package_dir, out)) remote_hash = '' for line in out.splitlines(): info = line.strip().split() if len(info) != 2: continue if info[1] == branch_ref: remote_hash = info[0] break if not remote_hash: raise GitError("Unable to determine remote branch's hash.") if remote_hash == branch_hash: return False print "Remote hash (%s) does not match local hash (%s) for branch %s." % ( remote_hash, branch_hash, branch) return True
def is_uncommitted(package_dir): """Return True if there are uncommitted changes or files in the git working dir.""" top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "status", "--porcelain" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git status for directory %s. Output:\n%s" % (err, package_dir, out)) if out: print "The following uncommitted changes exist:" print out print "Please commit these first." return True remote = get_current_branch_remote(package_dir) branch = get_branch(package_dir) branch_ref = "refs/heads/%s" % branch origin_ref = "refs/remotes/%s/%s" % (remote, branch) command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "show-ref" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git references for directory %s. Output:\n%s", (err, package_dir, out)) branch_hash = '' origin_hash = '' for line in out.splitlines(): info = line.split() if len(info) != 2: continue if info[1] == branch_ref: branch_hash = info[0] if info[1] == origin_ref: origin_hash = info[0] if not branch_hash and not origin_hash: raise GitError( "Could not find either local or remote hash for directory %s." % package_dir) if branch_hash != origin_hash: raise GitError("Local hash (%s) does not match remote hash " "(%s) for directory %s. Perhaps you need to perform 'git push'?" % \ (branch_hash, origin_hash, package_dir)) return False
def is_outdated(package_dir): """Return True if the package has been changed since the revision in the SVN working dir. """ if utils.is_url(package_dir): return False out, err = utils.sbacktick("svn status -u -q " + package_dir) if err: raise SVNError("Exit code %d getting SVN status. Output:\n%s" % (err, out)) outdated_files = [] for line in out.split("\n"): try: outdated_flag = line[8] except IndexError: continue if outdated_flag == "*": outdated_files.append(line) if outdated_files: print "The following outdated files exist:" print "\n".join(outdated_files) return True else: return False
def is_untracked_path(path): """Return True if the given path is untracked in SVN. Note: ignored files return False. """ output, ret = utils.sbacktick(["svn", "status", path]) return output.startswith('?')
def diff3(old_file, orig_file, new_file, dest_file=None): """Do a 3-way diff between `old_file`, `orig_file`, and `new_file`, where the differences are shown with markers like what SVN makes for a file with merge conflicts, e.g.: ''' <<<<<<< old_file old stuff ||||||| orig_file orig stuff ======= new stuff >>>>>>> new_file ''' Write the result to `dest_file` if it is specified. Return the text of the diff on success, None on failure. """ diff, ret = utils.sbacktick(["diff3", "-m", old_file, orig_file, new_file]) if not (ret == 0 or ret == 1): logging.warning("Error diffing %s %s %s: diff3 returned %d", old_file, orig_file, new_file, ret) return if dest_file: utils.unslurp(dest_file, diff) logging.info("Difference between %s, %s, and %s written to %s", old_file, orig_file, new_file, dest_file) return diff
def diff2(old_file, new_file, dest_file=None): """Do a 2-way diff, between `old_file` and `new_file`, where the differences are shown with markers like what SVN makes for a file with merge conflicts, e.g.: ''' <<<<<<< old_file old stuff ======= new stuff >>>>>>> new_file ''' Write the result to `dest_file` if it is specified. Return the text of the diff on success, None on failure. """ diff, ret = utils.sbacktick(["diff", """\ --changed-group-format=<<<<<<< %(old_file)s %%<======= %%>>>>>>>> %(new_file)s """ % locals(), old_file, new_file]) if not (ret == 0 or ret == 1): logging.warning("Error diffing %s %s: diff returned %d", old_file, new_file, ret) return if dest_file: utils.unslurp(dest_file, diff) logging.info("Difference between %s and %s written to %s", old_file, new_file, dest_file) return diff
def koji(package_dir, koji_obj, buildopts): """koji task with a git build.""" package_dir = os.path.abspath(package_dir) verify_package_dir(package_dir) package_name = os.path.basename(package_dir) if not re.match(r"\w+", package_name): # sanity check raise Error("Package directory '%s' gives invalid package name '%s'" % (package_dir, package_name)) if not buildopts.get('scratch'): koji_obj.add_pkg(package_name) remote = get_fetch_url(package_dir, get_known_remote(package_dir)[0]) top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "log", "-1", "--pretty=format:%H" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git hash for directory %s. Output:\n%s" % (err, package_dir, out)) rev = out.strip() return koji_obj.build_git(remote, rev, package_name)
def get_sha1sum(file_path): """Return the SHA1 checksum of the file located at `file_path` as a string.""" out, ret = utils.sbacktick(["sha1sum", file_path]) if ret != 0: raise Error("Unable to get sha1sum of %s: exit %d when running sha1sum" % (file_path, ret)) match = re.match(r"[a-f0-9]{40}", out) if not match: raise Error("Unable to get sha1sum of %s: unexpected output: %s" % (file_path, out)) return match.group(0)
def get_targets(self): """Get a list of the names of targets (as strings) from koji""" out, err = utils.sbacktick(self.koji_cmd + ["list-targets", "--quiet"]) if err: raise KojiError("koji list-targets failed with exit code " + str(err)) lines = out.split("\n") target_names = [re.split(r"\s+", x)[0] for x in lines] return target_names
def is_uncommitted(package_dir): """Return True if there are uncommitted changes in the SVN working dir.""" out, err = utils.sbacktick("svn status -q " + package_dir) if err: raise SVNError("Exit code %d getting SVN status. Output:\n%s" % (err, out)) if out: print "The following uncommitted changes exist:" print out return True else: return False
def srpm_nv(srpm): """Return the NV (Name, Version) from an SRPM.""" output, ret = utils.sbacktick( ["rpm", "-qp", "--qf", "%{name} %{version}", srpm]) if ret == 0: try: name, version = output.rstrip().split(" ") return name, version except ValueError: # not enough/too many items pass raise Error("Unable to extract name and version from SRPM %s: %s" % (srpm, output))
def search_names(self, terms, stype, match): search_subcmd = ["search", stype] if match == 'regex': search_subcmd.append("--regex") elif match == 'exact': search_subcmd.append("--exact") search_subcmd.append(terms) out, err = utils.sbacktick(self.koji_cmd + search_subcmd) if err: raise KojiError("koji search failed with exit code " + str(err)) return out.split("\n")
def get_sha1sum(file_path): """Return the SHA1 checksum of the file located at `file_path` as a string.""" out, ret = utils.sbacktick(["sha1sum", file_path]) if ret != 0: raise Error( "Unable to get sha1sum of %s: exit %d when running sha1sum" % (file_path, ret)) match = re.match(r"[a-f0-9]{40}", out) if not match: raise Error("Unable to get sha1sum of %s: unexpected output: %s" % (file_path, out)) return match.group(0)
def print_version_and_exit(): """Print version and exit""" # '@'+'VERSION'+'@' is so sed will leave it alone during 'make dist' if __version__ == '@' + 'VERSION' + '@': print "osg-build SVN" out, ret = utils.sbacktick("svn info " + sys.argv[0], err2out=True) if ret: print "no info" else: print "SVN info:\n" + out else: print "osg-build " + __version__ sys.exit(0)
def verify_package_info(package_info): """Check if package_info points to a valid package dir (i.e. contains at least an osg/ dir or an upstream/ dir). """ url = package_info['canon_url'] rev = package_info['revision'] command = ["svn", "ls", url, "-r", rev] out, err = utils.sbacktick(command, clocale=True, err2out=True) if err: raise SVNError("Exit code %d getting SVN listing of %s (rev %s). Output:\n%s" % (err, url, rev, out)) for line in out.split("\n"): if line.startswith('osg/') or line.startswith('upstream/'): return True return False
def get_current_branch_remote(package_dir): """Return the configured remote for the current branch.""" branch = get_branch(package_dir) top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "config", "branch.%s.remote" % branch ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError("Exit code %d getting git branch %s remote for directory '%s'. Output:\n%s" % \ (err, branch, package_dir, out)) return out.strip()
def get_branch(package_dir): """Return the current git branch for a given directory.""" top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "branch" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git branch for directory %s. Output:\n%s" % (err, package_dir, out)) out = out.strip() if not out: raise GitError("'git branch' returned no output.") return out.split()[-1]
def verify_package_info(package_info): """Check if package_info points to a valid package dir (i.e. contains at least an osg/ dir or an upstream/ dir). """ url = package_info['canon_url'] rev = package_info['revision'] command = ["svn", "ls", url, "-r", rev] out, err = utils.sbacktick(command, clocale=True, err2out=True) if err: raise SVNError( "Exit code %d getting SVN listing of %s (rev %s). Output:\n%s" % (err, url, rev, out)) for line in out.split("\n"): if line.startswith('osg/') or line.startswith('upstream/'): return True return False
def verify_git_svn_commit(package_dir): """Verify the last commit in the git repo actually came from git-svn.""" top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "log", "-n", "1" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git log for directory %s. Output:\n%s" % (err, package_dir, out)) for line in out.splitlines(): if line.find("git-svn-id:") >= 0: return raise GitError( "Last git commit not from SVN - possible inconsistency between git and SVN!" )
def get_package_info(package_dir, rev=None): """Return the svn info for a package dir.""" command = ["svn", "info", package_dir] if rev: command += ["-r", rev] else: command += ["-r", "HEAD"] out, err = utils.sbacktick(command, clocale=True, err2out=True) if err: raise SVNError("Exit code %d getting SVN info. Output:\n%s" % (err, out)) info = dict() for line in out.split("\n"): label, value = line.strip().split(": ", 1) label = label.strip().lower().replace(' ', '_') info[label] = value info['canon_url'] = re.sub("^" + re.escape(info['repository_root']), SVN_ROOT, info['url']) return info
def get_spec_name_in_srpm(srpm): """Return the name of the spec file present in an SRPM. Assumes there is exactly one spec file in the SRPM -- if there is more than one spec file, returns the name of the first one ``cpio'' prints. """ out, ret = utils.sbacktick("rpm2cpio %s | cpio -t '*.spec' 2> /dev/null" % utils.shell_quote(srpm), shell=True) if ret != 0: raise Error("Unable to get list of spec files from %s" % srpm) try: spec_name = [_f for _f in [x.strip() for x in out.split("\n")] if _f][0] except IndexError: spec_name = None if not spec_name: raise Error("No spec file inside %s" % srpm) return spec_name
def get_package_info(package_dir): """Return the svn info for a package dir.""" command = ["svn", "info", package_dir] # If we don't specify the revision in the argument (e.g. no foo@19999) # then explicitly specify HEAD to make sure we're not getting an older # version. if not re.search(r'@\d+$', package_dir): command += ['-r', 'HEAD'] out, err = utils.sbacktick(command, err2out=True) if err: raise SVNError("Exit code %d getting SVN info. Output:\n%s" % (err, out)) info = dict() for line in out.split("\n"): label, value = line.strip().split(": ", 1) label = label.strip().lower().replace(' ', '_') info[label] = value info['canon_url'] = re.sub("^" + re.escape(info['repository_root']), SVN_ROOT, info['url']) return info
def make_srpm(self, spec_fn): """Make an SRPM from a spec file. Raise OSGPrebuildError on failure""" cmd = (["rpmbuild", "-bs", "--nodeps"] + self.get_rpmbuild_defines(prebuild=True) + [spec_fn]) err_msg_prefix = ("Error making SRPM from %s\n" "Command used was: %s\n" % (spec_fn, " ".join(cmd))) out, err = utils.sbacktick(cmd, nostrip=True, clocale=True, err2out=True) if err: log.error("Rpmbuild failed. Output follows: " + out) raise OSGPrebuildError(err_msg_prefix + "Rpmbuild return code %d" % err) match = re.search(r"(?ms)^Wrote: ([^\n]+.src.rpm)$", out) if match: srpm = match.group(1).strip() if os.path.isfile(srpm): return srpm raise OSGPrebuildError(err_msg_prefix + "Unable to find resulting SRPM.")
def lint(self): """lint task. Prebuild the package and run rpmlint on the SRPM.""" if not utils.which("rpmlint"): raise ProgramNotFoundError("rpmlint") conf_file = utils.find_file("rpmlint.cfg", DATA_FILE_SEARCH_PATH) if not conf_file: raise FileNotFoundError("rpmlint.cfg", DATA_FILE_SEARCH_PATH) srpm = self.prebuild() lint_output, lint_returncode = utils.sbacktick( ["rpmlint", "-f", conf_file, srpm]) print lint_output if lint_returncode == 0: print "rpmlint ok for " + self.package_name elif lint_returncode < 64: print "Error running rpmlint for " + self.package_name elif lint_returncode == 64: print "rpmlint found problems with " + self.package_name elif lint_returncode == 66: print "rpmlint found many problems with " + self.package_name else: print "unrecognized return code from rpmlint: " + str(lint_returncode)
def is_svn(package_dir): """Determine whether a given directory is part of an SVN repo.""" # If package_dir is a URL, not a directory, then we can't cd into it to # check. Assume True for now. if utils.is_url(package_dir): return True # TODO: Allow specifying a git URL to build from. pwd = os.getcwd() try: try: os.chdir(package_dir) except OSError, ose: if ose.errno == errno.ENOENT: raise Error("%s is not a valid package directory\n(%s)" % (package_dir, ose)) command = ["svn", "info"] try: err = utils.sbacktick(command, err2out=True)[1] except OSError, ose: if ose.errno != errno.ENOENT: raise err = 1
def get_fetch_url(package_dir, remote): """Return a fetch url is on osg-build's configured whitelist of remotes.""" top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "remote", "-v" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git status for directory %s. Output:\n%s" % (err, package_dir, out)) for line in out.splitlines(): info = line.strip().split() if len(info) != 3: continue if info[2] != '(fetch)': continue if info[0] == remote: return constants.GIT_REMOTE_MAPS.setdefault(info[1], info[1]) raise GitError("Remote URL not found for remote %s in directory %s; are remotes " \ "configured correctly?" % (remote, package_dir))
def diff2(old_file, new_file, dest_file=None): """Do a 2-way diff, between `old_file` and `new_file`, where the differences are shown with markers like what SVN makes for a file with merge conflicts, e.g.: ''' <<<<<<< old_file old stuff ======= new stuff >>>>>>> new_file ''' Write the result to `dest_file` if it is specified. Return the text of the diff on success, None on failure. """ diff, ret = utils.sbacktick([ "diff", """\ --changed-group-format=<<<<<<< %(old_file)s %%<======= %%>>>>>>>> %(new_file)s """ % locals(), old_file, new_file ]) if not (ret == 0 or ret == 1): logging.warning("Error diffing %s %s: diff returned %d", old_file, new_file, ret) return if dest_file: utils.unslurp(dest_file, diff) logging.info("Difference between %s and %s written to %s", old_file, new_file, dest_file) return diff
def get_known_remote(package_dir): """Return the first remote in the current directory's list of remotes which is on osg-build's configured whitelist of remotes.""" top_dir = os.path.split(os.path.abspath(package_dir))[0] command = [ "git", "--work-tree", top_dir, "--git-dir", os.path.join(top_dir, ".git"), "remote", "-v" ] out, err = utils.sbacktick(command, err2out=True) if err: raise GitError( "Exit code %d getting git status for directory %s. Output:\n%s" % (err, package_dir, out)) for line in out.splitlines(): info = line.strip().split() if len(info) != 3: continue if info[2] != '(fetch)': continue if info[1] in constants.KNOWN_GIT_REMOTES: return info[0], info[1] raise GitError( "OSG remote not found for directory %s; are remotes configurated correctly?" % package_dir)