def verify_package_dir(package_dir):
"""Check if package_dir points to a valid package dir (i.e. contains
at least an osg/ dir or an upstream/ dir) and is in a git repo.
"""
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "rev-parse", "--show-toplevel"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git top-level directory of %s. Output:\n%s" %
(err, package_dir, out))
if top_dir != out.strip():
raise GitError("Specified package directory (%s) is not a top-level directory in the git repo (%s)." % \
(package_dir, top_dir))
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "ls-files", "osg", "upstream"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git subdirectories of %s. Output:\n%s" %
(err, package_dir, out))
for line in out.split("\n"):
if line.startswith('osg/') or line.startswith('upstream/'):
return True
return False
def is_outdated(package_dir):
"""Return True if the package has been changed since the revision in the
local git repo.
"""
remote = get_current_branch_remote(package_dir)
branch = get_branch(package_dir)
branch_ref = "refs/heads/%s" % branch
branch_hash = ''
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "show-ref"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git references for directory %s. Output:\n%s",
(err, package_dir, out))
for line in out.splitlines():
info = line.strip().split()
if len(info) != 2:
continue
if info[1] == branch_ref:
branch_hash = info[0]
break
if not branch_hash:
raise GitError("Unable to determine local branch's hash.")
out, err = utils.sbacktick([
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "ls-remote", "--heads", remote
])
if err:
raise GitError(
"Exit code %d getting remote git status for directory %s. Output:\n%s"
% (err, package_dir, out))
remote_hash = ''
for line in out.splitlines():
info = line.strip().split()
if len(info) != 2:
continue
if info[1] == branch_ref:
remote_hash = info[0]
break
if not remote_hash:
raise GitError("Unable to determine remote branch's hash.")
if remote_hash == branch_hash:
return False
print "Remote hash (%s) does not match local hash (%s) for branch %s." % (
remote_hash, branch_hash, branch)
return True
def is_uncommitted(package_dir):
"""Return True if there are uncommitted changes or files in the git working dir."""
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "status", "--porcelain"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git status for directory %s. Output:\n%s" %
(err, package_dir, out))
if out:
print "The following uncommitted changes exist:"
print out
print "Please commit these first."
return True
remote = get_current_branch_remote(package_dir)
branch = get_branch(package_dir)
branch_ref = "refs/heads/%s" % branch
origin_ref = "refs/remotes/%s/%s" % (remote, branch)
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "show-ref"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git references for directory %s. Output:\n%s",
(err, package_dir, out))
branch_hash = ''
origin_hash = ''
for line in out.splitlines():
info = line.split()
if len(info) != 2:
continue
if info[1] == branch_ref:
branch_hash = info[0]
if info[1] == origin_ref:
origin_hash = info[0]
if not branch_hash and not origin_hash:
raise GitError(
"Could not find either local or remote hash for directory %s." %
package_dir)
if branch_hash != origin_hash:
raise GitError("Local hash (%s) does not match remote hash "
"(%s) for directory %s. Perhaps you need to perform 'git push'?" % \
(branch_hash, origin_hash, package_dir))
return False
def is_outdated(package_dir):
"""Return True if the package has been changed since the revision in the
SVN working dir.
"""
if utils.is_url(package_dir):
return False
out, err = utils.sbacktick("svn status -u -q " + package_dir)
if err:
raise SVNError("Exit code %d getting SVN status. Output:\n%s" %
(err, out))
outdated_files = []
for line in out.split("\n"):
try:
outdated_flag = line[8]
except IndexError:
continue
if outdated_flag == "*":
outdated_files.append(line)
if outdated_files:
print "The following outdated files exist:"
print "\n".join(outdated_files)
return True
else:
return False
def is_untracked_path(path):
"""Return True if the given path is untracked in SVN.
Note: ignored files return False.
"""
output, ret = utils.sbacktick(["svn", "status", path])
return output.startswith('?')
def diff3(old_file, orig_file, new_file, dest_file=None):
"""Do a 3-way diff between `old_file`, `orig_file`, and `new_file`,
where the differences are shown with markers like what SVN makes
for a file with merge conflicts, e.g.:
'''
<<<<<<< old_file
old stuff
||||||| orig_file
orig stuff
=======
new stuff
>>>>>>> new_file
'''
Write the result to `dest_file` if it is specified.
Return the text of the diff on success, None on failure.
"""
diff, ret = utils.sbacktick(["diff3", "-m", old_file, orig_file, new_file])
if not (ret == 0 or ret == 1):
logging.warning("Error diffing %s %s %s: diff3 returned %d", old_file,
orig_file, new_file, ret)
return
if dest_file:
utils.unslurp(dest_file, diff)
logging.info("Difference between %s, %s, and %s written to %s",
old_file, orig_file, new_file, dest_file)
return diff
def is_untracked_path(path):
"""Return True if the given path is untracked in SVN.
Note: ignored files return False.
"""
output, ret = utils.sbacktick(["svn", "status", path])
return output.startswith('?')
def diff2(old_file, new_file, dest_file=None):
"""Do a 2-way diff, between `old_file` and `new_file`, where the
differences are shown with markers like what SVN makes for a file
with merge conflicts, e.g.:
'''
<<<<<<< old_file
old stuff
=======
new stuff
>>>>>>> new_file
'''
Write the result to `dest_file` if it is specified.
Return the text of the diff on success, None on failure.
"""
diff, ret = utils.sbacktick(["diff", """\
--changed-group-format=<<<<<<< %(old_file)s
%%<=======
%%>>>>>>>> %(new_file)s
""" % locals(), old_file, new_file])
if not (ret == 0 or ret == 1):
logging.warning("Error diffing %s %s: diff returned %d",
old_file, new_file, ret)
return
if dest_file:
utils.unslurp(dest_file, diff)
logging.info("Difference between %s and %s written to %s",
old_file, new_file, dest_file)
return diff
def koji(package_dir, koji_obj, buildopts):
"""koji task with a git build."""
package_dir = os.path.abspath(package_dir)
verify_package_dir(package_dir)
package_name = os.path.basename(package_dir)
if not re.match(r"\w+", package_name): # sanity check
raise Error("Package directory '%s' gives invalid package name '%s'" %
(package_dir, package_name))
if not buildopts.get('scratch'):
koji_obj.add_pkg(package_name)
remote = get_fetch_url(package_dir, get_known_remote(package_dir)[0])
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "log", "-1", "--pretty=format:%H"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git hash for directory %s. Output:\n%s" %
(err, package_dir, out))
rev = out.strip()
return koji_obj.build_git(remote, rev, package_name)
def diff3(old_file, orig_file, new_file, dest_file=None):
"""Do a 3-way diff between `old_file`, `orig_file`, and `new_file`,
where the differences are shown with markers like what SVN makes
for a file with merge conflicts, e.g.:
'''
<<<<<<< old_file
old stuff
||||||| orig_file
orig stuff
=======
new stuff
>>>>>>> new_file
'''
Write the result to `dest_file` if it is specified.
Return the text of the diff on success, None on failure.
"""
diff, ret = utils.sbacktick(["diff3", "-m", old_file, orig_file, new_file])
if not (ret == 0 or ret == 1):
logging.warning("Error diffing %s %s %s: diff3 returned %d",
old_file, orig_file, new_file, ret)
return
if dest_file:
utils.unslurp(dest_file, diff)
logging.info("Difference between %s, %s, and %s written to %s",
old_file, orig_file, new_file, dest_file)
return diff
def get_sha1sum(file_path):
"""Return the SHA1 checksum of the file located at `file_path` as a string."""
out, ret = utils.sbacktick(["sha1sum", file_path])
if ret != 0:
raise Error("Unable to get sha1sum of %s: exit %d when running sha1sum" % (file_path, ret))
match = re.match(r"[a-f0-9]{40}", out)
if not match:
raise Error("Unable to get sha1sum of %s: unexpected output: %s" % (file_path, out))
return match.group(0)
def get_targets(self):
"""Get a list of the names of targets (as strings) from koji"""
out, err = utils.sbacktick(self.koji_cmd + ["list-targets", "--quiet"])
if err:
raise KojiError("koji list-targets failed with exit code " +
str(err))
lines = out.split("\n")
target_names = [re.split(r"\s+", x)[0] for x in lines]
return target_names
def is_uncommitted(package_dir):
"""Return True if there are uncommitted changes in the SVN working dir."""
out, err = utils.sbacktick("svn status -q " + package_dir)
if err:
raise SVNError("Exit code %d getting SVN status. Output:\n%s" % (err, out))
if out:
print "The following uncommitted changes exist:"
print out
return True
else:
return False
def srpm_nv(srpm):
"""Return the NV (Name, Version) from an SRPM."""
output, ret = utils.sbacktick(
["rpm", "-qp", "--qf", "%{name} %{version}", srpm])
if ret == 0:
try:
name, version = output.rstrip().split(" ")
return name, version
except ValueError: # not enough/too many items
pass
raise Error("Unable to extract name and version from SRPM %s: %s" %
(srpm, output))
def search_names(self, terms, stype, match):
search_subcmd = ["search", stype]
if match == 'regex':
search_subcmd.append("--regex")
elif match == 'exact':
search_subcmd.append("--exact")
search_subcmd.append(terms)
out, err = utils.sbacktick(self.koji_cmd + search_subcmd)
if err:
raise KojiError("koji search failed with exit code " + str(err))
return out.split("\n")
def is_uncommitted(package_dir):
"""Return True if there are uncommitted changes in the SVN working dir."""
out, err = utils.sbacktick("svn status -q " + package_dir)
if err:
raise SVNError("Exit code %d getting SVN status. Output:\n%s" %
(err, out))
if out:
print "The following uncommitted changes exist:"
print out
return True
else:
return False
def get_sha1sum(file_path):
"""Return the SHA1 checksum of the file located at `file_path` as a string."""
out, ret = utils.sbacktick(["sha1sum", file_path])
if ret != 0:
raise Error(
"Unable to get sha1sum of %s: exit %d when running sha1sum" %
(file_path, ret))
match = re.match(r"[a-f0-9]{40}", out)
if not match:
raise Error("Unable to get sha1sum of %s: unexpected output: %s" %
(file_path, out))
return match.group(0)
def print_version_and_exit():
"""Print version and exit"""
# '@'+'VERSION'+'@' is so sed will leave it alone during 'make dist'
if __version__ == '@' + 'VERSION' + '@':
print "osg-build SVN"
out, ret = utils.sbacktick("svn info " + sys.argv[0], err2out=True)
if ret:
print "no info"
else:
print "SVN info:\n" + out
else:
print "osg-build " + __version__
sys.exit(0)
def print_version_and_exit():
"""Print version and exit"""
# '@'+'VERSION'+'@' is so sed will leave it alone during 'make dist'
if __version__ == '@' + 'VERSION' + '@':
print "osg-build SVN"
out, ret = utils.sbacktick("svn info " + sys.argv[0], err2out=True)
if ret:
print "no info"
else:
print "SVN info:\n" + out
else:
print "osg-build " + __version__
sys.exit(0)
def verify_package_info(package_info):
"""Check if package_info points to a valid package dir (i.e. contains
at least an osg/ dir or an upstream/ dir).
"""
url = package_info['canon_url']
rev = package_info['revision']
command = ["svn", "ls", url, "-r", rev]
out, err = utils.sbacktick(command, clocale=True, err2out=True)
if err:
raise SVNError("Exit code %d getting SVN listing of %s (rev %s). Output:\n%s" % (err, url, rev, out))
for line in out.split("\n"):
if line.startswith('osg/') or line.startswith('upstream/'):
return True
return False
def get_current_branch_remote(package_dir):
"""Return the configured remote for the current branch."""
branch = get_branch(package_dir)
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "config",
"branch.%s.remote" % branch
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError("Exit code %d getting git branch %s remote for directory '%s'. Output:\n%s" % \
(err, branch, package_dir, out))
return out.strip()
def get_branch(package_dir):
"""Return the current git branch for a given directory."""
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "branch"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git branch for directory %s. Output:\n%s" %
(err, package_dir, out))
out = out.strip()
if not out:
raise GitError("'git branch' returned no output.")
return out.split()[-1]
def verify_package_info(package_info):
"""Check if package_info points to a valid package dir (i.e. contains
at least an osg/ dir or an upstream/ dir).
"""
url = package_info['canon_url']
rev = package_info['revision']
command = ["svn", "ls", url, "-r", rev]
out, err = utils.sbacktick(command, clocale=True, err2out=True)
if err:
raise SVNError(
"Exit code %d getting SVN listing of %s (rev %s). Output:\n%s" %
(err, url, rev, out))
for line in out.split("\n"):
if line.startswith('osg/') or line.startswith('upstream/'):
return True
return False
def verify_git_svn_commit(package_dir):
"""Verify the last commit in the git repo actually came from git-svn."""
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "log", "-n", "1"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git log for directory %s. Output:\n%s" %
(err, package_dir, out))
for line in out.splitlines():
if line.find("git-svn-id:") >= 0:
return
raise GitError(
"Last git commit not from SVN - possible inconsistency between git and SVN!"
)
def get_package_info(package_dir, rev=None):
"""Return the svn info for a package dir."""
command = ["svn", "info", package_dir]
if rev:
command += ["-r", rev]
else:
command += ["-r", "HEAD"]
out, err = utils.sbacktick(command, clocale=True, err2out=True)
if err:
raise SVNError("Exit code %d getting SVN info. Output:\n%s" %
(err, out))
info = dict()
for line in out.split("\n"):
label, value = line.strip().split(": ", 1)
label = label.strip().lower().replace(' ', '_')
info[label] = value
info['canon_url'] = re.sub("^" + re.escape(info['repository_root']),
SVN_ROOT, info['url'])
return info
def get_spec_name_in_srpm(srpm):
"""Return the name of the spec file present in an SRPM. Assumes
there is exactly one spec file in the SRPM -- if there is more than
one spec file, returns the name of the first one ``cpio'' prints.
"""
out, ret = utils.sbacktick("rpm2cpio %s | cpio -t '*.spec' 2> /dev/null" %
utils.shell_quote(srpm),
shell=True)
if ret != 0:
raise Error("Unable to get list of spec files from %s" % srpm)
try:
spec_name = [_f for _f in [x.strip() for x in out.split("\n")]
if _f][0]
except IndexError:
spec_name = None
if not spec_name:
raise Error("No spec file inside %s" % srpm)
return spec_name
def get_package_info(package_dir):
"""Return the svn info for a package dir."""
command = ["svn", "info", package_dir]
# If we don't specify the revision in the argument (e.g. no foo@19999)
# then explicitly specify HEAD to make sure we're not getting an older
# version.
if not re.search(r'@\d+$', package_dir):
command += ['-r', 'HEAD']
out, err = utils.sbacktick(command, err2out=True)
if err:
raise SVNError("Exit code %d getting SVN info. Output:\n%s" %
(err, out))
info = dict()
for line in out.split("\n"):
label, value = line.strip().split(": ", 1)
label = label.strip().lower().replace(' ', '_')
info[label] = value
info['canon_url'] = re.sub("^" + re.escape(info['repository_root']),
SVN_ROOT, info['url'])
return info
def make_srpm(self, spec_fn):
"""Make an SRPM from a spec file. Raise OSGPrebuildError on failure"""
cmd = (["rpmbuild", "-bs", "--nodeps"] +
self.get_rpmbuild_defines(prebuild=True) +
[spec_fn])
err_msg_prefix = ("Error making SRPM from %s\n"
"Command used was: %s\n" %
(spec_fn, " ".join(cmd)))
out, err = utils.sbacktick(cmd, nostrip=True, clocale=True, err2out=True)
if err:
log.error("Rpmbuild failed. Output follows: " + out)
raise OSGPrebuildError(err_msg_prefix +
"Rpmbuild return code %d" % err)
match = re.search(r"(?ms)^Wrote: ([^\n]+.src.rpm)$", out)
if match:
srpm = match.group(1).strip()
if os.path.isfile(srpm):
return srpm
raise OSGPrebuildError(err_msg_prefix +
"Unable to find resulting SRPM.")
def lint(self):
"""lint task. Prebuild the package and run rpmlint on the SRPM."""
if not utils.which("rpmlint"):
raise ProgramNotFoundError("rpmlint")
conf_file = utils.find_file("rpmlint.cfg", DATA_FILE_SEARCH_PATH)
if not conf_file:
raise FileNotFoundError("rpmlint.cfg", DATA_FILE_SEARCH_PATH)
srpm = self.prebuild()
lint_output, lint_returncode = utils.sbacktick(
["rpmlint", "-f", conf_file, srpm])
print lint_output
if lint_returncode == 0:
print "rpmlint ok for " + self.package_name
elif lint_returncode < 64:
print "Error running rpmlint for " + self.package_name
elif lint_returncode == 64:
print "rpmlint found problems with " + self.package_name
elif lint_returncode == 66:
print "rpmlint found many problems with " + self.package_name
else:
print "unrecognized return code from rpmlint: " + str(lint_returncode)
def is_svn(package_dir):
"""Determine whether a given directory is part of an SVN repo."""
# If package_dir is a URL, not a directory, then we can't cd into it to
# check. Assume True for now.
if utils.is_url(package_dir):
return True
# TODO: Allow specifying a git URL to build from.
pwd = os.getcwd()
try:
try:
os.chdir(package_dir)
except OSError, ose:
if ose.errno == errno.ENOENT:
raise Error("%s is not a valid package directory\n(%s)" %
(package_dir, ose))
command = ["svn", "info"]
try:
err = utils.sbacktick(command, err2out=True)[1]
except OSError, ose:
if ose.errno != errno.ENOENT:
raise
err = 1
def get_fetch_url(package_dir, remote):
"""Return a fetch url
is on osg-build's configured whitelist of remotes."""
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "remote", "-v"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git status for directory %s. Output:\n%s" %
(err, package_dir, out))
for line in out.splitlines():
info = line.strip().split()
if len(info) != 3:
continue
if info[2] != '(fetch)':
continue
if info[0] == remote:
return constants.GIT_REMOTE_MAPS.setdefault(info[1], info[1])
raise GitError("Remote URL not found for remote %s in directory %s; are remotes " \
"configured correctly?" % (remote, package_dir))
def diff2(old_file, new_file, dest_file=None):
"""Do a 2-way diff, between `old_file` and `new_file`, where the
differences are shown with markers like what SVN makes for a file
with merge conflicts, e.g.:
'''
<<<<<<< old_file
old stuff
=======
new stuff
>>>>>>> new_file
'''
Write the result to `dest_file` if it is specified.
Return the text of the diff on success, None on failure.
"""
diff, ret = utils.sbacktick([
"diff",
"""\
--changed-group-format=<<<<<<< %(old_file)s
%%<=======
%%>>>>>>>> %(new_file)s
""" % locals(), old_file, new_file
])
if not (ret == 0 or ret == 1):
logging.warning("Error diffing %s %s: diff returned %d", old_file,
new_file, ret)
return
if dest_file:
utils.unslurp(dest_file, diff)
logging.info("Difference between %s and %s written to %s", old_file,
new_file, dest_file)
return diff
def get_known_remote(package_dir):
"""Return the first remote in the current directory's list of remotes which
is on osg-build's configured whitelist of remotes."""
top_dir = os.path.split(os.path.abspath(package_dir))[0]
command = [
"git", "--work-tree", top_dir, "--git-dir",
os.path.join(top_dir, ".git"), "remote", "-v"
]
out, err = utils.sbacktick(command, err2out=True)
if err:
raise GitError(
"Exit code %d getting git status for directory %s. Output:\n%s" %
(err, package_dir, out))
for line in out.splitlines():
info = line.strip().split()
if len(info) != 3:
continue
if info[2] != '(fetch)':
continue
if info[1] in constants.KNOWN_GIT_REMOTES:
return info[0], info[1]
raise GitError(
"OSG remote not found for directory %s; are remotes configurated correctly?"
% package_dir)
