def ensure_existing_ok(hashes, spec_path):
pkg_name = name_from_spec(spec_path)
one_correct = False
for srpm in glob.glob(os.path.join(SRPMS_DIR, '%s-*.src.rpm' % pkg_name)):
# Check it's for the right package:
cmd = ["rpm", "-qp", srpm, "--qf", "%{name}"]
result = run(cmd)['stdout'].strip().split('\n')
if result[0] == pkg_name:
cmd = ["rpm", "--dump", "-qp", srpm]
result = run(cmd)['stdout'].strip().split('\n')
is_ok = True
for line in result:
split = line.split()
fname = split[0]
thishash = split[3]
if fname not in hashes or hashes[fname] != thishash:
is_ok = False
if not is_ok:
print_col(bcolours.WARNING,
"WARNING: Removing SRPM '%s' "
"(hash mismatch with desired)" % srpm)
os.remove(srpm)
else:
one_correct = True
return one_correct
def pin(self):
dotgitdir = os.path.join(self.localpath, ".git")
# First, get the hash of the commit
cmd = ["git", "--git-dir=%s" % dotgitdir, "rev-parse", "HEAD"]
scmhash = run(cmd)['stdout'].strip()
# Verified good hash.
self.scmhash = scmhash
# Now lets describe that hash.
cmd = [
"git",
"--git-dir=%s" % dotgitdir, "describe", "--tags", scmhash
]
description = run(cmd, check=False)['stdout'].strip()
# if there are no tags, get the number of commits, which should
# always increase
if description == "":
cmd = [
"git",
"--git-dir=%s" % dotgitdir, "log", scmhash, "--oneline"
]
commits = run(cmd)['stdout'].strip()
description = str(len(commits.splitlines()))
match = re.search("[^0-9]*", description)
matchlen = len(match.group())
self.version = description[matchlen:].replace('-', '+')
def ensure_existing_ok(hashes, spec_path):
pkg_name = name_from_spec(spec_path)
one_correct = False
for srpm in glob.glob(os.path.join(SRPMS_DIR, '%s-*.src.rpm' % pkg_name)):
# Check it's for the right package:
cmd = ["rpm", "-qp", srpm, "--qf", "%{name}"]
result = run(cmd)['stdout'].strip().split('\n')
if result[0] == pkg_name:
cmd = ["rpm", "--dump", "-qp", srpm]
result = run(cmd)['stdout'].strip().split('\n')
ok = True
for line in result:
split = line.split()
fname = split[0]
thishash = split[3]
if fname not in hashes or hashes[fname] != thishash:
ok = False
if not ok:
print_col(
bcolours.WARNING,
"WARNING: Removing SRPM '%s' (hash mismatch with desired)"
% srpm)
os.remove(srpm)
else:
one_correct = True
return one_correct
def describe(repo, treeish="HEAD"):
"""
Return an RPM compatible version string for a git repo at a given commit
"""
dotgitdir = os.path.join(repo, ".git")
if not os.path.exists(dotgitdir):
raise Exception("Pin target is not a git repository: '%s'" % repo)
# First, get the hash of the commit
cmd = ["git", "--git-dir=%s" % dotgitdir, "rev-parse", treeish]
sha = run(cmd)['stdout'].strip()
# Now lets describe that hash
cmd = ["git", "--git-dir=%s" % dotgitdir, "describe", "--tags", sha]
description = run(cmd, check=False)['stdout'].strip()
# if there are no tags, use the number of commits
if description == "":
cmd = ["git", "--git-dir=%s" % dotgitdir, "log", "--oneline", sha]
commits = run(cmd)['stdout'].strip()
description = str(len(commits.splitlines()))
# replace '-' with '+' in description to not confuse rpm
match = re.search("[^0-9]*", description)
matchlen = len(match.group())
return description[matchlen:].replace('-', '+')
def pin(self):
dotgitdir = os.path.join(self.localpath, ".git")
if not os.path.exists(dotgitdir):
raise exceptions.NoRepository
# First, get the hash of the commit
cmd = ["git", "--git-dir=%s" % dotgitdir,
"rev-parse", "HEAD"]
scmhash = run(cmd)['stdout'].strip()
# Verified good hash.
self.scmhash = scmhash
# Now lets describe that hash.
cmd = ["git", "--git-dir=%s" % dotgitdir,
"describe", "--tags", scmhash]
description = run(cmd, check=False)['stdout'].strip()
# if there are no tags, get the number of commits, which should
# always increase
if description == "":
cmd = ["git", "--git-dir=%s" % dotgitdir,
"log", scmhash, "--oneline"]
commits = run(cmd)['stdout'].strip()
description = str(len(commits.splitlines()))
match = re.search("[^0-9]*", description)
matchlen = len(match.group())
self.version = description[matchlen:].replace('-', '+')
def pin(self):
cmd = ["hg", "-R", self.localpath, "tip", "--template", "{node}"]
scmhash = run(cmd)['stdout'].strip()
self.scmhash = scmhash
cmd = ["hg", "-R", self.localpath, "parents", "--template", "{rev}"]
description = run(cmd)['stdout'].strip()
self.version = str(description)
def pin(self):
cmd = ["hg", "-R", self.localpath, "tip", "--template", "{node}"]
scmhash = run(cmd)['stdout'].strip()
self.scmhash = scmhash
cmd = ["hg", "-R", self.localpath, "parents", "--template", "{rev}"]
description = run(cmd)['stdout'].strip()
self.version = str(description)
def archive(self, sources_dir=SOURCES_DIR):
# If it already exists, we're done.
if os.path.exists(os.path.join(sources_dir, self.archivename)):
print "File's already here!"
return
print "File's not here!"
cmd = ["hg", "-R", self.localpath, "archive", "-t", "tgz", "-p",
"%s/" % self.tarballprefix,
"%s/%s" % (sources_dir, self.archivename)]
run(cmd)
def prepare_buildroot():
"""Create a clean rpmbuild directory structure"""
if os.path.exists(SPECS_DIR):
shutil.rmtree(SPECS_DIR)
os.makedirs(SPECS_DIR)
for path in [SRPMS_DIR, SOURCES_DIR, RPMS_DIR]:
if not os.path.exists(path):
os.makedirs(path)
cmd = (["createrepo", RPMS_DIR])
run(cmd)
def archive(repo, commit_hash, output, prefix=None):
"""
Archive a git repo at a given commit with a specified version prefix.
Returns the path to an archive to be used as a source for building an RPM.
"""
dotgitdir = dotgitdir_of_path(repo)
cmd = [
"git",
"--git-dir=%s" % dotgitdir, "archive", commit_hash, "-o", output
]
if prefix is not None:
cmd += ["--prefix=%s-%s/" % (os.path.basename(repo), prefix)]
run(cmd)
def archive(self, sources_dir=SOURCES_DIR):
# If it already exists, we're done.
if os.path.exists(os.path.join(sources_dir, self.archivename)):
print "File's already here!"
return
print "File's not here!"
cmd = [
"hg", "-R", self.localpath, "archive", "-t", "tgz", "-p",
"%s/" % self.tarballprefix,
"%s/%s" % (sources_dir, self.archivename)
]
run(cmd)
def build_package(configdir, root, passthrough_args):
"""
Spawn a mock process to build the package. Some arguments
are intercepted and rewritten, for instance --resultdir.
"""
working_directory = tempfile.mkdtemp(prefix="planex-cache")
logging.debug("Mock working directory: %s", working_directory)
cmd = ["mock", "--configdir=%s" % configdir,
"--root=%s" % root,
"--resultdir=%s" % working_directory] + passthrough_args
util.run(cmd)
return working_directory
def archive(repo, commit_hash, prefix, target_dir):
"""
Archive a git repo at a given commit with a specified version prefix.
Returns the path to a tar.gz to be used as a source for building an RPM.
"""
dotgitdir = dotgitdir_of_path(repo)
prefix = "%s-%s" % (os.path.basename(repo), prefix)
path = os.path.join(target_dir, "%s.tar" % prefix)
run(["git", "--git-dir=%s" % dotgitdir, "archive", commit_hash,
"--prefix=%s/" % prefix, "-o", path])
run(["gzip", "--no-name", "-f", path])
return path + ".gz"
def build_srpm(hashes, spec_path):
"""
Builds an SRPM from the spec file at spec_path.
Assumes that all source files have already been downloaded to
the rpmbuild sources directory, and are correctly named.
"""
is_ok = ensure_existing_ok(hashes, spec_path)
if not is_ok:
cmd = (["rpmbuild", "-bs", spec_path,
"--nodeps", "--define", "_topdir %s" % BUILD_ROOT_DIR])
run(cmd)
return 1
else:
return 0
def do_build(srpm, target, build_number, use_mock, xs_build_sys):
if xs_build_sys:
mock = ["/usr/bin/mock"]
else:
mock = ["planex-cache", "--debug"]
if use_mock:
cmd = mock + ["--configdir=%s" % MOCK_DIR,
"--resultdir=%s" % TMP_RPM_PATH, "--rebuild",
"--target", target,
# "--enable-plugin=tmpfs",
"--define", "extrarelease .%d" % build_number,
"-v"]
if not xs_build_sys:
cmd = cmd + ["--disable-plugin=package_state"]
else:
cmd = ["rpmbuild", "--rebuild", "-v",
"--target", target, "--define",
"_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm"]
res = run(cmd + [srpm])
print "stdout: %s" % res['stdout']
srpms = glob.glob(os.path.join(TMP_RPM_PATH, "*.src.rpm"))
for srpm in srpms:
print_col(bcolours.WARNING, "Removing SRPM %s" % srpm)
os.unlink(srpm)
return glob.glob(os.path.join(TMP_RPM_PATH, "*.rpm"))
def get_srpm_info(srpm):
for spec_path in glob.glob(SPECS_GLOB):
os.unlink(spec_path)
myenv = os.environ.copy()
myenv['HOME'] = RPM_TOP_DIR
run(["rpm", "-i", srpm], check=True, env=myenv)
myspecfile = glob.glob(SPECS_GLOB)[0]
spec = rpm.ts().parseSpec(myspecfile)
info = {}
info['deps'] = spec.sourceHeader["requires"]
info['arch'] = DEFAULT_ARCH
info['packages'] = [{'name': p.header['name']} for p in spec.packages]
info['srcrpm'] = srpm
content_file = open(myspecfile, 'r')
info['spec'] = content_file.read()
content_file.close()
return info
def current_branch(repo):
"""
Return the name of the current branch on repo. Requires git 1.7+.
"""
return run(
["git",
"--work-tree=%s" % repo, "rev-parse", "--abbrev-ref",
"HEAD"])['stdout'].strip()
def archive(self, sources_dir=SOURCES_DIR):
# If it already exists, we're done.
dotgitdir = os.path.join(self.localpath, ".git")
if os.path.exists(os.path.join(sources_dir, self.archivename)):
return
# archive name always ends in .gz - strip it off
tarball_name = self.archivename[:-3]
cmd = ["git", "--git-dir=%s" % dotgitdir, "archive",
"--prefix=%s/" % self.tarballprefix, self.scmhash, "-o",
"%s/%s" % (sources_dir, tarball_name)]
run(cmd)
cmd = ["gzip", "-f", "%s/%s" % (sources_dir, tarball_name)]
run(cmd)
def build_package(configdir, root, passthrough_args):
"""
Spawn a mock process to build the package. Some arguments
are intercepted and rewritten, for instance --resultdir.
"""
working_directory = tempfile.mkdtemp(prefix="planex-cache")
logging.debug("Mock working directory: %s", working_directory)
cmd = ["/usr/bin/mock", "--configdir=%s" % configdir,
"--root=%s" % root,
"--resultdir=%s" % working_directory] + passthrough_args
logfiles = [os.path.join(working_directory, "root.log"),
os.path.join(working_directory, "build.log")]
util.run(cmd, logfiles=logfiles)
return working_directory
def build_srpm(hashes, spec_path):
"""
Builds an SRPM from the spec file at spec_path.
Assumes that all source files have already been downloaded to
the rpmbuild sources directory, and are correctly named.
"""
is_ok = ensure_existing_ok(hashes, spec_path)
if not is_ok:
cmd = ([
"rpmbuild", "-bs", spec_path, "--nodeps", "--define",
"_topdir %s" % BUILD_ROOT_DIR
])
run(cmd)
return 1
else:
return 0
def archive(repo, commit_hash, prefix, target_dir):
"""
Archive a git repo at a given commit with a specified version prefix.
Returns the path to a tar.gz to be used as a source for building an RPM.
"""
dotgitdir = dotgitdir_of_path(repo)
prefix = "%s-%s" % (os.path.basename(repo), prefix)
path = os.path.join(target_dir, "%s.tar" % prefix)
run([
"git",
"--git-dir=%s" % dotgitdir, "archive", commit_hash,
"--prefix=%s/" % prefix, "-o", path
])
run(["gzip", "--no-name", "-f", path])
return path + ".gz"
def origin_url(repo):
"""
Return the remote url for origin
"""
dotgitdir = dotgitdir_of_path(repo)
res = run(['git', '--git-dir=%s' % dotgitdir, 'remote', '-v'])
remotes = res['stdout'].strip()
match = re.search(r'origin\s*(\S*)\s*\(fetch\)', remotes)
return match.group(1).strip()
def origin_url(repo):
"""
Return the remote url for origin
"""
dotgitdir = dotgitdir_of_path(repo)
res = run(['git', '--git-dir=%s' % dotgitdir, 'remote', '-v'])
remotes = res['stdout'].strip()
match = re.search(r'origin\s*(\S*)\s*\(fetch\)', remotes)
return match.group(1).strip()
def archive(self, sources_dir=SOURCES_DIR):
# If it already exists, we're done.
dotgitdir = os.path.join(self.localpath, ".git")
if os.path.exists(os.path.join(sources_dir, self.archivename)):
return
# archive name always ends in .gz - strip it off
tarball_name = self.archivename[:-3]
cmd = [
"git",
"--git-dir=%s" % dotgitdir, "archive",
"--prefix=%s/" % self.tarballprefix, self.scmhash, "-o",
"%s/%s" % (sources_dir, tarball_name)
]
run(cmd)
cmd = ["gzip", "-f", "%s/%s" % (sources_dir, tarball_name)]
run(cmd)
def format_patch(repo, startref, endref, target_dir):
"""
Write patches from ref to HEAD out to target_dir.
Returns a list of patch filenames which can be used to create a
series file.
"""
dotgitdir = dotgitdir_of_path(repo)
commit_range = "%s..%s" % (startref, endref)
res = run(["git", "--git-dir=%s" % dotgitdir, "format-patch",
"--no-renames", commit_range, "--output-directory", target_dir])
return res['stdout'].split()
def format_patch(repo, startref, endref, target_dir):
"""
Write patches from ref to HEAD out to target_dir.
Returns a list of patch filenames which can be used to create a
series file.
"""
dotgitdir = dotgitdir_of_path(repo)
commit_range = "%s..%s" % (startref, endref)
res = run([
"git",
"--git-dir=%s" % dotgitdir, "format-patch", "--no-renames",
commit_range, "--output-directory", target_dir
])
return res['stdout'].split()
def describe(repo, treeish="HEAD"):
"""
Return an RPM compatible version string for a git repo at a given commit
"""
dotgitdir = dotgitdir_of_path(repo)
# First, get the hash of the commit
cmd = ["git", "--git-dir=%s" % dotgitdir, "rev-parse", treeish]
sha = run(cmd)['stdout'].strip()
# Now lets describe that hash
cmd = ["git", "--git-dir=%s" % dotgitdir, "describe", "--tags", sha]
description = run(cmd, check=False)['stdout'].strip()
# if there are no tags, use the number of commits
if description == "":
cmd = ["git", "--git-dir=%s" % dotgitdir, "log", "--oneline", sha]
commits = run(cmd)['stdout'].strip()
description = str(len(commits.splitlines()))
# replace '-' with '+' in description to not confuse rpm
match = re.search("[^0-9]*", description)
matchlen = len(match.group())
return description[matchlen:].replace('-', '+')
def best_effort_file_verify(path):
"""
Given a path, check if the file at that path has a sensible format.
If the file has an extension then it checks that the mime-type of this file
matches that of the file extension as defined by the IANA:
http://www.iana.org/assignments/media-types/media-types.xhtml
"""
_, ext = os.path.splitext(path)
if ext and ext in SUPPORTED_EXT_TO_MIME:
# output of `file` is of form: "<path>: <mime-type>"
cmd = ["file", "--mime-type", path]
stdout = run(cmd, check=False)["stdout"].strip()
_, _, mime_type = stdout.partition(": ")
if SUPPORTED_EXT_TO_MIME[ext] != mime_type:
sys.exit("%s: Fetched file format looks incorrect: %s: %s" % (sys.argv[0], path, mime_type))
def best_effort_file_verify(path):
"""
Given a path, check if the file at that path has a sensible format.
If the file has an extension then it checks that the mime-type of this file
matches that of the file extension as defined by the IANA:
http://www.iana.org/assignments/media-types/media-types.xhtml
"""
_, ext = os.path.splitext(path)
if ext and ext in SUPPORTED_EXT_TO_MIME:
# output of `file` is of form: "<path>: <mime-type>"
cmd = ["file", "--mime-type", path]
stdout = run(cmd, check=False)['stdout'].strip()
_, _, mime_type = stdout.partition(': ')
if SUPPORTED_EXT_TO_MIME[ext] != mime_type:
sys.exit("%s: Fetched file format looks incorrect: %s: %s" %
(sys.argv[0], path, mime_type))
def get_hashes(ty):
spec_files = glob.glob(os.path.join(SPECS_DIR,"*"))
sources_files = glob.glob(os.path.join(SOURCES_DIR,"*"))
all_files = spec_files + sources_files
if ty=="md5":
cmd = ["md5sum"] + all_files
elif ty=="sha256":
cmd = ["sha256sum"] + all_files
else:
print "Invalid hash type"
raise Exception
result = run(cmd)['stdout'].strip().split('\n')
def fix(x):
words = x.split()
if len(words) == 2:
fname = words[1].split("/")[-1]
return (fname,words[0])
else:
return None
fixed = [fix(x) for x in result]
return dict(fixed)
def build_srpm(srpm, srpm_infos, external, deps, use_mock, xs_build_sys):
cache_dir = get_cache_dir(srpm_infos, external, deps, srpm)
if need_to_build(srpm_infos, external, deps, srpm):
target = extract_target(srpm_infos, srpm)
build_number = get_new_number(srpm, cache_dir)
print_col(bcolours.OKGREEN,
"CACHE MISS: Building %s (%d)" % (srpm, build_number))
createrepo()
pkgs = do_build(srpm, target, build_number, use_mock, xs_build_sys)
if cache_dir:
try:
os.makedirs(cache_dir + ".tmp")
print "Archiving result in cache"
for pkg in pkgs:
shutil.copy(pkg, cache_dir + ".tmp")
os.rename(cache_dir + ".tmp", cache_dir)
except (OSError, IOError):
print bcolours.WARNING + \
"FAILED TO PUT BUILD RESULTS INTO CACHE"
else:
print_col(bcolours.OKGREEN, "CACHE HIT: Not building %s" % srpm)
pkgs = glob.glob(os.path.join(cache_dir, "*.rpm"))
for pkg in pkgs:
shutil.copy(pkg, TMP_RPM_PATH)
mytime = time.time()
os.utime(cache_dir, (mytime, mytime))
pkgs = glob.glob(os.path.join(TMP_RPM_PATH, "*.rpm"))
if not use_mock:
result = run(["rpm", "-U", "--force", "--nodeps"] + pkgs, check=False)
if result['rc'] != 0:
print "Ignoring failure installing rpm batch: %s" % pkgs
print result['stderr']
for pkg in pkgs:
shutil.move(pkg, RPMS_DIR)
def get_hashes(ty):
spec_files = glob.glob(os.path.join(SPECS_DIR, "*"))
sources_files = glob.glob(os.path.join(SOURCES_DIR, "*"))
all_files = spec_files + sources_files
if ty == "md5":
cmd = ["md5sum"] + all_files
elif ty == "sha256":
cmd = ["sha256sum"] + all_files
else:
print "Invalid hash type"
raise Exception
result = run(cmd)['stdout'].strip().split('\n')
def fix(x):
words = x.split()
if len(words) == 2:
fname = words[1].split("/")[-1]
return (fname, words[0])
else:
return None
fixed = [fix(x) for x in result]
return dict(fixed)
def doexec(args, inputtext=None, check=True):
"""Execute a subprocess, then return its return code, stdout and stderr"""
myenv = os.environ.copy()
myenv['HOME'] = RPM_TOP_DIR
return run(args, check=check, env=myenv, inputtext=inputtext)
def archive(self, sources_dir=SOURCES_DIR):
final_path = os.path.join(sources_dir, self.archivename)
if os.path.exists(final_path):
return
run(["curl", "-k", "-L", "-o", final_path, self.orig_url])
def archive(self, sources_dir=SOURCES_DIR):
final_path = os.path.join(sources_dir, self.archivename)
if os.path.exists(final_path):
return
run(["curl", "-k", "-L", "-o", final_path, self.orig_url])
def archive(self, sources_dir=SOURCES_DIR):
for cmd in self.archive_commands(sources_dir):
run(cmd)
def tags(repo):
"""
Return a list of all tags defined on repo.
"""
dotgitdir = dotgitdir_of_path(repo)
return run(["git", "--git-dir=%s" % dotgitdir, "tag"])['stdout'].split()
def tags(repo):
"""
Return a list of all tags defined on repo.
"""
dotgitdir = dotgitdir_of_path(repo)
return run(["git", "--git-dir=%s" % dotgitdir, "tag"])['stdout'].split()
def doexec(args, inputtext=None, check=True):
"""Execute a subprocess, then return its return code, stdout and stderr"""
myenv = os.environ.copy()
myenv['HOME'] = RPM_TOP_DIR
return run(args, check=check, env=myenv, inputtext=inputtext)
def createrepo():
run(["createrepo", "--update", RPMS_DIR])