def create_tarball(package, version, output_dir, merged_dir):
"""Create a tarball of a merge with conflicts."""
quilt_format = False
try:
with open(merged_dir + '/debian/source/format', 'r') as fd:
quilt_format = fd.read().strip() == '3.0 (quilt)'
except IOError:
pass
if quilt_format:
filename = '%s/%s_%s.debian.tar.gz' % (output_dir, package,
version.without_epoch)
contained = 'debian'
source = merged_dir + '/debian'
else:
filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
version.without_epoch)
contained = "%s-%s" % (package, version.without_epoch)
source = merged_dir
tree.ensure("%s/tmp/" % config.get('ROOT'))
parent = tempfile.mkdtemp(dir="%s/tmp/" % config.get('ROOT'))
try:
tree.copytree(source, "%s/%s" % (parent, contained))
shell.run(("tar", "czf", filename, contained), chdir=parent)
logger.info("Created %s", tree.subdir(config.get('ROOT'), filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def conflict_file(left_dir, left_distro, right_dir, right_distro,
dest_dir, filename):
"""Copy both files as conflicts of each other."""
left_src = "%s/%s" % (left_dir, filename)
right_src = "%s/%s" % (right_dir, filename)
dest = "%s/%s" % (dest_dir, filename)
logger.debug("Conflicted: %s", filename)
tree.remove(dest)
# We need to take care here .. if one of the items involved in a
# conflict is a directory then it might have children and we don't want
# to throw an error later.
#
# We get round this by making the directory a symlink to the conflicted
# one.
#
# Fortunately this is so rare it may never happen!
if tree.exists(left_src):
tree.copyfile(left_src, "%s.%s" % (dest, left_distro.upper()))
if os.path.isdir(left_src):
os.symlink("%s.%s" % (os.path.basename(dest), left_distro.upper()),
dest)
if tree.exists(right_src):
tree.copyfile(right_src, "%s.%s" % (dest, right_distro.upper()))
if os.path.isdir(right_src):
os.symlink("%s.%s" % (os.path.basename(dest), right_distro.upper()),
dest)
def conflict_file(left_dir, left_distro, right_dir, right_distro, dest_dir,
filename):
"""Copy both files as conflicts of each other."""
left_src = "%s/%s" % (left_dir, filename)
right_src = "%s/%s" % (right_dir, filename)
dest = "%s/%s" % (dest_dir, filename)
logger.debug("Conflicted: %s", filename)
tree.remove(dest)
# We need to take care here .. if one of the items involved in a
# conflict is a directory then it might have children and we don't want
# to throw an error later.
#
# We get round this by making the directory a symlink to the conflicted
# one.
#
# Fortunately this is so rare it may never happen!
if tree.exists(left_src):
tree.copyfile(left_src, "%s.%s" % (dest, left_distro.upper()))
if os.path.isdir(left_src):
os.symlink("%s.%s" % (os.path.basename(dest), left_distro.upper()),
dest)
if tree.exists(right_src):
tree.copyfile(right_src, "%s.%s" % (dest, right_distro.upper()))
if os.path.isdir(right_src):
os.symlink("%s.%s" % (os.path.basename(dest), right_distro.upper()),
dest)
def expire_pool_sources(pkg, base):
"""Remove sources older than the given base.
If the base doesn't exist, then the newest source that is older is also
kept.
"""
pooldir = pkg.poolPath
# Find sources older than the base, record the filenames of newer ones
bases = []
base_found = False
keep = []
for pv in pkg.getPoolVersions():
if base > pv:
bases.append(pv)
else:
if base == pv.version():
base_found = True
logger.info("Leaving %s %s (is base)", distro, pv)
else:
logger.info("Leaving %s %s (is newer)", distro, pv)
keep.append(pv)
# If the base wasn't found, we want the newest source below that
if not base_found and len(bases):
version_sort(bases)
pv = bases.pop()
logger.info("Leaving %s %s (is newest before base)", distro, pv)
keep.append(pv)
# Identify filenames we don't want to delete
keep_files = []
for pv in keep:
if has_files(pv):
for md5sum, size, name in files(pv):
keep_files.append(name)
# Expire the older packages
need_update = False
for pv in bases:
logger.info("Expiring %s %s", distro, pv)
for md5sum, size, name in files(pv):
if name in keep_files:
logger.debug("Not removing %s/%s", pooldir, name)
continue
tree.remove("%s/%s" % (pooldir, name))
logger.debug("Removed %s/%s", pooldir, name)
need_update = True
def cleanup(path):
"""Remove the path and any empty directories up to ROOT."""
tree.remove(path)
(dirname, basename) = os.path.split(path)
while dirname != config.get('ROOT'):
try:
os.rmdir(dirname)
except OSError, e:
if e.errno == errno.ENOTEMPTY or e.errno == errno.ENOENT:
break
raise
(dirname, basename) = os.path.split(dirname)
def cleanup(path):
"""Remove the path and any empty directories up to ROOT."""
tree.remove(path)
(dirname, basename) = os.path.split(path)
while dirname != ROOT:
try:
os.rmdir(dirname)
except OSError, e:
if e.errno == errno.ENOTEMPTY or e.errno == errno.ENOENT:
break
raise
(dirname, basename) = os.path.split(dirname)
def create_patch(version, filename, merged_dir,
basis_source, basis_dir):
"""Create the merged patch."""
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(basis_dir, "%s/%s" % (parent, basis_source["Version"]))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
basis_source["Version"], "%s" % version),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logger.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_patch(version, filename, merged_dir, basis, basis_dir):
"""Create the merged patch."""
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(basis_dir, "%s/%s" % (parent, basis.version))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
str(basis.version), str(version)),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logger.info("Created %s",
tree.subdir(config.get('ROOT'), filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_patch(package, version, output_dir, merged_dir,
right_source, right_dir):
"""Create the merged patch."""
filename = "%s/%s_%s.patch" % (output_dir, package, version)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
right_source["Version"], "%s" % version),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_tarball(package, version, output_dir, merged_dir):
"""Create a tarball of a merge with conflicts."""
filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
version.without_epoch)
contained = "%s-%s" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
debian_rules = "%s/%s/debian/rules" % (parent, contained)
if os.path.isfile(debian_rules):
os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)
shell.run(("tar", "czf", filename, contained), chdir=parent)
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_tarball(package, version, output_dir, merged_dir):
"""Create a tarball of a merge with conflicts."""
filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
version.without_epoch)
contained = "%s-%s" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
debian_rules = "%s/%s/debian/rules" % (parent, contained)
if os.path.isfile(debian_rules):
os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)
shell.run(("tar", "czf", filename, contained), chdir=parent)
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_patch(package, version, output_dir, merged_dir, right_source,
right_dir):
"""Create the merged patch."""
filename = "%s/%s_%s.patch" % (output_dir, package, version)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))
with open(filename, "w") as diff:
shell.run(
("diff", "-pruN", right_source["Version"], "%s" % version),
chdir=parent,
stdout=diff,
okstatus=(0, 1, 2))
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def publish_patch(distro, pv, filename, list_file):
"""Publish the latest version of the patch for all to see."""
publish_filename = published_file(distro, pv)
tree.ensure(publish_filename)
if os.path.isfile(publish_filename):
os.unlink(publish_filename)
os.link(filename, publish_filename)
logger.info("Published %s",
tree.subdir(config.get('ROOT'), publish_filename))
print >> list_file, "%s %s" % (
pv.package,
tree.subdir("%s/published" % config.get('ROOT'), publish_filename))
# Remove older patches
for junk in os.listdir(os.path.dirname(publish_filename)):
junkpath = "%s/%s" % (os.path.dirname(publish_filename), junk)
if os.path.isfile(junkpath) \
and junk != os.path.basename(publish_filename):
os.unlink(junkpath)
# Publish extracted patches
output = "%s/extracted" % os.path.dirname(publish_filename)
if os.path.isdir(output):
tree.remove(output)
dpatch_dir = dpatch_directory(distro, pv)
if os.path.isdir(dpatch_dir):
for dpatch in tree.walk(dpatch_dir):
if not len(dpatch):
continue
src_filename = "%s/%s" % (dpatch_dir, dpatch)
dest_filename = "%s/%s" % (output, dpatch)
logger.info("Published %s",
tree.subdir(config.get('ROOT'), dest_filename))
tree.ensure(dest_filename)
tree.copyfile(src_filename, dest_filename)
def create_source(package, version, since, output_dir, merged_dir):
"""Create a source package without conflicts."""
contained = "%s-%s" % (package, version.upstream)
filename = "%s_%s.dsc" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
os.link("%s/%s" % (output_dir, orig_filename),
"%s/%s" % (parent, orig_filename))
cmd = ("dpkg-source", )
if version.revision is not None and since.upstream != version.upstream:
cmd += ("-sa", )
cmd += ("-b", contained)
try:
shell.run(cmd, chdir=parent)
except (ValueError, OSError):
logging.error("dpkg-source failed")
return create_tarball(package, version, output_dir, merged_dir)
if os.path.isfile("%s/%s" % (parent, filename)):
logging.info("Created %s", filename)
for name in os.listdir(parent):
src = "%s/%s" % (parent, name)
dest = "%s/%s" % (output_dir, name)
if os.path.isfile(src) and not os.path.isfile(dest):
os.link(src, dest)
return os.path.basename(filename)
else:
logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
return create_tarball(package, version, output_dir, merged_dir)
finally:
tree.remove(parent)
def create_source(package, version, since, output_dir, merged_dir):
"""Create a source package without conflicts."""
contained = "%s-%s" % (package, version.upstream)
filename = "%s_%s.dsc" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
os.link("%s/%s" % (output_dir, orig_filename),
"%s/%s" % (parent, orig_filename))
cmd = ("dpkg-source",)
if version.revision is not None and since.upstream != version.upstream:
cmd += ("-sa",)
cmd += ("-b", contained)
try:
shell.run(cmd, chdir=parent)
except (ValueError, OSError):
logging.error("dpkg-source failed")
return create_tarball(package, version, output_dir, merged_dir)
if os.path.isfile("%s/%s" % (parent, filename)):
logging.info("Created %s", filename)
for name in os.listdir(parent):
src = "%s/%s" % (parent, name)
dest = "%s/%s" % (output_dir, name)
if os.path.isfile(src) and not os.path.isfile(dest):
os.link(src, dest)
return os.path.basename(filename)
else:
logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
return create_tarball(package, version, output_dir, merged_dir)
finally:
tree.remove(parent)
def publish_patch(distro, source, filename, list_file):
"""Publish the latest version of the patch for all to see."""
publish_filename = published_file(distro, source)
ensure(publish_filename)
if os.path.isfile(publish_filename):
os.unlink(publish_filename)
os.link(filename, publish_filename)
logging.info("Published %s", tree.subdir(ROOT, publish_filename))
print("%s %s" % (source["Package"],
tree.subdir("%s/published" % ROOT, publish_filename)),
file=list_file)
# Remove older patches
for junk in os.listdir(os.path.dirname(publish_filename)):
junkpath = "%s/%s" % (os.path.dirname(publish_filename), junk)
if os.path.isfile(junkpath) \
and junk != os.path.basename(publish_filename):
os.unlink(junkpath)
# Publish extracted patches
output = "%s/extracted" % os.path.dirname(publish_filename)
if os.path.isdir(output):
tree.remove(output)
dpatch_dir = dpatch_directory(distro, source)
if os.path.isdir(dpatch_dir):
for dpatch in tree.walk(dpatch_dir):
if not len(dpatch):
continue
src_filename = "%s/%s" % (dpatch_dir, dpatch)
dest_filename = "%s/%s" % (output, dpatch)
logging.info("Published %s", tree.subdir(ROOT, dest_filename))
ensure(dest_filename)
tree.copyfile(src_filename, dest_filename)
for name in os.listdir(parent):
src = "%s/%s" % (parent, name)
dest = "%s/%s" % (output_dir, name)
if os.path.isfile(src) and not os.path.isfile(dest):
os.link(src, dest)
return (MergeResult.MERGED, None, os.path.basename(filename))
else:
message = ("dpkg-source did not produce expected filename %s" %
tree.subdir(ROOT, filename))
logger.warning("%s", message)
return (MergeResult.FAILED,
"unable to build merged source package (%s)" % message,
create_tarball(package, version, output_dir, merged_dir))
finally:
tree.remove(parent)
def create_patch(version, filename, merged_dir,
basis_source, basis_dir):
"""Create the merged patch."""
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(basis_dir, "%s/%s" % (parent, basis_source["Version"]))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
basis_source["Version"], "%s" % version),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logger.info("Created %s", tree.subdir(ROOT, filename))
def expire_pool_sources(distro, component, package, base):
"""Remove sources older than the given base.
If the base doesn't exist, then the newest source that is older is also
kept.
"""
pooldir = PoolDirectory(distro, component, package)
try:
sources = pooldir.getSourceStanzas()
except Exception as e:
if isinstance(e, IOError) and e.errno == errno.ENOENT:
# keep relatively quiet about this
logger.debug('unable to read Sources file: %s', e)
else:
logger.exception('unable to read Sources file from %s:',
pooldir.path)
return
# Find sources older than the base, record the filenames of newer ones
bases = []
base_found = False
keep = []
for source in sources:
if base > source["Version"]:
bases.append(source)
else:
if base == source["Version"]:
base_found = True
logger.info("Leaving %s %s %s (is base)", distro, package,
source["Version"])
else:
logger.info("Leaving %s %s %s (is newer)", distro, package,
source["Version"])
keep.append(source)
# If the base wasn't found, we want the newest source below that
if not base_found and len(bases):
version_sort(bases)
source = bases.pop()
logger.info("Leaving %s %s %s (is newest before base)",
distro, package, source["Version"])
keep.append(source)
# Identify filenames we don't want to delete
keep_files = []
for source in keep:
if has_files(source):
for md5sum, size, name in files(source):
keep_files.append(name)
# Expire the older packages
need_update = False
for source in bases:
logger.info("Expiring %s %s %s", distro, package, source["Version"])
for md5sum, size, name in files(source):
if name in keep_files:
logger.debug("Not removing %s/%s", pooldir.path, name)
continue
tree.remove("%s/%s/%s" % (ROOT, pooldir.path, name))
logger.debug("Removed %s/%s", pooldir.path, name)
need_update = True
if need_update:
pooldir.updateSources()
def expire_pool_sources(distro, package, base):
"""Remove sources older than the given base.
If the base doesn't exist, then the newest source that is older is also
kept.
"""
pooldir = pool_directory(distro, package)
try:
sources = get_pool_sources(distro, package)
except IOError:
return
# Find sources older than the base, record the filenames of newer ones
bases = []
base_found = False
keep = []
for source in sources:
if base > source["Version"]:
bases.append(source)
else:
if base == source["Version"]:
base_found = True
logging.info("Leaving %s %s %s (is base)", distro, package,
source["Version"])
else:
logging.info("Leaving %s %s %s (is newer)", distro, package,
source["Version"])
keep.append(source)
# If the base wasn't found, we want the newest source below that
if not base_found and len(bases):
version_sort(bases)
source = bases.pop()
logging.info("Leaving %s %s %s (is newest before base)",
distro, package, source["Version"])
keep.append(source)
# Identify filenames we don't want to delete
keep_files = []
for source in keep:
for md5sum, size, name in files(source):
keep_files.append(name)
# Expire the older packages
need_update = False
for source in bases:
logging.info("Expiring %s %s %s", distro, package, source["Version"])
for md5sum, size, name in files(source):
if name in keep_files:
logging.debug("Not removing %s/%s", pooldir, name)
continue
tree.remove("%s/%s/%s" % (ROOT, pooldir, name))
logging.debug("Removed %s/%s", pooldir, name)
need_update = True
if need_update:
update_pool_sources(distro, package)
for name in os.listdir(parent):
src = "%s/%s" % (parent, name)
dest = "%s/%s" % (output_dir, name)
if os.path.isfile(src) and not os.path.isfile(dest):
os.link(src, dest)
return (MergeResult.MERGED, None, os.path.basename(dsc_filename))
else:
message = ("dpkg-source did not produce expected filename %s" %
tree.subdir(config.get('ROOT'), dsc_filename))
logger.warning("%s", message)
return (MergeResult.FAILED,
"unable to build merged source package (%s)" % message,
create_tarball(package, version, output_dir, merged_dir))
finally:
tree.remove(parent)
def create_patch(version, filename, merged_dir, basis, basis_dir):
"""Create the merged patch."""
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(basis_dir, "%s/%s" % (parent, basis.version))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
str(basis.version), str(version)),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logger.info("Created %s",