def create_tarball(package, version, output_dir, merged_dir):
"""Create a tarball of a merge with conflicts."""
quilt_format = False
try:
with open(merged_dir + '/debian/source/format', 'r') as fd:
quilt_format = fd.read().strip() == '3.0 (quilt)'
except IOError:
pass
if quilt_format:
filename = '%s/%s_%s.debian.tar.gz' % (output_dir, package,
version.without_epoch)
contained = 'debian'
source = merged_dir + '/debian'
else:
filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
version.without_epoch)
contained = "%s-%s" % (package, version.without_epoch)
source = merged_dir
tree.ensure("%s/tmp/" % config.get('ROOT'))
parent = tempfile.mkdtemp(dir="%s/tmp/" % config.get('ROOT'))
try:
tree.copytree(source, "%s/%s" % (parent, contained))
shell.run(("tar", "czf", filename, contained), chdir=parent)
logger.info("Created %s", tree.subdir(config.get('ROOT'), filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def unpack_source(pv):
"""Unpack the given source and return location."""
destdir = unpack_directory(pv)
if os.path.isdir(destdir):
return destdir
srcdir = pv.package.poolPath
dsc_file = pv.dscPath
logger.info("Unpacking %s from %s/%s", pv, srcdir, dsc_file)
tree.ensure(destdir)
try:
# output directory for "dpkg-source -x" must not exist
if (os.path.isdir(destdir)):
os.rmdir(destdir)
shell.run(("dpkg-source", "--skip-patches", "-x", dsc_file, destdir), chdir=srcdir, stdout=sys.stdout, stderr=sys.stderr)
# Make sure we can at least read everything under .pc, which isn't
# automatically true with dpkg-dev 1.15.4.
pc_dir = os.path.join(destdir, ".pc")
for filename in tree.walk(pc_dir):
pc_filename = os.path.join(pc_dir, filename)
pc_stat = os.lstat(pc_filename)
if pc_stat is not None and stat.S_IMODE(pc_stat.st_mode) == 0:
os.chmod(pc_filename, 0400)
except:
cleanup(destdir)
raise
return destdir
def unpack_source(distro, source):
"""Unpack the given source and return location."""
destdir = unpack_directory(source)
if os.path.isdir(destdir):
return destdir
srcdir = "%s/%s" % (ROOT, source["Directory"])
for md5sum, size, name in files(source):
if name.endswith(".dsc"):
dsc_file = name
break
else:
raise ValueError, "Missing dsc file"
ensure(destdir)
try:
env = dict(os.environ)
env['DEB_VENDOR'] = distro
shell.run(("dpkg-source", "-x", dsc_file, destdir), chdir=srcdir,
env=env)
# Make sure we can at least read everything under .pc, which isn't
# automatically true with dpkg-dev 1.15.4.
pc_dir = os.path.join(destdir, ".pc")
for filename in tree.walk(pc_dir):
pc_filename = os.path.join(pc_dir, filename)
pc_stat = os.lstat(pc_filename)
if pc_stat is not None and stat.S_IMODE(pc_stat.st_mode) == 0:
os.chmod(pc_filename, 0400)
except:
cleanup(destdir)
raise
return destdir
def update_pool_sources(distro, package):
"""Update the Sources files in the pool."""
pooldir = pool_directory(distro, package)
filename = pool_sources_file(distro, package)
if pool_sources_already_updated(pooldir, filename):
return
logging.info("Updating %s", tree.subdir(ROOT, filename))
with open("%s.new" % filename, "w") as sources:
shell.run(("apt-ftparchive", "sources", pooldir), chdir=ROOT,
stdout=sources)
os.rename("%s.new" % filename, filename)
def save_patch_file(filename, last, this):
"""Save a diff or patch file for the difference between two versions."""
lastdir = unpack_directory(last)
thisdir = unpack_directory(this)
diffdir = os.path.commonprefix((lastdir, thisdir))
diffdir = diffdir[:diffdir.rindex("/")]
lastdir = tree.subdir(diffdir, lastdir)
thisdir = tree.subdir(diffdir, thisdir)
tree.ensure(filename)
with open(filename, "w") as diff:
shell.run(("diff", "-pruN", lastdir, thisdir),
chdir=diffdir, stdout=diff, okstatus=(0, 1, 2))
def save_patch_file(filename, last, this):
"""Save a diff or patch file for the difference between two versions."""
lastdir = unpack_directory(last)
thisdir = unpack_directory(this)
diffdir = os.path.commonprefix((lastdir, thisdir))
diffdir = diffdir[:diffdir.rindex("/")]
lastdir = tree.subdir(diffdir, lastdir)
thisdir = tree.subdir(diffdir, thisdir)
tree.ensure(filename)
with open(filename, "w") as diff:
shell.run(("diff", "-pruN", lastdir, thisdir),
chdir=diffdir, stdout=diff, okstatus=(0, 1, 2))
def updateSources(self):
"""Update the Sources file at sourcesFilename() to contain every
package/version in this pool directory.
"""
pooldir = self.path
filename = self.sourcesFilename
if not os.path.isdir(pooldir):
return
tree.ensure(pooldir)
logger.debug("Updating %s", filename)
with open(filename, "w") as sources:
shell.run(("apt-ftparchive", "sources", pooldir),
chdir=config.get('ROOT'),
stdout=sources)
def save_changes_file(filename, pv, previous=None):
"""Save a changes file for the given source."""
srcdir = unpack_directory(pv)
tree.ensure(filename)
with open(filename, "w") as changes:
cmd = ("dpkg-genchanges", "-S", "-u%s" % pv.package.poolPath)
orig_cmd = cmd
if previous is not None:
cmd += ("-v%s" % previous.version,)
try:
shell.run(cmd, chdir=srcdir, stdout=changes)
except (ValueError, OSError):
shell.run(orig_cmd, chdir=srcdir, stdout=changes)
return filename
def merge_pot(left_dir, right_dir, merged_dir, filename):
"""Update a .po file using msgcat."""
merged_pot = "%s/%s" % (merged_dir, filename)
left_pot = "%s/%s" % (left_dir, filename)
right_pot = "%s/%s" % (right_dir, filename)
logger.debug("Merging POT file %s", filename)
try:
tree.ensure(merged_pot)
shell.run(("msgcat", "--force-po", "--use-first", "-o", merged_pot,
right_pot, left_pot))
except (ValueError, OSError):
logger.error("POT file merge failed: %s", filename)
return False
return True
def merge_pot(left_dir, right_dir, merged_dir, filename):
"""Update a .po file using msgcat."""
merged_pot = "%s/%s" % (merged_dir, filename)
left_pot = "%s/%s" % (left_dir, filename)
right_pot = "%s/%s" % (right_dir, filename)
logger.debug("Merging POT file %s", filename)
try:
tree.ensure(merged_pot)
shell.run(("msgcat", "--force-po", "--use-first", "-o", merged_pot,
right_pot, left_pot))
except (ValueError, OSError):
logger.error("POT file merge failed: %s", filename)
return True
return False
def create_patch(version, filename, merged_dir,
basis_source, basis_dir):
"""Create the merged patch."""
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(basis_dir, "%s/%s" % (parent, basis_source["Version"]))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
basis_source["Version"], "%s" % version),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logger.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_patch(version, filename, merged_dir, basis, basis_dir):
"""Create the merged patch."""
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(basis_dir, "%s/%s" % (parent, basis.version))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
str(basis.version), str(version)),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logger.info("Created %s",
tree.subdir(config.get('ROOT'), filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def save_changes_file(filename, source, previous=None):
"""Save a changes file for the given source."""
srcdir = unpack_directory(source)
filesdir = "%s/%s" % (ROOT, source["Directory"])
tree.ensure(filename)
with open(filename, "w") as changes:
cmd = ("dpkg-genchanges", "-S", "-u%s" % filesdir)
orig_cmd = cmd
if previous is not None:
cmd += ("-v%s" % previous["Version"],)
try:
shell.run(cmd, chdir=srcdir, stdout=changes)
except (ValueError, OSError):
shell.run(orig_cmd, chdir=srcdir, stdout=changes)
return filename
def create_patch(package, version, output_dir, merged_dir,
right_source, right_dir):
"""Create the merged patch."""
filename = "%s/%s_%s.patch" % (output_dir, package, version)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))
with open(filename, "w") as diff:
shell.run(("diff", "-pruN",
right_source["Version"], "%s" % version),
chdir=parent, stdout=diff, okstatus=(0, 1, 2))
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def merge_po(left_dir, right_dir, merged_dir, filename):
"""Update a .po file using msgcat or msgmerge."""
merged_po = "%s/%s" % (merged_dir, filename)
closest_pot = find_closest_pot(merged_po)
if closest_pot is None:
return merge_pot(left_dir, right_dir, merged_dir, filename)
left_po = "%s/%s" % (left_dir, filename)
right_po = "%s/%s" % (right_dir, filename)
logger.debug("Merging PO file %s", filename)
try:
tree.ensure(merged_po)
shell.run(("msgmerge", "--force-po", "-o", merged_po,
"-C", left_po, right_po, closest_pot))
except (ValueError, OSError):
logger.error("PO file merge failed: %s", filename)
return True
return False
def create_tarball(package, version, output_dir, merged_dir):
"""Create a tarball of a merge with conflicts."""
filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
version.without_epoch)
contained = "%s-%s" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
debian_rules = "%s/%s/debian/rules" % (parent, contained)
if os.path.isfile(debian_rules):
os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)
shell.run(("tar", "czf", filename, contained), chdir=parent)
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def merge_po(left_dir, right_dir, merged_dir, filename):
"""Update a .po file using msgcat or msgmerge."""
merged_po = "%s/%s" % (merged_dir, filename)
closest_pot = find_closest_pot(merged_po)
if closest_pot is None:
return merge_pot(left_dir, right_dir, merged_dir, filename)
left_po = "%s/%s" % (left_dir, filename)
right_po = "%s/%s" % (right_dir, filename)
logger.debug("Merging PO file %s", filename)
try:
tree.ensure(merged_po)
shell.run(("msgmerge", "--force-po", "-o", merged_po, "-C", left_po,
right_po, closest_pot))
except (ValueError, OSError):
logger.error("PO file merge failed: %s", filename)
return False
return True
def create_tarball(package, version, output_dir, merged_dir):
"""Create a tarball of a merge with conflicts."""
filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
version.without_epoch)
contained = "%s-%s" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
debian_rules = "%s/%s/debian/rules" % (parent, contained)
if os.path.isfile(debian_rules):
os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)
shell.run(("tar", "czf", filename, contained), chdir=parent)
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_patch(package, version, output_dir, merged_dir, right_source,
right_dir):
"""Create the merged patch."""
filename = "%s/%s_%s.patch" % (output_dir, package, version)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, version))
tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))
with open(filename, "w") as diff:
shell.run(
("diff", "-pruN", right_source["Version"], "%s" % version),
chdir=parent,
stdout=diff,
okstatus=(0, 1, 2))
logging.info("Created %s", tree.subdir(ROOT, filename))
return os.path.basename(filename)
finally:
tree.remove(parent)
def create_source(package, version, since, output_dir, merged_dir):
"""Create a source package without conflicts."""
contained = "%s-%s" % (package, version.upstream)
filename = "%s_%s.dsc" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
os.link("%s/%s" % (output_dir, orig_filename),
"%s/%s" % (parent, orig_filename))
cmd = ("dpkg-source", )
if version.revision is not None and since.upstream != version.upstream:
cmd += ("-sa", )
cmd += ("-b", contained)
try:
shell.run(cmd, chdir=parent)
except (ValueError, OSError):
logging.error("dpkg-source failed")
return create_tarball(package, version, output_dir, merged_dir)
if os.path.isfile("%s/%s" % (parent, filename)):
logging.info("Created %s", filename)
for name in os.listdir(parent):
src = "%s/%s" % (parent, name)
dest = "%s/%s" % (output_dir, name)
if os.path.isfile(src) and not os.path.isfile(dest):
os.link(src, dest)
return os.path.basename(filename)
else:
logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
return create_tarball(package, version, output_dir, merged_dir)
finally:
tree.remove(parent)
def create_source(package, version, since, output_dir, merged_dir):
"""Create a source package without conflicts."""
contained = "%s-%s" % (package, version.upstream)
filename = "%s_%s.dsc" % (package, version.without_epoch)
parent = tempfile.mkdtemp()
try:
tree.copytree(merged_dir, "%s/%s" % (parent, contained))
orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
os.link("%s/%s" % (output_dir, orig_filename),
"%s/%s" % (parent, orig_filename))
cmd = ("dpkg-source",)
if version.revision is not None and since.upstream != version.upstream:
cmd += ("-sa",)
cmd += ("-b", contained)
try:
shell.run(cmd, chdir=parent)
except (ValueError, OSError):
logging.error("dpkg-source failed")
return create_tarball(package, version, output_dir, merged_dir)
if os.path.isfile("%s/%s" % (parent, filename)):
logging.info("Created %s", filename)
for name in os.listdir(parent):
src = "%s/%s" % (parent, name)
dest = "%s/%s" % (output_dir, name)
if os.path.isfile(src) and not os.path.isfile(dest):
os.link(src, dest)
return os.path.basename(filename)
else:
logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
return create_tarball(package, version, output_dir, merged_dir)
finally:
tree.remove(parent)
def unpack_source(pv):
"""Unpack the given source and return location."""
source = pv.getSources()
destdir = unpack_directory(source)
if os.path.isdir(destdir):
return destdir
srcdir = "%s/%s" % (ROOT, pv.poolDirectory().path)
for md5sum, size, name in files(source):
if name.endswith(".dsc"):
dsc_file = name
break
else:
raise ValueError, "Missing dsc file"
logger.info("Unpacking %s from %s/%s", pv, srcdir, dsc_file)
tree.ensure(destdir)
try:
# output directory for "dpkg-source -x" must not exist
if (os.path.isdir(destdir)):
os.rmdir(destdir)
shell.run(("dpkg-source", "--skip-patches", "-x", dsc_file, destdir), chdir=srcdir, stdout=sys.stdout, stderr=sys.stderr)
# Make sure we can at least read everything under .pc, which isn't
# automatically true with dpkg-dev 1.15.4.
pc_dir = os.path.join(destdir, ".pc")
for filename in tree.walk(pc_dir):
pc_filename = os.path.join(pc_dir, filename)
pc_stat = os.lstat(pc_filename)
if pc_stat is not None and stat.S_IMODE(pc_stat.st_mode) == 0:
os.chmod(pc_filename, 0400)
except:
cleanup(destdir)
raise
return destdir
def merge_file(left_dir, left_name, left_distro, base_dir, right_dir,
right_name, right_distro, merged_dir, filename):
"""Merge a file using diff3."""
dest = "%s/%s" % (merged_dir, filename)
ensure(dest)
with open(dest, "w") as output:
status = shell.run(("diff3", "-E", "-m", "-L", left_name, "%s/%s" %
(left_dir, filename), "-L", "BASE", "%s/%s" %
(base_dir, filename), "-L", right_name, "%s/%s" %
(right_dir, filename)),
stdout=output,
okstatus=(0, 1, 2))
if status != 0:
if not tree.exists(dest) or os.stat(dest).st_size == 0:
# Probably binary
if same_file(os.stat("%s/%s" % (left_dir, filename)), left_dir,
os.stat("%s/%s" % (right_dir, filename)), right_dir,
filename):
logging.debug("binary files are the same: %s", filename)
tree.copyfile("%s/%s" % (left_dir, filename),
"%s/%s" % (merged_dir, filename))
elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
os.stat("%s/%s" % (left_dir, filename)), left_dir,
filename):
logging.debug("preserving binary change in %s: %s",
right_distro, filename)
tree.copyfile("%s/%s" % (right_dir, filename),
"%s/%s" % (merged_dir, filename))
elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
os.stat("%s/%s" % (right_dir, filename)), right_dir,
filename):
logging.debug("preserving binary change in %s: %s",
left_distro, filename)
tree.copyfile("%s/%s" % (left_dir, filename),
"%s/%s" % (merged_dir, filename))
else:
logging.debug("binary file conflict: %s", filename)
conflict_file(left_dir, left_distro, right_dir, right_distro,
merged_dir, filename)
return True
else:
logging.debug("Conflict in %s", filename)
return True
else:
return False
def merge_file(left_dir, left_name, left_distro, base_dir,
right_dir, right_name, right_distro, merged_dir, filename):
"""Merge a file using diff3."""
dest = "%s/%s" % (merged_dir, filename)
tree.ensure(dest)
with open(dest, "w") as output:
status = shell.run(("diff3", "-E", "-m",
"-L", left_name, "%s/%s" % (left_dir, filename),
"-L", "BASE", "%s/%s" % (base_dir, filename),
"-L", right_name, "%s/%s" % (right_dir, filename)),
stdout=output, okstatus=(0,1,2))
if status != 0:
if not tree.exists(dest) or os.stat(dest).st_size == 0:
# Probably binary
if same_file(os.stat("%s/%s" % (left_dir, filename)), left_dir,
os.stat("%s/%s" % (right_dir, filename)), right_dir,
filename):
logger.debug("binary files are the same: %s", filename)
tree.copyfile("%s/%s" % (left_dir, filename),
"%s/%s" % (merged_dir, filename))
elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
os.stat("%s/%s" % (left_dir, filename)), left_dir,
filename):
logger.debug("preserving binary change in %s: %s",
right_distro, filename)
tree.copyfile("%s/%s" % (right_dir, filename),
"%s/%s" % (merged_dir, filename))
elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
os.stat("%s/%s" % (right_dir, filename)), right_dir,
filename):
logger.debug("preserving binary change in %s: %s",
left_distro, filename)
tree.copyfile("%s/%s" % (left_dir, filename),
"%s/%s" % (merged_dir, filename))
else:
logger.debug("binary file conflict: %s", filename)
conflict_file(left_dir, left_distro, right_dir, right_distro,
merged_dir, filename)
return True
else:
logger.debug("Conflict in %s", filename)
return True
else:
return False