Esempio n. 1
0
def create_tarball(package, version, output_dir, merged_dir):
    """Create a tarball of a merge with conflicts."""
    quilt_format = False
    try:
        with open(merged_dir + '/debian/source/format', 'r') as fd:
            quilt_format = fd.read().strip() == '3.0 (quilt)'
    except IOError:
        pass

    if quilt_format:
        filename = '%s/%s_%s.debian.tar.gz' % (output_dir, package,
                                               version.without_epoch)
        contained = 'debian'
        source = merged_dir + '/debian'
    else:
        filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
                                            version.without_epoch)
        contained = "%s-%s" % (package, version.without_epoch)
        source = merged_dir

    tree.ensure("%s/tmp/" % config.get('ROOT'))
    parent = tempfile.mkdtemp(dir="%s/tmp/" % config.get('ROOT'))
    try:
        tree.copytree(source, "%s/%s" % (parent, contained))
        shell.run(("tar", "czf", filename, contained), chdir=parent)
        logger.info("Created %s", tree.subdir(config.get('ROOT'), filename))
        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 2
0
def unpack_source(pv):
    """Unpack the given source and return location."""
    destdir = unpack_directory(pv)
    if os.path.isdir(destdir):
        return destdir

    srcdir = pv.package.poolPath
    dsc_file = pv.dscPath

    logger.info("Unpacking %s from %s/%s", pv, srcdir, dsc_file)

    tree.ensure(destdir)
    try:
        # output directory for "dpkg-source -x" must not exist
        if (os.path.isdir(destdir)):
            os.rmdir(destdir)
        shell.run(("dpkg-source", "--skip-patches", "-x", dsc_file, destdir), chdir=srcdir, stdout=sys.stdout, stderr=sys.stderr)
        # Make sure we can at least read everything under .pc, which isn't
        # automatically true with dpkg-dev 1.15.4.
        pc_dir = os.path.join(destdir, ".pc")
        for filename in tree.walk(pc_dir):
            pc_filename = os.path.join(pc_dir, filename)
            pc_stat = os.lstat(pc_filename)
            if pc_stat is not None and stat.S_IMODE(pc_stat.st_mode) == 0:
                os.chmod(pc_filename, 0400)
    except:
        cleanup(destdir)
        raise

    return destdir
Esempio n. 3
0
def unpack_source(distro, source):
    """Unpack the given source and return location."""
    destdir = unpack_directory(source)
    if os.path.isdir(destdir):
        return destdir

    srcdir = "%s/%s" % (ROOT, source["Directory"])
    for md5sum, size, name in files(source):
        if name.endswith(".dsc"):
            dsc_file = name
            break
    else:
        raise ValueError, "Missing dsc file"

    ensure(destdir)
    try:
        env = dict(os.environ)
        env['DEB_VENDOR'] = distro
        shell.run(("dpkg-source", "-x", dsc_file, destdir), chdir=srcdir,
                  env=env)
        # Make sure we can at least read everything under .pc, which isn't
        # automatically true with dpkg-dev 1.15.4.
        pc_dir = os.path.join(destdir, ".pc")
        for filename in tree.walk(pc_dir):
            pc_filename = os.path.join(pc_dir, filename)
            pc_stat = os.lstat(pc_filename)
            if pc_stat is not None and stat.S_IMODE(pc_stat.st_mode) == 0:
                os.chmod(pc_filename, 0400)
    except:
        cleanup(destdir)
        raise

    return destdir
Esempio n. 4
0
def update_pool_sources(distro, package):
    """Update the Sources files in the pool."""
    pooldir = pool_directory(distro, package)
    filename = pool_sources_file(distro, package)
    if pool_sources_already_updated(pooldir, filename):
        return

    logging.info("Updating %s", tree.subdir(ROOT, filename))
    with open("%s.new" % filename, "w") as sources:
        shell.run(("apt-ftparchive", "sources", pooldir), chdir=ROOT,
                  stdout=sources)
    os.rename("%s.new" % filename, filename)
Esempio n. 5
0
def save_patch_file(filename, last, this):
    """Save a diff or patch file for the difference between two versions."""
    lastdir = unpack_directory(last)
    thisdir = unpack_directory(this)

    diffdir = os.path.commonprefix((lastdir, thisdir))
    diffdir = diffdir[:diffdir.rindex("/")]

    lastdir = tree.subdir(diffdir, lastdir)
    thisdir = tree.subdir(diffdir, thisdir)

    tree.ensure(filename)
    with open(filename, "w") as diff:
        shell.run(("diff", "-pruN", lastdir, thisdir),
                  chdir=diffdir, stdout=diff, okstatus=(0, 1, 2))
Esempio n. 6
0
def save_patch_file(filename, last, this):
    """Save a diff or patch file for the difference between two versions."""
    lastdir = unpack_directory(last)
    thisdir = unpack_directory(this)

    diffdir = os.path.commonprefix((lastdir, thisdir))
    diffdir = diffdir[:diffdir.rindex("/")]

    lastdir = tree.subdir(diffdir, lastdir)
    thisdir = tree.subdir(diffdir, thisdir)

    tree.ensure(filename)
    with open(filename, "w") as diff:
        shell.run(("diff", "-pruN", lastdir, thisdir),
                  chdir=diffdir, stdout=diff, okstatus=(0, 1, 2))
Esempio n. 7
0
  def updateSources(self):
    """Update the Sources file at sourcesFilename() to contain every
    package/version in this pool directory.
    """
    pooldir = self.path
    filename = self.sourcesFilename

    if not os.path.isdir(pooldir):
      return

    tree.ensure(pooldir)
    logger.debug("Updating %s", filename)
    with open(filename, "w") as sources:
      shell.run(("apt-ftparchive", "sources", pooldir),
          chdir=config.get('ROOT'),
          stdout=sources)
Esempio n. 8
0
def save_changes_file(filename, pv, previous=None):
    """Save a changes file for the given source."""
    srcdir = unpack_directory(pv)

    tree.ensure(filename)
    with open(filename, "w") as changes:
        cmd = ("dpkg-genchanges", "-S", "-u%s" % pv.package.poolPath)
        orig_cmd = cmd
        if previous is not None:
            cmd += ("-v%s" % previous.version,)

        try:
            shell.run(cmd, chdir=srcdir, stdout=changes)
        except (ValueError, OSError):
            shell.run(orig_cmd, chdir=srcdir, stdout=changes)

    return filename
Esempio n. 9
0
def merge_pot(left_dir, right_dir, merged_dir, filename):
    """Update a .po file using msgcat."""
    merged_pot = "%s/%s" % (merged_dir, filename)

    left_pot = "%s/%s" % (left_dir, filename)
    right_pot = "%s/%s" % (right_dir, filename)

    logger.debug("Merging POT file %s", filename)
    try:
        tree.ensure(merged_pot)
        shell.run(("msgcat", "--force-po", "--use-first", "-o", merged_pot,
                   right_pot, left_pot))
    except (ValueError, OSError):
        logger.error("POT file merge failed: %s", filename)
        return False

    return True
Esempio n. 10
0
def merge_pot(left_dir, right_dir, merged_dir, filename):
    """Update a .po file using msgcat."""
    merged_pot = "%s/%s" % (merged_dir, filename)

    left_pot = "%s/%s" % (left_dir, filename)
    right_pot = "%s/%s" % (right_dir, filename)

    logger.debug("Merging POT file %s", filename)
    try:
        tree.ensure(merged_pot)
        shell.run(("msgcat", "--force-po", "--use-first", "-o", merged_pot,
                   right_pot, left_pot))
    except (ValueError, OSError):
        logger.error("POT file merge failed: %s", filename)
        return True

    return False
Esempio n. 11
0
def create_patch(version, filename, merged_dir,
                 basis_source, basis_dir):
    """Create the merged patch."""

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(basis_dir, "%s/%s" % (parent, basis_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       basis_source["Version"], "%s" % version),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logger.info("Created %s", tree.subdir(ROOT, filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 12
0
def create_patch(version, filename, merged_dir, basis, basis_dir):
    """Create the merged patch."""

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(basis_dir, "%s/%s" % (parent, basis.version))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       str(basis.version), str(version)),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logger.info("Created %s",
                        tree.subdir(config.get('ROOT'), filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 13
0
def save_changes_file(filename, source, previous=None):
    """Save a changes file for the given source."""
    srcdir = unpack_directory(source)

    filesdir = "%s/%s" % (ROOT, source["Directory"])

    tree.ensure(filename)
    with open(filename, "w") as changes:
        cmd = ("dpkg-genchanges", "-S", "-u%s" % filesdir)
        orig_cmd = cmd
        if previous is not None:
            cmd += ("-v%s" % previous["Version"],)

        try:
            shell.run(cmd, chdir=srcdir, stdout=changes)
        except (ValueError, OSError):
            shell.run(orig_cmd, chdir=srcdir, stdout=changes)

    return filename
Esempio n. 14
0
def create_patch(package, version, output_dir, merged_dir,
                 right_source, right_dir):
    """Create the merged patch."""
    filename = "%s/%s_%s.patch" % (output_dir, package, version)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       right_source["Version"], "%s" % version),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logging.info("Created %s", tree.subdir(ROOT, filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 15
0
def merge_po(left_dir, right_dir, merged_dir, filename):
    """Update a .po file using msgcat or msgmerge."""
    merged_po = "%s/%s" % (merged_dir, filename)
    closest_pot = find_closest_pot(merged_po)
    if closest_pot is None:
        return merge_pot(left_dir, right_dir, merged_dir, filename)

    left_po = "%s/%s" % (left_dir, filename)
    right_po = "%s/%s" % (right_dir, filename)

    logger.debug("Merging PO file %s", filename)
    try:
        tree.ensure(merged_po)
        shell.run(("msgmerge", "--force-po", "-o", merged_po,
                   "-C", left_po, right_po, closest_pot))
    except (ValueError, OSError):
        logger.error("PO file merge failed: %s", filename)
        return True

    return False
Esempio n. 16
0
def create_tarball(package, version, output_dir, merged_dir):
    """Create a tarball of a merge with conflicts."""
    filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
                                        version.without_epoch)
    contained = "%s-%s" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        debian_rules = "%s/%s/debian/rules" % (parent, contained)
        if os.path.isfile(debian_rules):
            os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)

        shell.run(("tar", "czf", filename, contained), chdir=parent)

        logging.info("Created %s", tree.subdir(ROOT, filename))
        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 17
0
def merge_po(left_dir, right_dir, merged_dir, filename):
    """Update a .po file using msgcat or msgmerge."""
    merged_po = "%s/%s" % (merged_dir, filename)
    closest_pot = find_closest_pot(merged_po)
    if closest_pot is None:
        return merge_pot(left_dir, right_dir, merged_dir, filename)

    left_po = "%s/%s" % (left_dir, filename)
    right_po = "%s/%s" % (right_dir, filename)

    logger.debug("Merging PO file %s", filename)
    try:
        tree.ensure(merged_po)
        shell.run(("msgmerge", "--force-po", "-o", merged_po, "-C", left_po,
                   right_po, closest_pot))
    except (ValueError, OSError):
        logger.error("PO file merge failed: %s", filename)
        return False

    return True
Esempio n. 18
0
def create_tarball(package, version, output_dir, merged_dir):
    """Create a tarball of a merge with conflicts."""
    filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
                                        version.without_epoch)
    contained = "%s-%s" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        debian_rules = "%s/%s/debian/rules" % (parent, contained)
        if os.path.isfile(debian_rules):
            os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)

        shell.run(("tar", "czf", filename, contained), chdir=parent)

        logging.info("Created %s", tree.subdir(ROOT, filename))
        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 19
0
def create_patch(package, version, output_dir, merged_dir, right_source,
                 right_dir):
    """Create the merged patch."""
    filename = "%s/%s_%s.patch" % (output_dir, package, version)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(
                ("diff", "-pruN", right_source["Version"], "%s" % version),
                chdir=parent,
                stdout=diff,
                okstatus=(0, 1, 2))
            logging.info("Created %s", tree.subdir(ROOT, filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 20
0
def create_source(package, version, since, output_dir, merged_dir):
    """Create a source package without conflicts."""
    contained = "%s-%s" % (package, version.upstream)
    filename = "%s_%s.dsc" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
        if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
            os.link("%s/%s" % (output_dir, orig_filename),
                    "%s/%s" % (parent, orig_filename))

        cmd = ("dpkg-source", )
        if version.revision is not None and since.upstream != version.upstream:
            cmd += ("-sa", )
        cmd += ("-b", contained)

        try:
            shell.run(cmd, chdir=parent)
        except (ValueError, OSError):
            logging.error("dpkg-source failed")
            return create_tarball(package, version, output_dir, merged_dir)

        if os.path.isfile("%s/%s" % (parent, filename)):
            logging.info("Created %s", filename)
            for name in os.listdir(parent):
                src = "%s/%s" % (parent, name)
                dest = "%s/%s" % (output_dir, name)
                if os.path.isfile(src) and not os.path.isfile(dest):
                    os.link(src, dest)

            return os.path.basename(filename)
        else:
            logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
            return create_tarball(package, version, output_dir, merged_dir)
    finally:
        tree.remove(parent)
Esempio n. 21
0
def create_source(package, version, since, output_dir, merged_dir):
    """Create a source package without conflicts."""
    contained = "%s-%s" % (package, version.upstream)
    filename = "%s_%s.dsc" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
        if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
            os.link("%s/%s" % (output_dir, orig_filename),
                    "%s/%s" % (parent, orig_filename))

        cmd = ("dpkg-source",)
        if version.revision is not None and since.upstream != version.upstream:
            cmd += ("-sa",)
        cmd += ("-b", contained)

        try:
            shell.run(cmd, chdir=parent)
        except (ValueError, OSError):
            logging.error("dpkg-source failed")
            return create_tarball(package, version, output_dir, merged_dir)

        if os.path.isfile("%s/%s" % (parent, filename)):
            logging.info("Created %s", filename)
            for name in os.listdir(parent):
                src = "%s/%s" % (parent, name)
                dest = "%s/%s" % (output_dir, name)
                if os.path.isfile(src) and not os.path.isfile(dest):
                    os.link(src, dest)

            return os.path.basename(filename)
        else:
            logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
            return create_tarball(package, version, output_dir, merged_dir)
    finally:
        tree.remove(parent)
Esempio n. 22
0
def unpack_source(pv):
    """Unpack the given source and return location."""
    source = pv.getSources()
    destdir = unpack_directory(source)
    if os.path.isdir(destdir):
        return destdir

    srcdir = "%s/%s" % (ROOT, pv.poolDirectory().path)
    for md5sum, size, name in files(source):
        if name.endswith(".dsc"):
            dsc_file = name
            break
    else:
        raise ValueError, "Missing dsc file"

    logger.info("Unpacking %s from %s/%s", pv, srcdir, dsc_file)

    tree.ensure(destdir)
    try:
        # output directory for "dpkg-source -x" must not exist
        if (os.path.isdir(destdir)):
            os.rmdir(destdir)
        shell.run(("dpkg-source", "--skip-patches", "-x", dsc_file, destdir), chdir=srcdir, stdout=sys.stdout, stderr=sys.stderr)
        # Make sure we can at least read everything under .pc, which isn't
        # automatically true with dpkg-dev 1.15.4.
        pc_dir = os.path.join(destdir, ".pc")
        for filename in tree.walk(pc_dir):
            pc_filename = os.path.join(pc_dir, filename)
            pc_stat = os.lstat(pc_filename)
            if pc_stat is not None and stat.S_IMODE(pc_stat.st_mode) == 0:
                os.chmod(pc_filename, 0400)
    except:
        cleanup(destdir)
        raise

    return destdir
Esempio n. 23
0
def merge_file(left_dir, left_name, left_distro, base_dir, right_dir,
               right_name, right_distro, merged_dir, filename):
    """Merge a file using diff3."""
    dest = "%s/%s" % (merged_dir, filename)
    ensure(dest)

    with open(dest, "w") as output:
        status = shell.run(("diff3", "-E", "-m", "-L", left_name, "%s/%s" %
                            (left_dir, filename), "-L", "BASE", "%s/%s" %
                            (base_dir, filename), "-L", right_name, "%s/%s" %
                            (right_dir, filename)),
                           stdout=output,
                           okstatus=(0, 1, 2))

    if status != 0:
        if not tree.exists(dest) or os.stat(dest).st_size == 0:
            # Probably binary
            if same_file(os.stat("%s/%s" % (left_dir, filename)), left_dir,
                         os.stat("%s/%s" % (right_dir, filename)), right_dir,
                         filename):
                logging.debug("binary files are the same: %s", filename)
                tree.copyfile("%s/%s" % (left_dir, filename),
                              "%s/%s" % (merged_dir, filename))
            elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
                           os.stat("%s/%s" % (left_dir, filename)), left_dir,
                           filename):
                logging.debug("preserving binary change in %s: %s",
                              right_distro, filename)
                tree.copyfile("%s/%s" % (right_dir, filename),
                              "%s/%s" % (merged_dir, filename))
            elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
                           os.stat("%s/%s" % (right_dir, filename)), right_dir,
                           filename):
                logging.debug("preserving binary change in %s: %s",
                              left_distro, filename)
                tree.copyfile("%s/%s" % (left_dir, filename),
                              "%s/%s" % (merged_dir, filename))
            else:
                logging.debug("binary file conflict: %s", filename)
                conflict_file(left_dir, left_distro, right_dir, right_distro,
                              merged_dir, filename)
                return True
        else:
            logging.debug("Conflict in %s", filename)
            return True
    else:
        return False
Esempio n. 24
0
def merge_file(left_dir, left_name, left_distro, base_dir,
               right_dir, right_name, right_distro, merged_dir, filename):
    """Merge a file using diff3."""
    dest = "%s/%s" % (merged_dir, filename)
    tree.ensure(dest)

    with open(dest, "w") as output:
        status = shell.run(("diff3", "-E", "-m",
                            "-L", left_name, "%s/%s" % (left_dir, filename),
                            "-L", "BASE", "%s/%s" % (base_dir, filename),
                            "-L", right_name, "%s/%s" % (right_dir, filename)),
                           stdout=output, okstatus=(0,1,2))

    if status != 0:
        if not tree.exists(dest) or os.stat(dest).st_size == 0:
            # Probably binary
            if same_file(os.stat("%s/%s" % (left_dir, filename)), left_dir,
                         os.stat("%s/%s" % (right_dir, filename)), right_dir,
                         filename):
                logger.debug("binary files are the same: %s", filename)
                tree.copyfile("%s/%s" % (left_dir, filename),
                              "%s/%s" % (merged_dir, filename))
            elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
                           os.stat("%s/%s" % (left_dir, filename)), left_dir,
                           filename):
                logger.debug("preserving binary change in %s: %s",
                              right_distro, filename)
                tree.copyfile("%s/%s" % (right_dir, filename),
                              "%s/%s" % (merged_dir, filename))
            elif same_file(os.stat("%s/%s" % (base_dir, filename)), base_dir,
                           os.stat("%s/%s" % (right_dir, filename)), right_dir,
                           filename):
                logger.debug("preserving binary change in %s: %s",
                              left_distro, filename)
                tree.copyfile("%s/%s" % (left_dir, filename),
                              "%s/%s" % (merged_dir, filename))
            else:
                logger.debug("binary file conflict: %s", filename)
                conflict_file(left_dir, left_distro, right_dir, right_distro,
                              merged_dir, filename)
                return True
        else:
            logger.debug("Conflict in %s", filename)
            return True
    else:
        return False