Esempio n. 1
0
def create_tarball(package, version, output_dir, merged_dir):
    """Create a tarball of a merge with conflicts."""
    quilt_format = False
    try:
        with open(merged_dir + '/debian/source/format', 'r') as fd:
            quilt_format = fd.read().strip() == '3.0 (quilt)'
    except IOError:
        pass

    if quilt_format:
        filename = '%s/%s_%s.debian.tar.gz' % (output_dir, package,
                                               version.without_epoch)
        contained = 'debian'
        source = merged_dir + '/debian'
    else:
        filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
                                            version.without_epoch)
        contained = "%s-%s" % (package, version.without_epoch)
        source = merged_dir

    tree.ensure("%s/tmp/" % config.get('ROOT'))
    parent = tempfile.mkdtemp(dir="%s/tmp/" % config.get('ROOT'))
    try:
        tree.copytree(source, "%s/%s" % (parent, contained))
        shell.run(("tar", "czf", filename, contained), chdir=parent)
        logger.info("Created %s", tree.subdir(config.get('ROOT'), filename))
        return os.path.basename(filename)
    finally:
        tree.remove(parent)
def conflict_file(left_dir, left_distro, right_dir, right_distro,
                  dest_dir, filename):
    """Copy both files as conflicts of each other."""
    left_src = "%s/%s" % (left_dir, filename)
    right_src = "%s/%s" % (right_dir, filename)
    dest = "%s/%s" % (dest_dir, filename)

    logger.debug("Conflicted: %s", filename)
    tree.remove(dest)

    # We need to take care here .. if one of the items involved in a
    # conflict is a directory then it might have children and we don't want
    # to throw an error later.
    #
    # We get round this by making the directory a symlink to the conflicted
    # one.
    #
    # Fortunately this is so rare it may never happen!

    if tree.exists(left_src):
        tree.copyfile(left_src, "%s.%s" % (dest, left_distro.upper()))
    if os.path.isdir(left_src):
        os.symlink("%s.%s" % (os.path.basename(dest), left_distro.upper()),
                   dest)

    if tree.exists(right_src):
        tree.copyfile(right_src, "%s.%s" % (dest, right_distro.upper()))
    if os.path.isdir(right_src):
        os.symlink("%s.%s" % (os.path.basename(dest), right_distro.upper()),
                   dest)
Esempio n. 3
0
def conflict_file(left_dir, left_distro, right_dir, right_distro, dest_dir,
                  filename):
    """Copy both files as conflicts of each other."""
    left_src = "%s/%s" % (left_dir, filename)
    right_src = "%s/%s" % (right_dir, filename)
    dest = "%s/%s" % (dest_dir, filename)

    logger.debug("Conflicted: %s", filename)
    tree.remove(dest)

    # We need to take care here .. if one of the items involved in a
    # conflict is a directory then it might have children and we don't want
    # to throw an error later.
    #
    # We get round this by making the directory a symlink to the conflicted
    # one.
    #
    # Fortunately this is so rare it may never happen!

    if tree.exists(left_src):
        tree.copyfile(left_src, "%s.%s" % (dest, left_distro.upper()))
    if os.path.isdir(left_src):
        os.symlink("%s.%s" % (os.path.basename(dest), left_distro.upper()),
                   dest)

    if tree.exists(right_src):
        tree.copyfile(right_src, "%s.%s" % (dest, right_distro.upper()))
    if os.path.isdir(right_src):
        os.symlink("%s.%s" % (os.path.basename(dest), right_distro.upper()),
                   dest)
Esempio n. 4
0
def expire_pool_sources(pkg, base):
    """Remove sources older than the given base.

    If the base doesn't exist, then the newest source that is older is also
    kept.
    """
    pooldir = pkg.poolPath

    # Find sources older than the base, record the filenames of newer ones
    bases = []
    base_found = False
    keep = []
    for pv in pkg.getPoolVersions():
        if base > pv:
            bases.append(pv)
        else:
            if base == pv.version():
                base_found = True
                logger.info("Leaving %s %s (is base)", distro, pv)
            else:
                logger.info("Leaving %s %s (is newer)", distro, pv)

            keep.append(pv)

    # If the base wasn't found, we want the newest source below that
    if not base_found and len(bases):
        version_sort(bases)
        pv = bases.pop()
        logger.info("Leaving %s %s (is newest before base)", distro, pv)

        keep.append(pv)

    # Identify filenames we don't want to delete
    keep_files = []
    for pv in keep:
        if has_files(pv):
            for md5sum, size, name in files(pv):
                keep_files.append(name)

    # Expire the older packages
    need_update = False
    for pv in bases:
        logger.info("Expiring %s %s", distro, pv)

        for md5sum, size, name in files(pv):
            if name in keep_files:
                logger.debug("Not removing %s/%s", pooldir, name)
                continue

            tree.remove("%s/%s" % (pooldir, name))
            logger.debug("Removed %s/%s", pooldir, name)
            need_update = True
Esempio n. 5
0
def cleanup(path):
    """Remove the path and any empty directories up to ROOT."""
    tree.remove(path)

    (dirname, basename) = os.path.split(path)
    while dirname != config.get('ROOT'):
        try:
            os.rmdir(dirname)
        except OSError, e:
            if e.errno == errno.ENOTEMPTY or e.errno == errno.ENOENT:
                break
            raise

        (dirname, basename) = os.path.split(dirname)
Esempio n. 6
0
def cleanup(path):
    """Remove the path and any empty directories up to ROOT."""
    tree.remove(path)

    (dirname, basename) = os.path.split(path)
    while dirname != ROOT:
        try:
            os.rmdir(dirname)
        except OSError, e:
            if e.errno == errno.ENOTEMPTY or e.errno == errno.ENOENT:
                break
            raise

        (dirname, basename) = os.path.split(dirname)
def create_patch(version, filename, merged_dir,
                 basis_source, basis_dir):
    """Create the merged patch."""

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(basis_dir, "%s/%s" % (parent, basis_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       basis_source["Version"], "%s" % version),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logger.info("Created %s", tree.subdir(ROOT, filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 8
0
def create_patch(version, filename, merged_dir, basis, basis_dir):
    """Create the merged patch."""

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(basis_dir, "%s/%s" % (parent, basis.version))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       str(basis.version), str(version)),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logger.info("Created %s",
                        tree.subdir(config.get('ROOT'), filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 9
0
def create_patch(package, version, output_dir, merged_dir,
                 right_source, right_dir):
    """Create the merged patch."""
    filename = "%s/%s_%s.patch" % (output_dir, package, version)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       right_source["Version"], "%s" % version),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logging.info("Created %s", tree.subdir(ROOT, filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 10
0
def create_tarball(package, version, output_dir, merged_dir):
    """Create a tarball of a merge with conflicts."""
    filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
                                        version.without_epoch)
    contained = "%s-%s" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        debian_rules = "%s/%s/debian/rules" % (parent, contained)
        if os.path.isfile(debian_rules):
            os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)

        shell.run(("tar", "czf", filename, contained), chdir=parent)

        logging.info("Created %s", tree.subdir(ROOT, filename))
        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 11
0
def create_tarball(package, version, output_dir, merged_dir):
    """Create a tarball of a merge with conflicts."""
    filename = "%s/%s_%s.src.tar.gz" % (output_dir, package,
                                        version.without_epoch)
    contained = "%s-%s" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        debian_rules = "%s/%s/debian/rules" % (parent, contained)
        if os.path.isfile(debian_rules):
            os.chmod(debian_rules, os.stat(debian_rules).st_mode | 0111)

        shell.run(("tar", "czf", filename, contained), chdir=parent)

        logging.info("Created %s", tree.subdir(ROOT, filename))
        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 12
0
def create_patch(package, version, output_dir, merged_dir, right_source,
                 right_dir):
    """Create the merged patch."""
    filename = "%s/%s_%s.patch" % (output_dir, package, version)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(right_dir, "%s/%s" % (parent, right_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(
                ("diff", "-pruN", right_source["Version"], "%s" % version),
                chdir=parent,
                stdout=diff,
                okstatus=(0, 1, 2))
            logging.info("Created %s", tree.subdir(ROOT, filename))

        return os.path.basename(filename)
    finally:
        tree.remove(parent)
Esempio n. 13
0
def publish_patch(distro, pv, filename, list_file):
    """Publish the latest version of the patch for all to see."""
    publish_filename = published_file(distro, pv)

    tree.ensure(publish_filename)
    if os.path.isfile(publish_filename):
        os.unlink(publish_filename)
    os.link(filename, publish_filename)

    logger.info("Published %s",
                tree.subdir(config.get('ROOT'), publish_filename))
    print >> list_file, "%s %s" % (
        pv.package,
        tree.subdir("%s/published" % config.get('ROOT'), publish_filename))

    # Remove older patches
    for junk in os.listdir(os.path.dirname(publish_filename)):
        junkpath = "%s/%s" % (os.path.dirname(publish_filename), junk)
        if os.path.isfile(junkpath) \
                and junk != os.path.basename(publish_filename):
            os.unlink(junkpath)

    # Publish extracted patches
    output = "%s/extracted" % os.path.dirname(publish_filename)
    if os.path.isdir(output):
        tree.remove(output)

    dpatch_dir = dpatch_directory(distro, pv)
    if os.path.isdir(dpatch_dir):
        for dpatch in tree.walk(dpatch_dir):
            if not len(dpatch):
                continue

            src_filename = "%s/%s" % (dpatch_dir, dpatch)
            dest_filename = "%s/%s" % (output, dpatch)

            logger.info("Published %s",
                        tree.subdir(config.get('ROOT'), dest_filename))
            tree.ensure(dest_filename)
            tree.copyfile(src_filename, dest_filename)
Esempio n. 14
0
def create_source(package, version, since, output_dir, merged_dir):
    """Create a source package without conflicts."""
    contained = "%s-%s" % (package, version.upstream)
    filename = "%s_%s.dsc" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
        if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
            os.link("%s/%s" % (output_dir, orig_filename),
                    "%s/%s" % (parent, orig_filename))

        cmd = ("dpkg-source", )
        if version.revision is not None and since.upstream != version.upstream:
            cmd += ("-sa", )
        cmd += ("-b", contained)

        try:
            shell.run(cmd, chdir=parent)
        except (ValueError, OSError):
            logging.error("dpkg-source failed")
            return create_tarball(package, version, output_dir, merged_dir)

        if os.path.isfile("%s/%s" % (parent, filename)):
            logging.info("Created %s", filename)
            for name in os.listdir(parent):
                src = "%s/%s" % (parent, name)
                dest = "%s/%s" % (output_dir, name)
                if os.path.isfile(src) and not os.path.isfile(dest):
                    os.link(src, dest)

            return os.path.basename(filename)
        else:
            logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
            return create_tarball(package, version, output_dir, merged_dir)
    finally:
        tree.remove(parent)
Esempio n. 15
0
def create_source(package, version, since, output_dir, merged_dir):
    """Create a source package without conflicts."""
    contained = "%s-%s" % (package, version.upstream)
    filename = "%s_%s.dsc" % (package, version.without_epoch)

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, contained))

        orig_filename = "%s_%s.orig.tar.gz" % (package, version.upstream)
        if os.path.isfile("%s/%s" % (output_dir, orig_filename)):
            os.link("%s/%s" % (output_dir, orig_filename),
                    "%s/%s" % (parent, orig_filename))

        cmd = ("dpkg-source",)
        if version.revision is not None and since.upstream != version.upstream:
            cmd += ("-sa",)
        cmd += ("-b", contained)

        try:
            shell.run(cmd, chdir=parent)
        except (ValueError, OSError):
            logging.error("dpkg-source failed")
            return create_tarball(package, version, output_dir, merged_dir)

        if os.path.isfile("%s/%s" % (parent, filename)):
            logging.info("Created %s", filename)
            for name in os.listdir(parent):
                src = "%s/%s" % (parent, name)
                dest = "%s/%s" % (output_dir, name)
                if os.path.isfile(src) and not os.path.isfile(dest):
                    os.link(src, dest)

            return os.path.basename(filename)
        else:
            logging.warning("Dropped dsc %s", tree.subdir(ROOT, filename))
            return create_tarball(package, version, output_dir, merged_dir)
    finally:
        tree.remove(parent)
Esempio n. 16
0
def publish_patch(distro, source, filename, list_file):
    """Publish the latest version of the patch for all to see."""
    publish_filename = published_file(distro, source)

    ensure(publish_filename)
    if os.path.isfile(publish_filename):
        os.unlink(publish_filename)
    os.link(filename, publish_filename)

    logging.info("Published %s", tree.subdir(ROOT, publish_filename))
    print("%s %s" % (source["Package"],
                     tree.subdir("%s/published" % ROOT, publish_filename)),
          file=list_file)

    # Remove older patches
    for junk in os.listdir(os.path.dirname(publish_filename)):
        junkpath = "%s/%s" % (os.path.dirname(publish_filename), junk)
        if os.path.isfile(junkpath) \
                and junk != os.path.basename(publish_filename):
            os.unlink(junkpath)

    # Publish extracted patches
    output = "%s/extracted" % os.path.dirname(publish_filename)
    if os.path.isdir(output):
        tree.remove(output)

    dpatch_dir = dpatch_directory(distro, source)
    if os.path.isdir(dpatch_dir):
        for dpatch in tree.walk(dpatch_dir):
            if not len(dpatch):
                continue

            src_filename = "%s/%s" % (dpatch_dir, dpatch)
            dest_filename = "%s/%s" % (output, dpatch)

            logging.info("Published %s", tree.subdir(ROOT, dest_filename))
            ensure(dest_filename)
            tree.copyfile(src_filename, dest_filename)
Esempio n. 17
0
            for name in os.listdir(parent):
                src = "%s/%s" % (parent, name)
                dest = "%s/%s" % (output_dir, name)
                if os.path.isfile(src) and not os.path.isfile(dest):
                    os.link(src, dest)

            return (MergeResult.MERGED, None, os.path.basename(filename))
        else:
            message = ("dpkg-source did not produce expected filename %s" %
                tree.subdir(ROOT, filename))
            logger.warning("%s", message)
            return (MergeResult.FAILED,
                    "unable to build merged source package (%s)" % message,
                create_tarball(package, version, output_dir, merged_dir))
    finally:
        tree.remove(parent)

def create_patch(version, filename, merged_dir,
                 basis_source, basis_dir):
    """Create the merged patch."""

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(basis_dir, "%s/%s" % (parent, basis_source["Version"]))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       basis_source["Version"], "%s" % version),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logger.info("Created %s", tree.subdir(ROOT, filename))
Esempio n. 18
0
def expire_pool_sources(distro, component, package, base):
    """Remove sources older than the given base.

    If the base doesn't exist, then the newest source that is older is also
    kept.
    """
    pooldir = PoolDirectory(distro, component, package)
    try:
        sources = pooldir.getSourceStanzas()
    except Exception as e:
        if isinstance(e, IOError) and e.errno == errno.ENOENT:
            # keep relatively quiet about this
            logger.debug('unable to read Sources file: %s', e)
        else:
            logger.exception('unable to read Sources file from %s:',
                    pooldir.path)
        return

    # Find sources older than the base, record the filenames of newer ones
    bases = []
    base_found = False
    keep = []
    for source in sources:
        if base > source["Version"]:
            bases.append(source)
        else:
            if base == source["Version"]:
                base_found = True
                logger.info("Leaving %s %s %s (is base)", distro, package,
                             source["Version"])
            else:
                logger.info("Leaving %s %s %s (is newer)", distro, package,
                             source["Version"])

            keep.append(source)

    # If the base wasn't found, we want the newest source below that
    if not base_found and len(bases):
        version_sort(bases)
        source = bases.pop()
        logger.info("Leaving %s %s %s (is newest before base)",
                     distro, package, source["Version"])

        keep.append(source)

    # Identify filenames we don't want to delete
    keep_files = []
    for source in keep:
        if has_files(source):
            for md5sum, size, name in files(source):
                keep_files.append(name)

    # Expire the older packages
    need_update = False
    for source in bases:
        logger.info("Expiring %s %s %s", distro, package, source["Version"])

        for md5sum, size, name in files(source):
            if name in keep_files:
                logger.debug("Not removing %s/%s", pooldir.path, name)
                continue

            tree.remove("%s/%s/%s" % (ROOT, pooldir.path, name))
            logger.debug("Removed %s/%s", pooldir.path, name)
            need_update = True

    if need_update:
        pooldir.updateSources()
Esempio n. 19
0
def expire_pool_sources(distro, package, base):
    """Remove sources older than the given base.

    If the base doesn't exist, then the newest source that is older is also
    kept.
    """
    pooldir = pool_directory(distro, package)
    try:
        sources = get_pool_sources(distro, package)
    except IOError:
        return

    # Find sources older than the base, record the filenames of newer ones
    bases = []
    base_found = False
    keep = []
    for source in sources:
        if base > source["Version"]:
            bases.append(source)
        else:
            if base == source["Version"]:
                base_found = True
                logging.info("Leaving %s %s %s (is base)", distro, package,
                             source["Version"])
            else:
                logging.info("Leaving %s %s %s (is newer)", distro, package,
                             source["Version"])

            keep.append(source)

    # If the base wasn't found, we want the newest source below that
    if not base_found and len(bases):
        version_sort(bases)
        source = bases.pop()
        logging.info("Leaving %s %s %s (is newest before base)",
                     distro, package, source["Version"])

        keep.append(source)

    # Identify filenames we don't want to delete
    keep_files = []
    for source in keep:
        for md5sum, size, name in files(source):
            keep_files.append(name)

    # Expire the older packages
    need_update = False
    for source in bases:
        logging.info("Expiring %s %s %s", distro, package, source["Version"])

        for md5sum, size, name in files(source):
            if name in keep_files:
                logging.debug("Not removing %s/%s", pooldir, name)
                continue

            tree.remove("%s/%s/%s" % (ROOT, pooldir, name))
            logging.debug("Removed %s/%s", pooldir, name)
            need_update = True

    if need_update:
        update_pool_sources(distro, package)
Esempio n. 20
0
            for name in os.listdir(parent):
                src = "%s/%s" % (parent, name)
                dest = "%s/%s" % (output_dir, name)
                if os.path.isfile(src) and not os.path.isfile(dest):
                    os.link(src, dest)

            return (MergeResult.MERGED, None, os.path.basename(dsc_filename))
        else:
            message = ("dpkg-source did not produce expected filename %s" %
                       tree.subdir(config.get('ROOT'), dsc_filename))
            logger.warning("%s", message)
            return (MergeResult.FAILED,
                    "unable to build merged source package (%s)" % message,
                    create_tarball(package, version, output_dir, merged_dir))
    finally:
        tree.remove(parent)


def create_patch(version, filename, merged_dir, basis, basis_dir):
    """Create the merged patch."""

    parent = tempfile.mkdtemp()
    try:
        tree.copytree(merged_dir, "%s/%s" % (parent, version))
        tree.copytree(basis_dir, "%s/%s" % (parent, basis.version))

        with open(filename, "w") as diff:
            shell.run(("diff", "-pruN",
                       str(basis.version), str(version)),
                      chdir=parent, stdout=diff, okstatus=(0, 1, 2))
            logger.info("Created %s",