Example #1
0
def parse_prod(logdate):
    global stats
    global users
    maildate = ''.join([x[-2:] for x in logdate.split('-')])
    mailarchive = join(utils.get_conf()['Dir::Base'], 'mail/archive',
                       'mail-%s.xz' % maildate)
    if not isfile(mailarchive):
        return
    (fd, tmpfile) = utils.temp_filename(utils.get_conf()['Dir::TempPath'])
    system('xzcat %s > %s' % (mailarchive, tmpfile))
    for message in mbox(tmpfile):
        if (message['subject'] and
                message['subject'].startswith('Comments regarding')):
            try:
                member = users[' '.join(message['From'].split()[:-1])]
            except KeyError:
                continue
            ts = mktime_tz(parsedate_tz(message['date']))
            timestamp = datetime.fromtimestamp(ts).strftime("%Y%m%d%H%M%S")
            date = parse_timestamp(timestamp)
            if date not in stats:
                stats[date] = {'stats': {'NEW': 0, 'ACCEPT': 0,
                                 'REJECT': 0, 'PROD': 0}, 'members': {}}
            if member not in stats[date]['members']:
                stats[date]['members'][member] = {'ACCEPT': 0, 'REJECT': 0,
                                                     'PROD': 0}
            if member not in stats['history']['members']:
                stats['history']['members'][member] = {'ACCEPT': 0,
                                                       'REJECT': 0, 'PROD': 0}
            stats[date]['stats']['PROD'] += 1
            stats[date]['members'][member]['PROD'] += 1
            stats['history']['stats']['PROD'] += 1
            stats['history']['members'][member]['PROD'] += 1
    unlink(tmpfile)
Example #2
0
File: stats.py Project: Debian/dak
def parse_prod(logdate):
    global stats
    global users
    maildate = ''.join([x[-2:] for x in logdate.split('-')])
    mailarchive = join(utils.get_conf()['Dir::Base'], 'mail/archive',
                       'mail-%s.xz' % maildate)
    if not isfile(mailarchive):
        return
    (fd, tmpfile) = utils.temp_filename(utils.get_conf()['Dir::TempPath'])
    system('xzcat %s > %s' % (mailarchive, tmpfile))
    for message in mbox(tmpfile):
        if (message['subject']
                and message['subject'].startswith('Comments regarding')):
            try:
                member = users[' '.join(message['From'].split()[:-1])]
            except KeyError:
                continue
            ts = mktime_tz(parsedate_tz(message['date']))
            timestamp = datetime.fromtimestamp(ts).strftime("%Y%m%d%H%M%S")
            date = parse_timestamp(timestamp)
            if date not in stats:
                stats[date] = {'stats': {'NEW': 0, 'ACCEPT': 0,
                                 'REJECT': 0, 'PROD': 0}, 'members': {}}
            if member not in stats[date]['members']:
                stats[date]['members'][member] = {'ACCEPT': 0, 'REJECT': 0,
                                                     'PROD': 0}
            if member not in stats['history']['members']:
                stats['history']['members'][member] = {'ACCEPT': 0,
                                                       'REJECT': 0, 'PROD': 0}
            stats[date]['stats']['PROD'] += 1
            stats[date]['members'][member]['PROD'] += 1
            stats['history']['stats']['PROD'] += 1
            stats['history']['members'][member]['PROD'] += 1
    unlink(tmpfile)
Example #3
0
def validate_packages(suite, component, architecture):
    """
    Ensure files mentioned in Packages exist
    """
    cnf = Config()

    filename = "%s/dists/%s/%s/binary-%s/Packages.gz" \
               % (cnf["Dir::Root"], suite, component, architecture)
    print "Processing %s..." % (filename)
    # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
    (fd, temp_filename) = utils.temp_filename()
    (result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
                                                (filename, temp_filename))
    if (result != 0):
        sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
        sys.exit(result)
    packages = utils.open_file(temp_filename)
    Packages = apt_pkg.TagFile(packages)
    while Packages.step():
        filename = "%s/%s" % (cnf["Dir::Root"],
                              Packages.section.find('Filename'))
        if not os.path.exists(filename):
            print "W: %s missing." % (filename)
    packages.close()
    os.unlink(temp_filename)
Example #4
0
    def check(self, upload):
        changes = upload.changes

        # Only check sourceful uploads.
        if changes.source is None:
            return True
        # Only check uploads to unstable or experimental.
        if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions:
            return True

        cnf = Config()
        if 'Dinstall::LintianTags' not in cnf:
            return True
        tagfile = cnf['Dinstall::LintianTags']

        with open(tagfile, 'r') as sourcefile:
            sourcecontent = sourcefile.read()
        try:
            lintiantags = yaml.safe_load(sourcecontent)['lintian']
        except yaml.YAMLError as msg:
            raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))

        fd, temp_filename = utils.temp_filename(mode=0o644)
        temptagfile = os.fdopen(fd, 'w')
        for tags in lintiantags.itervalues():
            for tag in tags:
                print >>temptagfile, tag
        temptagfile.close()

        changespath = os.path.join(upload.directory, changes.filename)
        try:
            cmd = []
            result = 0

            user = cnf.get('Dinstall::UnprivUser') or None
            if user is not None:
                cmd.extend(['sudo', '-H', '-u', user])

            cmd.extend(['/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath])
            output = daklib.daksubprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            result = e.returncode
            output = e.output
        finally:
            os.unlink(temp_filename)

        if result == 2:
            utils.warn("lintian failed for %s [return code: %s]." % \
                (changespath, result))
            utils.warn(utils.prefix_multi_line_string(output, \
                " [possible output:] "))

        parsed_tags = lintian.parse_lintian_output(output)
        rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags))
        if len(rejects) != 0:
            raise Reject('\n'.join(rejects))

        return True
Example #5
0
def new_stats(logdir, yaml):
    global Cnf
    global stats
    try:
        with open(yaml, 'r') as fd:
            stats = safe_load(fd)
    except IOError:
        pass
    if not stats:
        stats = {
            'history': {
                'stats': {
                    'NEW': 0,
                    'ACCEPT': 0,
                    'REJECT': 0,
                    'PROD': 0
                },
                'members': {}
            },
            'timestamp': '19700101000000'
        }
    latest_timestamp = stats['timestamp']
    for fn in sorted(listdir(logdir)):
        if fn == 'current':
            continue
        log = splitext(fn)[0]
        if log < parse_timestamp(stats['timestamp']):
            continue
        logfile = join(logdir, fn)
        if isfile(logfile):
            if fn.endswith('.bz2'):
                # This hack is required becaue python2 does not support
                # multi-stream files (http://bugs.python.org/issue1625)
                (fd, tmpfile) = utils.temp_filename(Cnf['Dir::TempPath'])
                system('bzcat %s > %s' % (logfile, tmpfile))
                with open(tmpfile, 'r') as fd:
                    data = fd.read()
                unlink(tmpfile)
            else:
                with open(logfile, 'r') as fd:
                    data = fd.read()
            ts = parse_new_uploads(data)
            if ts > latest_timestamp:
                latest_timestamp = ts
            ts = parse_actions(data, log)
            if ts > latest_timestamp:
                latest_timestamp = ts
            stderr.write('.')
            stderr.flush()
    stderr.write('\n')
    stderr.flush()
    stats['timestamp'] = latest_timestamp
    with open(yaml, 'w') as fd:
        safe_dump(stats, fd)
Example #6
0
def edit_new(overrides, upload, session):
    # Write the current data to a temporary file
    (fd, temp_filename) = utils.temp_filename()
    temp_file = os.fdopen(fd, 'w')
    print_new(upload, overrides, indexed=0, session=session, file=temp_file)
    temp_file.close()
    # Spawn an editor on that file
    editor = os.environ.get("EDITOR", "vi")
    result = os.system("%s %s" % (editor, temp_filename))
    if result != 0:
        utils.fubar("%s invocation failed for %s." % (editor, temp_filename),
                    result)
    # Read the edited data back in
    temp_file = utils.open_file(temp_filename)
    lines = temp_file.readlines()
    temp_file.close()
    os.unlink(temp_filename)

    overrides_map = dict([((o['type'], o['package']), o) for o in overrides])
    new_overrides = []
    # Parse the new data
    for line in lines:
        line = line.strip()
        if line == "" or line[0] == '#':
            continue
        s = line.split()
        # Pad the list if necessary
        s[len(s):3] = [None] * (3 - len(s))
        (pkg, priority, section) = s[:3]
        if pkg.find(':') != -1:
            type, pkg = pkg.split(':', 1)
        else:
            type = 'deb'
        o = overrides_map.get((type, pkg), None)
        if o is None:
            utils.warn("Ignoring unknown package '%s'" % (pkg))
        else:
            if section.find('/') != -1:
                component = section.split('/', 1)[0]
            else:
                component = 'main'
            new_overrides.append(
                dict(
                    package=pkg,
                    type=type,
                    section=section,
                    component=component,
                    priority=priority,
                    included=o['included'],
                ))
    return new_overrides
Example #7
0
def edit_new (overrides, upload, session):
    # Write the current data to a temporary file
    (fd, temp_filename) = utils.temp_filename()
    temp_file = os.fdopen(fd, 'w')
    print_new (upload, overrides, indexed=0, session=session, file=temp_file)
    temp_file.close()
    # Spawn an editor on that file
    editor = os.environ.get("EDITOR","vi")
    result = os.system("%s %s" % (editor, temp_filename))
    if result != 0:
        utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
    # Read the edited data back in
    temp_file = utils.open_file(temp_filename)
    lines = temp_file.readlines()
    temp_file.close()
    os.unlink(temp_filename)

    overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ])
    new_overrides = []
    # Parse the new data
    for line in lines:
        line = line.strip()
        if line == "" or line[0] == '#':
            continue
        s = line.split()
        # Pad the list if necessary
        s[len(s):3] = [None] * (3-len(s))
        (pkg, priority, section) = s[:3]
        if pkg.find(':') != -1:
            type, pkg = pkg.split(':', 1)
        else:
            type = 'deb'
        o = overrides_map.get((type, pkg), None)
        if o is None:
            utils.warn("Ignoring unknown package '%s'" % (pkg))
        else:
            if section.find('/') != -1:
                component = section.split('/', 1)[0]
            else:
                component = 'main'
            new_overrides.append(dict(
                    package=pkg,
                    type=type,
                    section=section,
                    component=component,
                    priority=priority,
                    included=o['included'],
                    ))
    return new_overrides
Example #8
0
File: stats.py Project: Debian/dak
def new_stats(logdir, yaml):
    global Cnf
    global stats
    try:
        with open(yaml, 'r') as fd:
            stats = safe_load(fd)
    except IOError:
        pass
    if not stats:
        stats = {'history': {'stats': {'NEW': 0, 'ACCEPT': 0,
                 'REJECT': 0, 'PROD': 0}, 'members': {}},
                 'timestamp': '19700101000000'}
    latest_timestamp = stats['timestamp']
    for fn in sorted(listdir(logdir)):
        if fn == 'current':
            continue
        log = splitext(fn)[0]
        if log < parse_timestamp(stats['timestamp']):
            continue
        logfile = join(logdir, fn)
        if isfile(logfile):
            if fn.endswith('.bz2'):
                # This hack is required becaue python2 does not support
                # multi-stream files (http://bugs.python.org/issue1625)
                (fd, tmpfile) = utils.temp_filename(Cnf['Dir::TempPath'])
                system('bzcat %s > %s' % (logfile, tmpfile))
                with open(tmpfile, 'r') as fd:
                    data = fd.read()
                unlink(tmpfile)
            else:
                with open(logfile, 'r') as fd:
                    data = fd.read()
            ts = parse_new_uploads(data)
            if ts > latest_timestamp:
                latest_timestamp = ts
            ts = parse_actions(data, log)
            if ts > latest_timestamp:
                latest_timestamp = ts
            stderr.write('.')
            stderr.flush()
    stderr.write('\n')
    stderr.flush()
    stats['timestamp'] = latest_timestamp
    with open(yaml, 'w') as fd:
        safe_dump(stats, fd)
Example #9
0
def validate_sources(suite, component):
    """
    Ensure files mentioned in Sources exist
    """
    cnf = Config()

    filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite,
                                                     component)
    print "Processing %s..." % (filename)
    # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
    (fd, temp_filename) = utils.temp_filename()
    (result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
                                                (filename, temp_filename))
    if (result != 0):
        sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
        sys.exit(result)
    sources = utils.open_file(temp_filename)
    Sources = apt_pkg.TagFile(sources)
    while Sources.step():
        source = Sources.section.find('Package')
        directory = Sources.section.find('Directory')
        files = Sources.section.find('Files')
        for i in files.split('\n'):
            (md5, size, name) = i.split()
            filename = "%s/%s/%s" % (cnf["Dir::Root"], directory, name)
            if not os.path.exists(filename):
                if directory.find("potato") == -1:
                    print "W: %s missing." % (filename)
                else:
                    pool_location = utils.poolify(source, component)
                    pool_filename = "%s/%s/%s" % (cnf["Dir::Pool"],
                                                  pool_location, name)
                    if not os.path.exists(pool_filename):
                        print "E: %s missing (%s)." % (filename, pool_filename)
                    else:
                        # Create symlink
                        pool_filename = os.path.normpath(pool_filename)
                        filename = os.path.normpath(filename)
                        src = utils.clean_symlink(pool_filename, filename,
                                                  cnf["Dir::Root"])
                        print "Symlinking: %s -> %s" % (filename, src)
                        #os.symlink(src, filename)
    sources.close()
    os.unlink(temp_filename)
Example #10
0
def validate_packages(suite, component, architecture):
    """
    Ensure files mentioned in Packages exist
    """
    filename = "%s/dists/%s/%s/binary-%s/Packages.gz" \
               % (Cnf["Dir::Root"], suite, component, architecture)
    print "Processing %s..." % (filename)
    # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
    (fd, temp_filename) = utils.temp_filename()
    (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
    if (result != 0):
        sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
        sys.exit(result)
    packages = utils.open_file(temp_filename)
    Packages = apt_pkg.ParseTagFile(packages)
    while Packages.Step():
        filename = "%s/%s" % (Cnf["Dir::Root"], Packages.Section.Find('Filename'))
        if not os.path.exists(filename):
            print "W: %s missing." % (filename)
    packages.close()
    os.unlink(temp_filename)
Example #11
0
def validate_sources(suite, component):
    """
    Ensure files mentioned in Sources exist
    """
    cnf = Config()

    filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite, component)
    print "Processing %s..." % (filename)
    # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
    (fd, temp_filename) = utils.temp_filename()
    (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
    if (result != 0):
        sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
        sys.exit(result)
    sources = utils.open_file(temp_filename)
    Sources = apt_pkg.TagFile(sources)
    while Sources.step():
        source = Sources.section.find('Package')
        directory = Sources.section.find('Directory')
        files = Sources.section.find('Files')
        for i in files.split('\n'):
            (md5, size, name) = i.split()
            filename = "%s/%s/%s" % (cnf["Dir::Root"], directory, name)
            if not os.path.exists(filename):
                if directory.find("potato") == -1:
                    print "W: %s missing." % (filename)
                else:
                    pool_location = utils.poolify (source, component)
                    pool_filename = "%s/%s/%s" % (cnf["Dir::Pool"], pool_location, name)
                    if not os.path.exists(pool_filename):
                        print "E: %s missing (%s)." % (filename, pool_filename)
                    else:
                        # Create symlink
                        pool_filename = os.path.normpath(pool_filename)
                        filename = os.path.normpath(filename)
                        src = utils.clean_symlink(pool_filename, filename, cnf["Dir::Root"])
                        print "Symlinking: %s -> %s" % (filename, src)
                        #os.symlink(src, filename)
    sources.close()
    os.unlink(temp_filename)
Example #12
0
def edit_new (new, upload):
    # Write the current data to a temporary file
    (fd, temp_filename) = utils.temp_filename()
    temp_file = os.fdopen(fd, 'w')
    print_new (new, upload, indexed=0, file=temp_file)
    temp_file.close()
    # Spawn an editor on that file
    editor = os.environ.get("EDITOR","vi")
    result = os.system("%s %s" % (editor, temp_filename))
    if result != 0:
        utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
    # Read the edited data back in
    temp_file = utils.open_file(temp_filename)
    lines = temp_file.readlines()
    temp_file.close()
    os.unlink(temp_filename)
    # Parse the new data
    for line in lines:
        line = line.strip()
        if line == "":
            continue
        s = line.split()
        # Pad the list if necessary
        s[len(s):3] = [None] * (3-len(s))
        (pkg, priority, section) = s[:3]
        if not new.has_key(pkg):
            utils.warn("Ignoring unknown package '%s'" % (pkg))
        else:
            # Strip off any invalid markers, print_new will readd them.
            if section.endswith("[!]"):
                section = section[:-3]
            if priority.endswith("[!]"):
                priority = priority[:-3]
            for f in new[pkg]["files"]:
                upload.pkg.files[f]["section"] = section
                upload.pkg.files[f]["priority"] = priority
            new[pkg]["section"] = section
            new[pkg]["priority"] = priority
Example #13
0
def main ():
    global suite, suite_id, source_binaries, source_versions

    cnf = Config()

    Arguments = [('h',"help","Cruft-Report::Options::Help"),
                 ('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
                 ('R',"rdep-check", "Cruft-Report::Options::Rdep-Check"),
                 ('s',"suite","Cruft-Report::Options::Suite","HasArg"),
                 ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
    for i in [ "help", "Rdep-Check" ]:
        if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
            cnf["Cruft-Report::Options::%s" % (i)] = ""

    cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")

    if not cnf.has_key("Cruft-Report::Options::Mode"):
        cnf["Cruft-Report::Options::Mode"] = "daily"

    if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
        cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"

    apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)

    Options = cnf.subtree("Cruft-Report::Options")
    if Options["Help"]:
        usage()

    if Options["Rdep-Check"]:
        rdeps = True
    else:
        rdeps = False

    # Set up checks based on mode
    if Options["Mode"] == "daily":
        checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu" ]
    elif Options["Mode"] == "full":
        checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
    elif Options["Mode"] == "bdo":
        checks = [ "nbs",  "obsolete source" ]
    else:
        utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"]))
        usage(1)

    session = DBConn().session()

    bin_pkgs = {}
    src_pkgs = {}
    bin2source = {}
    bins_in_suite = {}
    nbs = {}
    source_versions = {}

    anais_output = ""

    nfu_packages = {}

    suite = get_suite(Options["Suite"].lower(), session)
    if not suite:
        utils.fubar("Cannot find suite %s" % Options["Suite"].lower())

    suite_id = suite.suite_id
    suite_name = suite.suite_name.lower()

    if "obsolete source" in checks:
        report_obsolete_source(suite_name, session)

    if "nbs" in checks:
        reportAllNBS(suite_name, suite_id, session, rdeps)

    if "outdated non-free" in checks:
        report_outdated_nonfree(suite_name, session, rdeps)

    bin_not_built = {}

    if "bnb" in checks:
        bins_in_suite = get_suite_binaries(suite, session)

    # Checks based on the Sources files
    components = get_component_names(session)
    for component in components:
        filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path, suite_name, component)
        # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
        (fd, temp_filename) = utils.temp_filename()
        (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
        if (result != 0):
            sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
            sys.exit(result)
        sources = utils.open_file(temp_filename)
        Sources = apt_pkg.TagFile(sources)
        while Sources.step():
            source = Sources.section.find('Package')
            source_version = Sources.section.find('Version')
            architecture = Sources.section.find('Architecture')
            binaries = Sources.section.find('Binary')
            binaries_list = [ i.strip() for i in  binaries.split(',') ]

            if "bnb" in checks:
                # Check for binaries not built on any architecture.
                for binary in binaries_list:
                    if not bins_in_suite.has_key(binary):
                        bin_not_built.setdefault(source, {})
                        bin_not_built[source][binary] = ""

            if "anais" in checks:
                anais_output += do_anais(architecture, binaries_list, source, session)

            # build indices for checking "no source" later
            source_index = component + '/' + source
            src_pkgs[source] = source_index
            for binary in binaries_list:
                bin_pkgs[binary] = source
            source_binaries[source] = binaries
            source_versions[source] = source_version

        sources.close()
        os.unlink(temp_filename)

    # Checks based on the Packages files
    check_components = components[:]
    if suite_name != "experimental":
        check_components.append('main/debian-installer');

    for component in check_components:
        architectures = [ a.arch_string for a in get_suite_architectures(suite_name,
                                                                         skipsrc=True, skipall=True,
                                                                         session=session) ]
        for architecture in architectures:
            if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
                continue
            filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (suite.archive.path, suite_name, component, architecture)
            # apt_pkg.TagFile needs a real file handle
            (fd, temp_filename) = utils.temp_filename()
            (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
            if (result != 0):
                sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
                sys.exit(result)

            if "nfu" in checks:
                nfu_packages.setdefault(architecture,[])
                nfu_entries = parse_nfu(architecture)

            packages = utils.open_file(temp_filename)
            Packages = apt_pkg.TagFile(packages)
            while Packages.step():
                package = Packages.section.find('Package')
                source = Packages.section.find('Source', "")
                version = Packages.section.find('Version')
                if source == "":
                    source = package
                if bin2source.has_key(package) and \
                       apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
                    bin2source[package]["version"] = version
                    bin2source[package]["source"] = source
                else:
                    bin2source[package] = {}
                    bin2source[package]["version"] = version
                    bin2source[package]["source"] = source
                if source.find("(") != -1:
                    m = re_extract_src_version.match(source)
                    source = m.group(1)
                    version = m.group(2)
                if not bin_pkgs.has_key(package):
                    nbs.setdefault(source,{})
                    nbs[source].setdefault(package, {})
                    nbs[source][package][version] = ""
                else:
                    if "nfu" in checks:
                        if package in nfu_entries and \
                               version != source_versions[source]: # only suggest to remove out-of-date packages
                            nfu_packages[architecture].append((package,version,source_versions[source]))
                    
            packages.close()
            os.unlink(temp_filename)

    # Distinguish dubious (version numbers match) and 'real' NBS (they don't)
    dubious_nbs = {}
    for source in nbs.keys():
        for package in nbs[source].keys():
            versions = nbs[source][package].keys()
            versions.sort(apt_pkg.version_compare)
            latest_version = versions.pop()
            source_version = source_versions.get(source,"0")
            if apt_pkg.version_compare(latest_version, source_version) == 0:
                add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)

    if "nviu" in checks:
        do_newer_version('unstable', 'experimental', 'NVIU', session)

    if "nvit" in checks:
        do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)

    ###

    if Options["Mode"] == "full":
        print "="*75
        print

    if "nfu" in checks:
        do_nfu(nfu_packages)

    if "bnb" in checks:
        print "Unbuilt binary packages"
        print "-----------------------"
        print
        keys = bin_not_built.keys()
        keys.sort()
        for source in keys:
            binaries = bin_not_built[source].keys()
            binaries.sort()
            print " o %s: %s" % (source, ", ".join(binaries))
        print

    if "bms" in checks:
        report_multiple_source(suite)

    if "anais" in checks:
        print "Architecture Not Allowed In Source"
        print "----------------------------------"
        print anais_output
        print

    if "dubious nbs" in checks:
        do_dubious_nbs(dubious_nbs)
Example #14
0
def main ():
    global Options

    cnf = Config()

    Arguments = [('h',"help","Rm::Options::Help"),
                 ('A','no-arch-all-rdeps','Rm::Options::NoArchAllRdeps'),
                 ('a',"architecture","Rm::Options::Architecture", "HasArg"),
                 ('b',"binary", "Rm::Options::Binary"),
                 ('B',"binary-only", "Rm::Options::Binary-Only"),
                 ('c',"component", "Rm::Options::Component", "HasArg"),
                 ('C',"carbon-copy", "Rm::Options::Carbon-Copy", "HasArg"), # Bugs to Cc
                 ('d',"done","Rm::Options::Done", "HasArg"), # Bugs fixed
                 ('D',"do-close","Rm::Options::Do-Close"),
                 ('R',"rdep-check", "Rm::Options::Rdep-Check"),
                 ('m',"reason", "Rm::Options::Reason", "HasArg"), # Hysterical raisins; -m is old-dinstall option for rejection reason
                 ('n',"no-action","Rm::Options::No-Action"),
                 ('p',"partial", "Rm::Options::Partial"),
                 ('s',"suite","Rm::Options::Suite", "HasArg"),
                 ('S',"source-only", "Rm::Options::Source-Only"),
                 ]

    for i in [ 'NoArchAllRdeps',
               "architecture", "binary", "binary-only", "carbon-copy", "component",
               "done", "help", "no-action", "partial", "rdep-check", "reason",
               "source-only", "Do-Close" ]:
        if not cnf.has_key("Rm::Options::%s" % (i)):
            cnf["Rm::Options::%s" % (i)] = ""
    if not cnf.has_key("Rm::Options::Suite"):
        cnf["Rm::Options::Suite"] = "unstable"

    arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Rm::Options")

    if Options["Help"]:
        usage()

    session = DBConn().session()

    # Sanity check options
    if not arguments:
        utils.fubar("need at least one package name as an argument.")
    if Options["Architecture"] and Options["Source-Only"]:
        utils.fubar("can't use -a/--architecture and -S/--source-only options simultaneously.")
    if ((Options["Binary"] and Options["Source-Only"])
            or (Options["Binary"] and Options["Binary-Only"])
            or (Options["Binary-Only"] and Options["Source-Only"])):
        utils.fubar("Only one of -b/--binary, -B/--binary-only and -S/--source-only can be used.")
    if Options.has_key("Carbon-Copy") and not Options.has_key("Done"):
        utils.fubar("can't use -C/--carbon-copy without also using -d/--done option.")
    if Options["Architecture"] and not Options["Partial"]:
        utils.warn("-a/--architecture implies -p/--partial.")
        Options["Partial"] = "true"
    if Options["Do-Close"] and not Options["Done"]:
        utils.fubar("No.")
    if (Options["Do-Close"]
           and (Options["Binary"] or Options["Binary-Only"] or Options["Source-Only"])):
        utils.fubar("No.")

    # Force the admin to tell someone if we're not doing a 'dak
    # cruft-report' inspired removal (or closing a bug, which counts
    # as telling someone).
    if not Options["No-Action"] and not Options["Carbon-Copy"] \
           and not Options["Done"] and Options["Reason"].find("[auto-cruft]") == -1:
        utils.fubar("Need a -C/--carbon-copy if not closing a bug and not doing a cruft removal.")

    # Process -C/--carbon-copy
    #
    # Accept 3 types of arguments (space separated):
    #  1) a number - assumed to be a bug number, i.e. [email protected]
    #  2) the keyword 'package' - cc's [email protected] for every argument
    #  3) contains a '@' - assumed to be an email address, used unmodified
    #
    carbon_copy = []
    for copy_to in utils.split_args(Options.get("Carbon-Copy")):
        if copy_to.isdigit():
            if cnf.has_key("Dinstall::BugServer"):
                carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
            else:
                utils.fubar("Asked to send mail to #%s in BTS but Dinstall::BugServer is not configured" % copy_to)
        elif copy_to == 'package':
            for package in arguments:
                if cnf.has_key("Dinstall::PackagesServer"):
                    carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"])
                if cnf.has_key("Dinstall::TrackingServer"):
                    carbon_copy.append(package + "@" + cnf["Dinstall::TrackingServer"])
        elif '@' in copy_to:
            carbon_copy.append(copy_to)
        else:
            utils.fubar("Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address." % (copy_to))

    if Options["Binary"]:
        field = "b.package"
    else:
        field = "s.source"
    con_packages = "AND %s IN (%s)" % (field, ", ".join([ repr(i) for i in arguments ]))

    (con_suites, con_architectures, con_components, check_source) = \
                 utils.parse_args(Options)

    # Additional suite checks
    suite_ids_list = []
    whitelists = []
    suites = utils.split_args(Options["Suite"])
    suites_list = utils.join_with_commas_and(suites)
    if not Options["No-Action"]:
        for suite in suites:
            s = get_suite(suite, session=session)
            if s is not None:
                suite_ids_list.append(s.suite_id)
                whitelists.append(s.mail_whitelist)
            if suite in ("oldstable", "stable"):
                print "**WARNING** About to remove from the (old)stable suite!"
                print "This should only be done just prior to a (point) release and not at"
                print "any other time."
                game_over()
            elif suite == "testing":
                print "**WARNING About to remove from the testing suite!"
                print "There's no need to do this normally as removals from unstable will"
                print "propogate to testing automagically."
                game_over()

    # Additional architecture checks
    if Options["Architecture"] and check_source:
        utils.warn("'source' in -a/--argument makes no sense and is ignored.")

    # Don't do dependency checks on multiple suites
    if Options["Rdep-Check"] and len(suites) > 1:
        utils.fubar("Reverse dependency check on multiple suites is not implemented.")

    to_remove = []
    maintainers = {}

    # We have 3 modes of package selection: binary, source-only, binary-only
    # and source+binary.

    # XXX: TODO: This all needs converting to use placeholders or the object
    #            API. It's an SQL injection dream at the moment

    if Options["Binary"]:
        # Removal by binary package name
        q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, files_archive_map af, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
        to_remove.extend(q)
    else:
        # Source-only
        if not Options["Binary-Only"]:
            q = session.execute("SELECT s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, archive, files f, files_archive_map af, component c WHERE sa.source = s.id AND sa.suite = su.id AND archive.id = su.archive_id AND s.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s" % (con_packages, con_suites, con_components))
            to_remove.extend(q)
        if not Options["Source-Only"]:
            # Source + Binary
            q = session.execute("""
                    SELECT b.package, b.version, a.arch_string, b.id, b.maintainer
                    FROM binaries b
                         JOIN bin_associations ba ON b.id = ba.bin
                         JOIN architecture a ON b.architecture = a.id
                         JOIN suite su ON ba.suite = su.id
                         JOIN archive ON archive.id = su.archive_id
                         JOIN files_archive_map af ON b.file = af.file_id AND af.archive_id = archive.id
                         JOIN component c ON af.component_id = c.id
                         JOIN source s ON b.source = s.id
                         JOIN src_associations sa ON s.id = sa.source AND sa.suite = su.id
                    WHERE TRUE %s %s %s %s""" % (con_packages, con_suites, con_components, con_architectures))
            to_remove.extend(q)

    if not to_remove:
        print "Nothing to do."
        sys.exit(0)

    # If we don't have a reason; spawn an editor so the user can add one
    # Write the rejection email out as the <foo>.reason file
    if not Options["Reason"] and not Options["No-Action"]:
        (fd, temp_filename) = utils.temp_filename()
        editor = os.environ.get("EDITOR","vi")
        result = os.system("%s %s" % (editor, temp_filename))
        if result != 0:
            utils.fubar ("vi invocation failed for `%s'!" % (temp_filename), result)
        temp_file = utils.open_file(temp_filename)
        for line in temp_file.readlines():
            Options["Reason"] += line
        temp_file.close()
        os.unlink(temp_filename)

    # Generate the summary of what's to be removed
    d = {}
    for i in to_remove:
        package = i[0]
        version = i[1]
        architecture = i[2]
        maintainer = i[4]
        maintainers[maintainer] = ""
        if not d.has_key(package):
            d[package] = {}
        if not d[package].has_key(version):
            d[package][version] = []
        if architecture not in d[package][version]:
            d[package][version].append(architecture)

    maintainer_list = []
    for maintainer_id in maintainers.keys():
        maintainer_list.append(get_maintainer(maintainer_id).name)
    summary = ""
    removals = d.keys()
    removals.sort()
    for package in removals:
        versions = d[package].keys()
        versions.sort(apt_pkg.version_compare)
        for version in versions:
            d[package][version].sort(utils.arch_compare_sw)
            summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
    print "Will remove the following packages from %s:" % (suites_list)
    print
    print summary
    print "Maintainer: %s" % ", ".join(maintainer_list)
    if Options["Done"]:
        print "Will also close bugs: "+Options["Done"]
    if carbon_copy:
        print "Will also send CCs to: " + ", ".join(carbon_copy)
    if Options["Do-Close"]:
        print "Will also close associated bug reports."
    print
    print "------------------- Reason -------------------"
    print Options["Reason"]
    print "----------------------------------------------"
    print

    if Options["Rdep-Check"]:
        arches = utils.split_args(Options["Architecture"])
        include_arch_all = Options['NoArchAllRdeps'] == ''
        reverse_depends_check(removals, suites[0], arches, session, include_arch_all=include_arch_all)

    # If -n/--no-action, drop out here
    if Options["No-Action"]:
        sys.exit(0)

    print "Going to remove the packages now."
    game_over()

    # Do the actual deletion
    print "Deleting...",
    sys.stdout.flush()

    try:
        bugs = utils.split_args(Options["Done"])
        remove(session, Options["Reason"], suites, to_remove,
               partial=Options["Partial"], components=utils.split_args(Options["Component"]),
               done_bugs=bugs, carbon_copy=carbon_copy, close_related_bugs=Options["Do-Close"]
               )
    except ValueError as ex:
        utils.fubar(ex.message)
    else:
        print "done."
Example #15
0
def main():
    global Options

    cnf = Config()

    Arguments = [
        ('h', "help", "Rm::Options::Help"),
        ('A', 'no-arch-all-rdeps', 'Rm::Options::NoArchAllRdeps'),
        ('a', "architecture", "Rm::Options::Architecture", "HasArg"),
        ('b', "binary", "Rm::Options::Binary"),
        ('B', "binary-only", "Rm::Options::Binary-Only"),
        ('c', "component", "Rm::Options::Component", "HasArg"),
        ('C', "carbon-copy", "Rm::Options::Carbon-Copy",
         "HasArg"),  # Bugs to Cc
        ('d', "done", "Rm::Options::Done", "HasArg"),  # Bugs fixed
        ('D', "do-close", "Rm::Options::Do-Close"),
        ('R', "rdep-check", "Rm::Options::Rdep-Check"),
        (
            'm', "reason", "Rm::Options::Reason", "HasArg"
        ),  # Hysterical raisins; -m is old-dinstall option for rejection reason
        ('n', "no-action", "Rm::Options::No-Action"),
        ('p', "partial", "Rm::Options::Partial"),
        ('s', "suite", "Rm::Options::Suite", "HasArg"),
        ('S', "source-only", "Rm::Options::Source-Only"),
    ]

    for i in [
            'NoArchAllRdeps', "architecture", "binary", "binary-only",
            "carbon-copy", "component", "done", "help", "no-action", "partial",
            "rdep-check", "reason", "source-only", "Do-Close"
    ]:
        key = "Rm::Options::%s" % (i)
        if key not in cnf:
            cnf[key] = ""
    if "Rm::Options::Suite" not in cnf:
        cnf["Rm::Options::Suite"] = "unstable"

    arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Rm::Options")

    if Options["Help"]:
        usage()

    session = DBConn().session()

    # Sanity check options
    if not arguments:
        utils.fubar("need at least one package name as an argument.")
    if Options["Architecture"] and Options["Source-Only"]:
        utils.fubar(
            "can't use -a/--architecture and -S/--source-only options simultaneously."
        )
    if ((Options["Binary"] and Options["Source-Only"])
            or (Options["Binary"] and Options["Binary-Only"])
            or (Options["Binary-Only"] and Options["Source-Only"])):
        utils.fubar(
            "Only one of -b/--binary, -B/--binary-only and -S/--source-only can be used."
        )
    if "Carbon-Copy" not in Options and "Done" not in Options:
        utils.fubar(
            "can't use -C/--carbon-copy without also using -d/--done option.")
    if Options["Architecture"] and not Options["Partial"]:
        utils.warn("-a/--architecture implies -p/--partial.")
        Options["Partial"] = "true"
    if Options["Do-Close"] and not Options["Done"]:
        utils.fubar("No.")
    if (Options["Do-Close"] and (Options["Binary"] or Options["Binary-Only"]
                                 or Options["Source-Only"])):
        utils.fubar("No.")

    # Force the admin to tell someone if we're not doing a 'dak
    # cruft-report' inspired removal (or closing a bug, which counts
    # as telling someone).
    if not Options["No-Action"] and not Options["Carbon-Copy"] \
           and not Options["Done"] and Options["Reason"].find("[auto-cruft]") == -1:
        utils.fubar(
            "Need a -C/--carbon-copy if not closing a bug and not doing a cruft removal."
        )

    if Options["Binary"]:
        field = "b.package"
    else:
        field = "s.source"
    con_packages = "AND %s IN (%s)" % (field, ", ".join(
        [repr(i) for i in arguments]))

    (con_suites, con_architectures, con_components, check_source) = \
                 utils.parse_args(Options)

    # Additional suite checks
    suite_ids_list = []
    whitelists = []
    suites = utils.split_args(Options["Suite"])
    suites_list = utils.join_with_commas_and(suites)
    if not Options["No-Action"]:
        for suite in suites:
            s = get_suite(suite, session=session)
            if s is not None:
                suite_ids_list.append(s.suite_id)
                whitelists.append(s.mail_whitelist)
            if suite in ("oldstable", "stable"):
                print(
                    "**WARNING** About to remove from the (old)stable suite!")
                print(
                    "This should only be done just prior to a (point) release and not at"
                )
                print("any other time.")
                game_over()
            elif suite == "testing":
                print("**WARNING About to remove from the testing suite!")
                print(
                    "There's no need to do this normally as removals from unstable will"
                )
                print("propogate to testing automagically.")
                game_over()

    # Additional architecture checks
    if Options["Architecture"] and check_source:
        utils.warn("'source' in -a/--argument makes no sense and is ignored.")

    # Don't do dependency checks on multiple suites
    if Options["Rdep-Check"] and len(suites) > 1:
        utils.fubar(
            "Reverse dependency check on multiple suites is not implemented.")

    to_remove = []
    maintainers = {}

    # We have 3 modes of package selection: binary, source-only, binary-only
    # and source+binary.

    # XXX: TODO: This all needs converting to use placeholders or the object
    #            API. It's an SQL injection dream at the moment

    if Options["Binary"]:
        # Removal by binary package name
        q = session.execute("""
                SELECT b.package, b.version, a.arch_string, b.id, b.maintainer, s.source
                FROM binaries b
                     JOIN source s ON s.id = b.source
                     JOIN bin_associations ba ON ba.bin = b.id
                     JOIN architecture a ON a.id = b.architecture
                     JOIN suite su ON su.id = ba.suite
                     JOIN files f ON f.id = b.file
                     JOIN files_archive_map af ON af.file_id = f.id AND af.archive_id = su.archive_id
                     JOIN component c ON c.id = af.component_id
                WHERE TRUE %s %s %s %s
        """ % (con_packages, con_suites, con_components, con_architectures))
        to_remove.extend(q)
    else:
        # Source-only
        if not Options["Binary-Only"]:
            q = session.execute("""
                    SELECT s.source, s.version, 'source', s.id, s.maintainer, s.source
                    FROM source s
                         JOIN src_associations sa ON sa.source = s.id
                         JOIN suite su ON su.id = sa.suite
                         JOIN archive ON archive.id = su.archive_id
                         JOIN files f ON f.id = s.file
                         JOIN files_archive_map af ON af.file_id = f.id AND af.archive_id = su.archive_id
                         JOIN component c ON c.id = af.component_id
                    WHERE TRUE %s %s %s
            """ % (con_packages, con_suites, con_components))
            to_remove.extend(q)
        if not Options["Source-Only"]:
            # Source + Binary
            q = session.execute(
                """
                    SELECT b.package, b.version, a.arch_string, b.id, b.maintainer, s.source
                    FROM binaries b
                         JOIN bin_associations ba ON b.id = ba.bin
                         JOIN architecture a ON b.architecture = a.id
                         JOIN suite su ON ba.suite = su.id
                         JOIN archive ON archive.id = su.archive_id
                         JOIN files_archive_map af ON b.file = af.file_id AND af.archive_id = archive.id
                         JOIN component c ON af.component_id = c.id
                         JOIN source s ON b.source = s.id
                         JOIN src_associations sa ON s.id = sa.source AND sa.suite = su.id
                    WHERE TRUE %s %s %s %s""" %
                (con_packages, con_suites, con_components, con_architectures))
            to_remove.extend(q)

    if not to_remove:
        print("Nothing to do.")
        sys.exit(0)

    # Process -C/--carbon-copy
    #
    # Accept 3 types of arguments (space separated):
    #  1) a number - assumed to be a bug number, i.e. [email protected]
    #  2) the keyword 'package' - cc's [email protected] for every argument
    #  3) contains a '@' - assumed to be an email address, used unmodified
    #
    carbon_copy = []
    for copy_to in utils.split_args(Options.get("Carbon-Copy")):
        if copy_to.isdigit():
            if "Dinstall::BugServer" in cnf:
                carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
            else:
                utils.fubar(
                    "Asked to send mail to #%s in BTS but Dinstall::BugServer is not configured"
                    % copy_to)
        elif copy_to == 'package':
            for package in set([s[5] for s in to_remove]):
                if "Dinstall::PackagesServer" in cnf:
                    carbon_copy.append(package + "@" +
                                       cnf["Dinstall::PackagesServer"])
        elif '@' in copy_to:
            carbon_copy.append(copy_to)
        else:
            utils.fubar(
                "Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address."
                % (copy_to))

    # If we don't have a reason; spawn an editor so the user can add one
    # Write the rejection email out as the <foo>.reason file
    if not Options["Reason"] and not Options["No-Action"]:
        (fd, temp_filename) = utils.temp_filename()
        editor = os.environ.get("EDITOR", "vi")
        result = os.system("%s %s" % (editor, temp_filename))
        if result != 0:
            utils.fubar("vi invocation failed for `%s'!" % (temp_filename),
                        result)
        temp_file = utils.open_file(temp_filename)
        for line in temp_file.readlines():
            Options["Reason"] += line
        temp_file.close()
        os.unlink(temp_filename)

    # Generate the summary of what's to be removed
    d = {}
    for i in to_remove:
        package = i[0]
        version = i[1]
        architecture = i[2]
        maintainer = i[4]
        maintainers[maintainer] = ""
        if package not in d:
            d[package] = {}
        if version not in d[package]:
            d[package][version] = []
        if architecture not in d[package][version]:
            d[package][version].append(architecture)

    maintainer_list = []
    for maintainer_id in maintainers.keys():
        maintainer_list.append(get_maintainer(maintainer_id).name)
    summary = ""
    removals = d.keys()
    removals.sort()
    for package in removals:
        versions = d[package].keys()
        versions.sort(key=functools.cmp_to_key(apt_pkg.version_compare))
        for version in versions:
            d[package][version].sort(key=utils.ArchKey)
            summary += "%10s | %10s | %s\n" % (package, version, ", ".join(
                d[package][version]))
    print("Will remove the following packages from %s:" % (suites_list))
    print()
    print(summary)
    print("Maintainer: %s" % ", ".join(maintainer_list))
    if Options["Done"]:
        print("Will also close bugs: " + Options["Done"])
    if carbon_copy:
        print("Will also send CCs to: " + ", ".join(carbon_copy))
    if Options["Do-Close"]:
        print("Will also close associated bug reports.")
    print()
    print("------------------- Reason -------------------")
    print(Options["Reason"])
    print("----------------------------------------------")
    print()

    if Options["Rdep-Check"]:
        arches = utils.split_args(Options["Architecture"])
        include_arch_all = Options['NoArchAllRdeps'] == ''
        reverse_depends_check(removals,
                              suites[0],
                              arches,
                              session,
                              include_arch_all=include_arch_all)

    # If -n/--no-action, drop out here
    if Options["No-Action"]:
        sys.exit(0)

    print("Going to remove the packages now.")
    game_over()

    # Do the actual deletion
    print("Deleting...", end=' ')
    sys.stdout.flush()

    try:
        bugs = utils.split_args(Options["Done"])
        remove(session,
               Options["Reason"],
               suites,
               to_remove,
               partial=Options["Partial"],
               components=utils.split_args(Options["Component"]),
               done_bugs=bugs,
               carbon_copy=carbon_copy,
               close_related_bugs=Options["Do-Close"])
    except ValueError as ex:
        utils.fubar(ex.message)
    else:
        print("done.")
Example #16
0
def main ():
    global Options

    cnf = Config()

    Arguments = [('h',"help","Rm::Options::Help"),
                 ('a',"architecture","Rm::Options::Architecture", "HasArg"),
                 ('b',"binary", "Rm::Options::Binary"),
                 ('B',"binary-only", "Rm::Options::Binary-Only"),
                 ('c',"component", "Rm::Options::Component", "HasArg"),
                 ('C',"carbon-copy", "Rm::Options::Carbon-Copy", "HasArg"), # Bugs to Cc
                 ('d',"done","Rm::Options::Done", "HasArg"), # Bugs fixed
                 ('D',"do-close","Rm::Options::Do-Close"),
                 ('R',"rdep-check", "Rm::Options::Rdep-Check"),
                 ('m',"reason", "Rm::Options::Reason", "HasArg"), # Hysterical raisins; -m is old-dinstall option for rejection reason
                 ('n',"no-action","Rm::Options::No-Action"),
                 ('p',"partial", "Rm::Options::Partial"),
                 ('s',"suite","Rm::Options::Suite", "HasArg"),
                 ('S',"source-only", "Rm::Options::Source-Only"),
                 ]

    for i in [ "architecture", "binary", "binary-only", "carbon-copy", "component",
               "done", "help", "no-action", "partial", "rdep-check", "reason",
               "source-only", "Do-Close" ]:
        if not cnf.has_key("Rm::Options::%s" % (i)):
            cnf["Rm::Options::%s" % (i)] = ""
    if not cnf.has_key("Rm::Options::Suite"):
        cnf["Rm::Options::Suite"] = "unstable"

    arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Rm::Options")

    if Options["Help"]:
        usage()

    session = DBConn().session()

    # Sanity check options
    if not arguments:
        utils.fubar("need at least one package name as an argument.")
    if Options["Architecture"] and Options["Source-Only"]:
        utils.fubar("can't use -a/--architecture and -S/--source-only options simultaneously.")
    if ((Options["Binary"] and Options["Source-Only"])
            or (Options["Binary"] and Options["Binary-Only"])
            or (Options["Binary-Only"] and Options["Source-Only"])):
        utils.fubar("Only one of -b/--binary, -B/--binary-only and -S/--source-only can be used.")
    if Options.has_key("Carbon-Copy") and not Options.has_key("Done"):
        utils.fubar("can't use -C/--carbon-copy without also using -d/--done option.")
    if Options["Architecture"] and not Options["Partial"]:
        utils.warn("-a/--architecture implies -p/--partial.")
        Options["Partial"] = "true"
    if Options["Do-Close"] and not Options["Done"]:
        utils.fubar("No.")
    if (Options["Do-Close"]
           and (Options["Binary"] or Options["Binary-Only"] or Options["Source-Only"])):
        utils.fubar("No.")

    # Force the admin to tell someone if we're not doing a 'dak
    # cruft-report' inspired removal (or closing a bug, which counts
    # as telling someone).
    if not Options["No-Action"] and not Options["Carbon-Copy"] \
           and not Options["Done"] and Options["Reason"].find("[auto-cruft]") == -1:
        utils.fubar("Need a -C/--carbon-copy if not closing a bug and not doing a cruft removal.")

    # Process -C/--carbon-copy
    #
    # Accept 3 types of arguments (space separated):
    #  1) a number - assumed to be a bug number, i.e. [email protected]
    #  2) the keyword 'package' - cc's [email protected] for every argument
    #  3) contains a '@' - assumed to be an email address, used unmofidied
    #
    carbon_copy = []
    for copy_to in utils.split_args(Options.get("Carbon-Copy")):
        if copy_to.isdigit():
            if cnf.has_key("Dinstall::BugServer"):
                carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
            else:
                utils.fubar("Asked to send mail to #%s in BTS but Dinstall::BugServer is not configured" % copy_to)
        elif copy_to == 'package':
            for package in arguments:
                if cnf.has_key("Dinstall::PackagesServer"):
                    carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"])
                if cnf.has_key("Dinstall::TrackingServer"):
                    carbon_copy.append(package + "@" + cnf["Dinstall::TrackingServer"])
        elif '@' in copy_to:
            carbon_copy.append(copy_to)
        else:
            utils.fubar("Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address." % (copy_to))

    if Options["Binary"]:
        field = "b.package"
    else:
        field = "s.source"
    con_packages = "AND %s IN (%s)" % (field, ", ".join([ repr(i) for i in arguments ]))

    (con_suites, con_architectures, con_components, check_source) = \
                 utils.parse_args(Options)

    # Additional suite checks
    suite_ids_list = []
    whitelists = []
    suites = utils.split_args(Options["Suite"])
    suites_list = utils.join_with_commas_and(suites)
    if not Options["No-Action"]:
        for suite in suites:
            s = get_suite(suite, session=session)
            if s is not None:
                suite_ids_list.append(s.suite_id)
                whitelists.append(s.mail_whitelist)
            if suite in ("oldstable", "stable"):
                print "**WARNING** About to remove from the (old)stable suite!"
                print "This should only be done just prior to a (point) release and not at"
                print "any other time."
                game_over()
            elif suite == "testing":
                print "**WARNING About to remove from the testing suite!"
                print "There's no need to do this normally as removals from unstable will"
                print "propogate to testing automagically."
                game_over()

    # Additional architecture checks
    if Options["Architecture"] and check_source:
        utils.warn("'source' in -a/--argument makes no sense and is ignored.")

    # Additional component processing
    over_con_components = con_components.replace("c.id", "component")

    # Don't do dependency checks on multiple suites
    if Options["Rdep-Check"] and len(suites) > 1:
        utils.fubar("Reverse dependency check on multiple suites is not implemented.")

    to_remove = []
    maintainers = {}

    # We have 3 modes of package selection: binary, source-only, binary-only
    # and source+binary.

    # XXX: TODO: This all needs converting to use placeholders or the object
    #            API. It's an SQL injection dream at the moment

    if Options["Binary"]:
        # Removal by binary package name
        q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, files_archive_map af, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
        to_remove.extend(q)
    else:
        # Source-only
        if not Options["Binary-Only"]:
            q = session.execute("SELECT s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, archive, files f, files_archive_map af, component c WHERE sa.source = s.id AND sa.suite = su.id AND archive.id = su.archive_id AND s.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s" % (con_packages, con_suites, con_components))
            to_remove.extend(q)
        if not Options["Source-Only"]:
            # Source + Binary
            q = session.execute("""
                    SELECT b.package, b.version, a.arch_string, b.id, b.maintainer
                    FROM binaries b
                         JOIN bin_associations ba ON b.id = ba.bin
                         JOIN architecture a ON b.architecture = a.id
                         JOIN suite su ON ba.suite = su.id
                         JOIN archive ON archive.id = su.archive_id
                         JOIN files_archive_map af ON b.file = af.file_id AND af.archive_id = archive.id
                         JOIN component c ON af.component_id = c.id
                         JOIN source s ON b.source = s.id
                         JOIN src_associations sa ON s.id = sa.source AND sa.suite = su.id
                    WHERE TRUE %s %s %s %s""" % (con_packages, con_suites, con_components, con_architectures))
            to_remove.extend(q)

    if not to_remove:
        print "Nothing to do."
        sys.exit(0)

    # If we don't have a reason; spawn an editor so the user can add one
    # Write the rejection email out as the <foo>.reason file
    if not Options["Reason"] and not Options["No-Action"]:
        (fd, temp_filename) = utils.temp_filename()
        editor = os.environ.get("EDITOR","vi")
        result = os.system("%s %s" % (editor, temp_filename))
        if result != 0:
            utils.fubar ("vi invocation failed for `%s'!" % (temp_filename), result)
        temp_file = utils.open_file(temp_filename)
        for line in temp_file.readlines():
            Options["Reason"] += line
        temp_file.close()
        os.unlink(temp_filename)

    # Generate the summary of what's to be removed
    d = {}
    for i in to_remove:
        package = i[0]
        version = i[1]
        architecture = i[2]
        maintainer = i[4]
        maintainers[maintainer] = ""
        if not d.has_key(package):
            d[package] = {}
        if not d[package].has_key(version):
            d[package][version] = []
        if architecture not in d[package][version]:
            d[package][version].append(architecture)

    maintainer_list = []
    for maintainer_id in maintainers.keys():
        maintainer_list.append(get_maintainer(maintainer_id).name)
    summary = ""
    removals = d.keys()
    removals.sort()
    versions = []
    for package in removals:
        versions = d[package].keys()
        versions.sort(apt_pkg.version_compare)
        for version in versions:
            d[package][version].sort(utils.arch_compare_sw)
            summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
    print "Will remove the following packages from %s:" % (suites_list)
    print
    print summary
    print "Maintainer: %s" % ", ".join(maintainer_list)
    if Options["Done"]:
        print "Will also close bugs: "+Options["Done"]
    if carbon_copy:
        print "Will also send CCs to: " + ", ".join(carbon_copy)
    if Options["Do-Close"]:
        print "Will also close associated bug reports."
    print
    print "------------------- Reason -------------------"
    print Options["Reason"]
    print "----------------------------------------------"
    print

    if Options["Rdep-Check"]:
        arches = utils.split_args(Options["Architecture"])
        reverse_depends_check(removals, suites[0], arches, session)

    # If -n/--no-action, drop out here
    if Options["No-Action"]:
        sys.exit(0)

    print "Going to remove the packages now."
    game_over()

    whoami = utils.whoami()
    date = commands.getoutput('date -R')

    # Log first; if it all falls apart I want a record that we at least tried.
    logfile = utils.open_file(cnf["Rm::LogFile"], 'a')
    logfile.write("=========================================================================\n")
    logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
    logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
    if Options["Done"]:
        logfile.write("Closed bugs: %s\n" % (Options["Done"]))
    logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"]))
    logfile.write("----------------------------------------------\n")

    # Do the same in rfc822 format
    logfile822 = utils.open_file(cnf["Rm::LogFile822"], 'a')
    logfile822.write("Date: %s\n" % date)
    logfile822.write("Ftpmaster: %s\n" % whoami)
    logfile822.write("Suite: %s\n" % suites_list)
    sources = []
    binaries = []
    for package in summary.split("\n"):
        for row in package.split("\n"):
            element = row.split("|")
            if len(element) == 3:
                if element[2].find("source") > 0:
                    sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
                    element[2] = sub("source\s?,?", "", element[2]).strip(" ")
                if element[2]:
                    binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
    if sources:
        logfile822.write("Sources:\n")
        for source in sources:
            logfile822.write(" %s\n" % source)
    if binaries:
        logfile822.write("Binaries:\n")
        for binary in binaries:
            logfile822.write(" %s\n" % binary)
    logfile822.write("Reason: %s\n" % Options["Reason"].replace('\n', '\n '))
    if Options["Done"]:
        logfile822.write("Bug: %s\n" % Options["Done"])

    dsc_type_id = get_override_type('dsc', session).overridetype_id
    deb_type_id = get_override_type('deb', session).overridetype_id

    # Do the actual deletion
    print "Deleting...",
    sys.stdout.flush()

    for i in to_remove:
        package = i[0]
        architecture = i[2]
        package_id = i[3]
        for suite_id in suite_ids_list:
            if architecture == "source":
                session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
                                {'packageid': package_id, 'suiteid': suite_id})
                #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)
            else:
                session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
                                {'packageid': package_id, 'suiteid': suite_id})
                #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)
            # Delete from the override file
            if not Options["Partial"]:
                if architecture == "source":
                    type_id = dsc_type_id
                else:
                    type_id = deb_type_id
                # TODO: Again, fix this properly to remove the remaining non-bind argument
                session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
    session.commit()
    print "done."

    # If we don't have a Bug server configured, we're done
    if not cnf.has_key("Dinstall::BugServer"):
        if Options["Done"] or Options["Do-Close"]:
            print "Cannot send mail to BugServer as Dinstall::BugServer is not configured"

        logfile.write("=========================================================================\n")
        logfile.close()

        logfile822.write("\n")
        logfile822.close()

        return

    # read common subst variables for all bug closure mails
    Subst_common = {}
    Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
    Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
    Subst_common["__CC__"] = "X-DAK: dak rm"
    if carbon_copy:
        Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
    Subst_common["__SUITE_LIST__"] = suites_list
    Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
    Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
    Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
    Subst_common["__WHOAMI__"] = whoami

    # Send the bug closing messages
    if Options["Done"]:
        Subst_close_rm = Subst_common
        bcc = []
        if cnf.find("Dinstall::Bcc") != "":
            bcc.append(cnf["Dinstall::Bcc"])
        if cnf.find("Rm::Bcc") != "":
            bcc.append(cnf["Rm::Bcc"])
        if bcc:
            Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
        else:
            Subst_close_rm["__BCC__"] = "X-Filler: 42"
        summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, Options["Reason"])
        summarymail += "----------------------------------------------\n"
        Subst_close_rm["__SUMMARY__"] = summarymail

        for bug in utils.split_args(Options["Done"]):
            Subst_close_rm["__BUG_NUMBER__"] = bug
            if Options["Do-Close"]:
                mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
            else:
                mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
            utils.send_mail(mail_message, whitelists=whitelists)

    # close associated bug reports
    if Options["Do-Close"]:
        Subst_close_other = Subst_common
        bcc = []
        wnpp = utils.parse_wnpp_bug_file()
        versions = list(set([re_bin_only_nmu.sub('', v) for v in versions]))
        if len(versions) == 1:
            Subst_close_other["__VERSION__"] = versions[0]
        else:
            utils.fubar("Closing bugs with multiple package versions is not supported.  Do it yourself.")
        if bcc:
            Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
        else:
            Subst_close_other["__BCC__"] = "X-Filler: 42"
        # at this point, I just assume, that the first closed bug gives
        # some useful information on why the package got removed
        Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0]
        if len(sources) == 1:
            source_pkg = source.split("_", 1)[0]
        else:
            utils.fubar("Closing bugs for multiple source packages is not supported.  Do it yourself.")
        Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
        Subst_close_other["__SOURCE__"] = source_pkg
        merged_bugs = set()
        other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
        if other_bugs:
            for bugno in other_bugs:
                if bugno not in merged_bugs:
                    for bug in bts.get_status(bugno):
                        for merged in bug.mergedwith:
                            other_bugs.remove(merged)
                            merged_bugs.add(merged)
            logfile.write("Also closing bug(s):")
            logfile822.write("Also-Bugs:")
            for bug in other_bugs:
                Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
                logfile.write(" " + str(bug))
                logfile822.write(" " + str(bug))
            logfile.write("\n")
            logfile822.write("\n")
        if source_pkg in wnpp.keys():
            logfile.write("Also closing WNPP bug(s):")
            logfile822.write("Also-WNPP:")
            for bug in wnpp[source_pkg]:
                # the wnpp-rm file we parse also contains our removal
                # bugs, filtering that out
                if bug != Subst_close_other["__BUG_NUMBER__"]:
                    Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
                    logfile.write(" " + str(bug))
                    logfile822.write(" " + str(bug))
            logfile.write("\n")
            logfile822.write("\n")

        mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related")
        if Subst_close_other["__BUG_NUMBER_ALSO__"]:
            utils.send_mail(mail_message)


    logfile.write("=========================================================================\n")
    logfile.close()

    logfile822.write("\n")
    logfile822.close()
Example #17
0
def main():
    global suite, suite_id, source_binaries, source_versions

    cnf = Config()

    Arguments = [('h', "help", "Cruft-Report::Options::Help"),
                 ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
                 ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
                 ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
                 ('w', "wanna-build-dump",
                  "Cruft-Report::Options::Wanna-Build-Dump", "HasArg")]
    for i in ["help", "Rdep-Check"]:
        if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
            cnf["Cruft-Report::Options::%s" % (i)] = ""

    cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite",
                                                  "unstable")

    if not cnf.has_key("Cruft-Report::Options::Mode"):
        cnf["Cruft-Report::Options::Mode"] = "daily"

    if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
        cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"

    apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)

    Options = cnf.subtree("Cruft-Report::Options")
    if Options["Help"]:
        usage()

    if Options["Rdep-Check"]:
        rdeps = True
    else:
        rdeps = False

    # Set up checks based on mode
    if Options["Mode"] == "daily":
        checks = [
            "nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
            "nfu"
        ]
    elif Options["Mode"] == "full":
        checks = [
            "nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
            "nfu", "dubious nbs", "bnb", "bms", "anais"
        ]
    elif Options["Mode"] == "bdo":
        checks = ["nbs", "obsolete source"]
    else:
        utils.warn(
            "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood."
            % (Options["Mode"]))
        usage(1)

    session = DBConn().session()

    bin_pkgs = {}
    src_pkgs = {}
    bin2source = {}
    bins_in_suite = {}
    nbs = {}
    source_versions = {}

    anais_output = ""

    nfu_packages = {}

    suite = get_suite(Options["Suite"].lower(), session)
    if not suite:
        utils.fubar("Cannot find suite %s" % Options["Suite"].lower())

    suite_id = suite.suite_id
    suite_name = suite.suite_name.lower()

    if "obsolete source" in checks:
        report_obsolete_source(suite_name, session)

    if "nbs" in checks:
        reportAllNBS(suite_name, suite_id, session, rdeps)

    if "outdated non-free" in checks:
        report_outdated_nonfree(suite_name, session, rdeps)

    bin_not_built = {}

    if "bnb" in checks:
        bins_in_suite = get_suite_binaries(suite, session)

    # Checks based on the Sources files
    components = get_component_names(session)
    for component in components:
        filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path,
                                                         suite_name, component)
        # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
        (fd, temp_filename) = utils.temp_filename()
        (result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
                                                    (filename, temp_filename))
        if (result != 0):
            sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
            sys.exit(result)
        sources = utils.open_file(temp_filename)
        Sources = apt_pkg.TagFile(sources)
        while Sources.step():
            source = Sources.section.find('Package')
            source_version = Sources.section.find('Version')
            architecture = Sources.section.find('Architecture')
            binaries = Sources.section.find('Binary')
            binaries_list = [i.strip() for i in binaries.split(',')]

            if "bnb" in checks:
                # Check for binaries not built on any architecture.
                for binary in binaries_list:
                    if not bins_in_suite.has_key(binary):
                        bin_not_built.setdefault(source, {})
                        bin_not_built[source][binary] = ""

            if "anais" in checks:
                anais_output += do_anais(architecture, binaries_list, source,
                                         session)

            # build indices for checking "no source" later
            source_index = component + '/' + source
            src_pkgs[source] = source_index
            for binary in binaries_list:
                bin_pkgs[binary] = source
            source_binaries[source] = binaries
            source_versions[source] = source_version

        sources.close()
        os.unlink(temp_filename)

    # Checks based on the Packages files
    check_components = components[:]
    if suite_name != "staging":
        check_components.append('main/debian-installer')

    for component in check_components:
        architectures = [
            a.arch_string for a in get_suite_architectures(
                suite_name, skipsrc=True, skipall=True, session=session)
        ]
        for architecture in architectures:
            if component == 'main/debian-installer' and re.match(
                    "kfreebsd", architecture):
                continue
            filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (
                suite.archive.path, suite_name, component, architecture)
            # apt_pkg.TagFile needs a real file handle
            (fd, temp_filename) = utils.temp_filename()
            (result, output) = commands.getstatusoutput(
                "gunzip -c %s > %s" % (filename, temp_filename))
            if (result != 0):
                sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
                sys.exit(result)

            if "nfu" in checks:
                nfu_packages.setdefault(architecture, [])
                nfu_entries = parse_nfu(architecture)

            packages = utils.open_file(temp_filename)
            Packages = apt_pkg.TagFile(packages)
            while Packages.step():
                package = Packages.section.find('Package')
                source = Packages.section.find('Source', "")
                version = Packages.section.find('Version')
                if source == "":
                    source = package
                if bin2source.has_key(package) and \
                       apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
                    bin2source[package]["version"] = version
                    bin2source[package]["source"] = source
                else:
                    bin2source[package] = {}
                    bin2source[package]["version"] = version
                    bin2source[package]["source"] = source
                if source.find("(") != -1:
                    m = re_extract_src_version.match(source)
                    source = m.group(1)
                    version = m.group(2)
                if not bin_pkgs.has_key(package):
                    nbs.setdefault(source, {})
                    nbs[source].setdefault(package, {})
                    nbs[source][package][version] = ""
                else:
                    if "nfu" in checks:
                        if package in nfu_entries and \
                               version != source_versions[source]: # only suggest to remove out-of-date packages
                            nfu_packages[architecture].append(
                                (package, version, source_versions[source]))

            packages.close()
            os.unlink(temp_filename)

    # Distinguish dubious (version numbers match) and 'real' NBS (they don't)
    dubious_nbs = {}
    for source in nbs.keys():
        for package in nbs[source].keys():
            versions = nbs[source][package].keys()
            versions.sort(apt_pkg.version_compare)
            latest_version = versions.pop()
            source_version = source_versions.get(source, "0")
            if apt_pkg.version_compare(latest_version, source_version) == 0:
                add_nbs(dubious_nbs, source, latest_version, package, suite_id,
                        session)

    if "nviu" in checks:
        do_newer_version('chromodoris', 'staging', 'NVIU', session)

    # FIXME: Not used in Tanglu
    #if "nvit" in checks:
    #    do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)

    ###

    if Options["Mode"] == "full":
        print "=" * 75
        print

    if "nfu" in checks:
        do_nfu(nfu_packages)

    if "bnb" in checks:
        print "Unbuilt binary packages"
        print "-----------------------"
        print
        keys = bin_not_built.keys()
        keys.sort()
        for source in keys:
            binaries = bin_not_built[source].keys()
            binaries.sort()
            print " o %s: %s" % (source, ", ".join(binaries))
        print

    if "bms" in checks:
        report_multiple_source(suite)

    if "anais" in checks:
        print "Architecture Not Allowed In Source"
        print "----------------------------------"
        print anais_output
        print

    if "dubious nbs" in checks:
        do_dubious_nbs(dubious_nbs)