示例#1
0
def process_file(file, suite, component, otype, mode, action, session):
    cnf = Config()

    s = get_suite(suite, session=session)
    if s is None:
        utils.fubar("Suite '%s' not recognised." % (suite))
    suite_id = s.suite_id

    c = get_component(component, session=session)
    if c is None:
        utils.fubar("Component '%s' not recognised." % (component))
    component_id = c.component_id

    o = get_override_type(otype)
    if o is None:
        utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (otype))
    type_id = o.overridetype_id

    # --set is done mostly internal for performance reasons; most
    # invocations of --set will be updates and making people wait 2-3
    # minutes while 6000 select+inserts are run needlessly isn't cool.

    original = {}
    new = {}
    c_skipped = 0
    c_added = 0
    c_updated = 0
    c_removed = 0
    c_error = 0

    q = session.execute("""SELECT o.package, o.priority, o.section, o.maintainer, p.priority, s.section
                           FROM override o, priority p, section s
                           WHERE o.suite = :suiteid AND o.component = :componentid AND o.type = :typeid
                             and o.priority = p.id and o.section = s.id""",
                           {'suiteid': suite_id, 'componentid': component_id, 'typeid': type_id})
    for i in q.fetchall():
        original[i[0]] = i[1:]

    start_time = time.time()

    section_cache = get_sections(session)
    priority_cache = get_priorities(session)

    # Our session is already in a transaction

    for line in file.readlines():
        line = re_comments.sub('', line).strip()
        if line == "":
            continue

        maintainer_override = None
        if otype == "dsc":
            split_line = line.split(None, 2)
            if len(split_line) == 2:
                (package, section) = split_line
            elif len(split_line) == 3:
                (package, section, maintainer_override) = split_line
            else:
                utils.warn("'%s' does not break into 'package section [maintainer-override]'." % (line))
                c_error += 1
                continue
            priority = "extra"
        else: # binary or udeb
            split_line = line.split(None, 3)
            if len(split_line) == 3:
                (package, priority, section) = split_line
            elif len(split_line) == 4:
                (package, priority, section, maintainer_override) = split_line
            else:
                utils.warn("'%s' does not break into 'package priority section [maintainer-override]'." % (line))
                c_error += 1
                continue

        if section not in section_cache:
            utils.warn("'%s' is not a valid section. ['%s' in suite %s, component %s]." % (section, package, suite, component))
            c_error += 1
            continue

        section_id = section_cache[section]

        if priority not in priority_cache:
            utils.warn("'%s' is not a valid priority. ['%s' in suite %s, component %s]." % (priority, package, suite, component))
            c_error += 1
            continue

        priority_id = priority_cache[priority]

        if package in new:
            utils.warn("Can't insert duplicate entry for '%s'; ignoring all but the first. [suite %s, component %s]" % (package, suite, component))
            c_error += 1
            continue
        new[package] = ""

        if package in original:
            (old_priority_id, old_section_id, old_maintainer_override, old_priority, old_section) = original[package]
            if mode == "add" or old_priority_id == priority_id and \
               old_section_id == section_id and \
               old_maintainer_override == maintainer_override:
                # If it's unchanged or we're in 'add only' mode, ignore it
                c_skipped += 1
                continue
            else:
                # If it's changed, delete the old one so we can
                # reinsert it with the new information
                c_updated += 1
                if action:
                    session.execute("""DELETE FROM override WHERE suite = :suite AND component = :component
                                                              AND package = :package AND type = :typeid""",
                                    {'suite': suite_id,  'component': component_id,
                                     'package': package, 'typeid': type_id})

                # Log changes
                if old_priority_id != priority_id:
                    Logger.log(["changed priority", package, old_priority, priority])
                if old_section_id != section_id:
                    Logger.log(["changed section", package, old_section, section])
                if old_maintainer_override != maintainer_override:
                    Logger.log(["changed maintainer override", package, old_maintainer_override, maintainer_override])
                update_p = 1
        elif mode == "change":
            # Ignore additions in 'change only' mode
            c_skipped += 1
            continue
        else:
            c_added += 1
            update_p = 0

        if action:
            if not maintainer_override:
                m_o = None
            else:
                m_o = maintainer_override
            session.execute("""INSERT INTO override (suite, component, type, package,
                                                     priority, section, maintainer)
                                             VALUES (:suiteid, :componentid, :typeid,
                                                     :package, :priorityid, :sectionid,
                                                     :maintainer)""",
                              {'suiteid': suite_id, 'componentid': component_id,
                               'typeid':  type_id,  'package': package,
                               'priorityid': priority_id, 'sectionid': section_id,
                               'maintainer': m_o})

        if not update_p:
            Logger.log(["new override", suite, component, otype, package, priority, section, maintainer_override])

    if mode == "set":
        # Delete any packages which were removed
        for package in original.keys():
            if package not in new:
                if action:
                    session.execute("""DELETE FROM override
                                       WHERE suite = :suiteid AND component = :componentid
                                         AND package = :package AND type = :typeid""",
                                    {'suiteid': suite_id, 'componentid': component_id,
                                     'package': package, 'typeid': type_id})
                c_removed += 1
                Logger.log(["removed override", suite, component, otype, package])

    if action:
        session.commit()

    if not cnf["Control-Overrides::Options::Quiet"]:
        print("Done in %d seconds. [Updated = %d, Added = %d, Removed = %d, Skipped = %d, Errors = %d]" % (int(time.time() - start_time), c_updated, c_added, c_removed, c_skipped, c_error))

    Logger.log(["set complete", c_updated, c_added, c_removed, c_skipped, c_error])
示例#2
0
def main():
    cnf = Config()

    Arguments = [('h', "help", "Make-Maintainers::Options::Help"),
                 ('a', "archive", "Make-Maintainers::Options::Archive",
                  'HasArg'),
                 ('s', "source", "Make-Maintainers::Options::Source"),
                 ('p', "print", "Make-Maintainers::Options::Print")]
    for i in ["Help", "Source", "Print"]:
        key = "Make-Maintainers::Options::%s" % i
        if key not in cnf:
            cnf[key] = ""

    extra_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Make-Maintainers::Options")

    if Options["Help"] or not Options.get('Archive'):
        usage()

    Logger = daklog.Logger('make-maintainers')
    session = DBConn().session()

    archive = session.query(Archive).filter_by(
        archive_name=Options['Archive']).one()

    # dictionary packages to maintainer names
    maintainers = dict()
    # dictionary packages to list of uploader names
    uploaders = dict()

    query = session.execute(
        text('''
SELECT
    bs.package,
    bs.name AS maintainer,
    array_agg(mu.name) OVER (
        PARTITION BY bs.source, bs.id
        ORDER BY mu.name
        ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
    ) AS uploaders
    FROM (
        SELECT DISTINCT ON (package)
            *
            FROM (
                SELECT
                    s.id AS source,
                    0 AS id,
                    s.source AS package,
                    s.version,
                    m.name
                    FROM
                        source AS s INNER JOIN
                        maintainer AS m ON s.maintainer = m.id INNER JOIN
                        src_associations AS sa ON s.id = sa.source INNER JOIN
                        suite on sa.suite = suite.id
                    WHERE
                        suite.archive_id = :archive_id
                UNION SELECT
                    b.source,
                    b.id,
                    b.package,
                    b.version,
                    m.name
                    FROM
                        binaries AS b INNER JOIN
                        maintainer AS m ON b.maintainer = m.id INNER JOIN
                        bin_associations AS ba ON b.id = ba.bin INNER JOIN
                        suite on ba.suite = suite.id
                    WHERE
                        NOT :source_only AND
                        suite.archive_id = :archive_id
                ) AS bs
            ORDER BY package, version desc
        ) AS bs LEFT OUTER JOIN
        -- find all uploaders for a given source
        src_uploaders AS su ON bs.source = su.source LEFT OUTER JOIN
        maintainer AS mu ON su.maintainer = mu.id
''').params(archive_id=archive.archive_id,
            source_only="True" if Options["Source"] else "False"))

    Logger.log(['database'])
    for entry in query:
        maintainers[entry['package']] = entry['maintainer']
        if all(x is None for x in entry['uploaders']):
            uploaders[entry['package']] = ['']
        else:
            uploaders[entry['package']] = entry['uploaders']

    Logger.log(['files'])
    # Process any additional Maintainer files (e.g. from pseudo
    # packages)
    for filename in extra_files:
        extrafile = utils.open_file(filename)
        for line in extrafile.readlines():
            line = re_comments.sub('', line).strip()
            if line == "":
                continue
            (package, maintainer) = line.split(None, 1)
            maintainers[package] = maintainer
            uploaders[package] = [maintainer]

    if Options["Print"]:
        for package in sorted(maintainers):
            sys.stdout.write(format(package, maintainers[package]))
    else:
        maintainer_file = open('Maintainers', 'w')
        uploader_file = open('Uploaders', 'w')
        for package in sorted(uploaders):
            maintainer_file.write(format(package, maintainers[package]))
            for uploader in uploaders[package]:
                uploader_file.write(format(package, uploader))
        uploader_file.close()
        maintainer_file.close()

        Logger.close()
示例#3
0
def main():
    cnf = Config()

    Arguments = [('h',"help","Make-Maintainers::Options::Help")]
    if not cnf.has_key("Make-Maintainers::Options::Help"):
        cnf["Make-Maintainers::Options::Help"] = ""

    extra_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Make-Maintainers::Options")

    if Options["Help"]:
        usage()

    Logger = daklog.Logger('make-maintainers')
    session = DBConn().session()

    # dictionary packages to maintainer names
    maintainers = dict()
    # dictionary packages to list of uploader names
    uploaders = dict()

    source_query = session.query(DBSource).from_statement('''
        select distinct on (source) * from source
            order by source, version desc''')

    binary_query = session.query(DBBinary).from_statement('''
        select distinct on (package) * from binaries
            order by package, version desc''')

    Logger.log(['sources'])
    for source in source_query:
        maintainers[source.source] = source.maintainer.name
        uploaders[source.source] = uploader_list(source)

    Logger.log(['binaries'])
    for binary in binary_query:
        if binary.package not in maintainers:
            maintainers[binary.package] = binary.maintainer.name
            uploaders[binary.package] = uploader_list(binary.source)

    Logger.log(['files'])
    # Process any additional Maintainer files (e.g. from pseudo
    # packages)
    for filename in extra_files:
        extrafile = utils.open_file(filename)
        for line in extrafile.readlines():
            line = re_comments.sub('', line).strip()
            if line == "":
                continue
            (package, maintainer) = line.split(None, 1)
            maintainers[package] = maintainer
            uploaders[package] = [maintainer]

    maintainer_file = open('Maintainers', 'w')
    uploader_file = open('Uploaders', 'w')
    for package in sorted(uploaders):
        maintainer_file.write(format(package, maintainers[package]))
        for uploader in uploaders[package]:
            uploader_file.write(format(package, uploader))
    uploader_file.close()
    maintainer_file.close()
    Logger.close()
示例#4
0
def main():
    cnf = Config()

    Arguments = [('h',"help","Make-Maintainers::Options::Help"),
                 ('a',"archive","Make-Maintainers::Options::Archive",'HasArg'),
                 ('s',"source","Make-Maintainers::Options::Source"),
                 ('p',"print","Make-Maintainers::Options::Print")]
    for i in ["Help", "Source", "Print" ]:
        if not cnf.has_key("Make-Maintainers::Options::%s" % (i)):
            cnf["Make-Maintainers::Options::%s" % (i)] = ""

    extra_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Make-Maintainers::Options")

    if Options["Help"] or not Options.get('Archive'):
        usage()

    Logger = daklog.Logger('make-maintainers')
    session = DBConn().session()

    archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()

    # dictionary packages to maintainer names
    maintainers = dict()
    # dictionary packages to list of uploader names
    uploaders = dict()

    source_query = session.query(DBSource).from_statement('''
        select distinct on (source.source) source.* from source
            join src_associations sa on source.id = sa.source
            join suite on sa.suite = suite.id
            where suite.archive_id = :archive_id
            order by source.source, source.version desc''') \
        .params(archive_id=archive.archive_id)

    binary_query = session.query(DBBinary).from_statement('''
        select distinct on (binaries.package) binaries.* from binaries
            join bin_associations ba on binaries.id = ba.bin
            join suite on ba.suite = suite.id
            where suite.archive_id = :archive_id
            order by binaries.package, binaries.version desc''') \
        .params(archive_id=archive.archive_id)

    Logger.log(['sources'])
    for source in source_query:
        maintainers[source.source] = source.maintainer.name
        uploaders[source.source] = uploader_list(source)

    if not Options["Source"]:
        Logger.log(['binaries'])
        for binary in binary_query:
                if binary.package not in maintainers:
                    maintainers[binary.package] = binary.maintainer.name
                    uploaders[binary.package] = uploader_list(binary.source)

    Logger.log(['files'])
    # Process any additional Maintainer files (e.g. from pseudo
    # packages)
    for filename in extra_files:
        extrafile = utils.open_file(filename)
        for line in extrafile.readlines():
            line = re_comments.sub('', line).strip()
            if line == "":
                continue
            (package, maintainer) = line.split(None, 1)
            maintainers[package] = maintainer
            uploaders[package] = [maintainer]

    if Options["Print"]:
        for package in sorted(maintainers):
            sys.stdout.write(format(package, maintainers[package]))
    else:
        maintainer_file = open('Maintainers', 'w')
        uploader_file = open('Uploaders', 'w')
        for package in sorted(uploaders):
            maintainer_file.write(format(package, maintainers[package]))
            for uploader in uploaders[package]:
                uploader_file.write(format(package, uploader))
        uploader_file.close()
        maintainer_file.close()

    Logger.close()
示例#5
0
def main():
    cnf = Config()

    Arguments = [('h', "help", "Make-Maintainers::Options::Help"),
                 ('a', "archive", "Make-Maintainers::Options::Archive", 'HasArg'),
                 ('s', "source", "Make-Maintainers::Options::Source"),
                 ('p', "print", "Make-Maintainers::Options::Print")]
    for i in ["Help", "Source", "Print"]:
        key = "Make-Maintainers::Options::%s" % i
        if key not in cnf:
            cnf[key] = ""

    extra_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("Make-Maintainers::Options")

    if Options["Help"] or not Options.get('Archive'):
        usage()

    Logger = daklog.Logger('make-maintainers')
    session = DBConn().session()

    archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()

    # dictionary packages to maintainer names
    maintainers = dict()
    # dictionary packages to list of uploader names
    uploaders = dict()

    query = session.execute(text('''
SELECT
    bs.package,
    bs.name AS maintainer,
    array_agg(mu.name) OVER (
        PARTITION BY bs.source, bs.id
        ORDER BY mu.name
        ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
    ) AS uploaders
    FROM (
        SELECT DISTINCT ON (package)
            *
            FROM (
                SELECT
                    s.id AS source,
                    0 AS id,
                    s.source AS package,
                    s.version,
                    m.name
                    FROM
                        source AS s INNER JOIN
                        maintainer AS m ON s.maintainer = m.id INNER JOIN
                        src_associations AS sa ON s.id = sa.source INNER JOIN
                        suite on sa.suite = suite.id
                    WHERE
                        suite.archive_id = :archive_id
                UNION SELECT
                    b.source,
                    b.id,
                    b.package,
                    b.version,
                    m.name
                    FROM
                        binaries AS b INNER JOIN
                        maintainer AS m ON b.maintainer = m.id INNER JOIN
                        bin_associations AS ba ON b.id = ba.bin INNER JOIN
                        suite on ba.suite = suite.id
                    WHERE
                        NOT :source_only AND
                        suite.archive_id = :archive_id
                ) AS bs
            ORDER BY package, version desc
        ) AS bs LEFT OUTER JOIN
        -- find all uploaders for a given source
        src_uploaders AS su ON bs.source = su.source LEFT OUTER JOIN
        maintainer AS mu ON su.maintainer = mu.id
''').params(
    archive_id=archive.archive_id,
    source_only="True" if Options["Source"] else "False"
))

    Logger.log(['database'])
    for entry in query:
        maintainers[entry['package']] = entry['maintainer']
        if all(x is None for x in entry['uploaders']):
            uploaders[entry['package']] = ['']
        else:
            uploaders[entry['package']] = entry['uploaders']

    Logger.log(['files'])
    # Process any additional Maintainer files (e.g. from pseudo
    # packages)
    for filename in extra_files:
        extrafile = utils.open_file(filename)
        for line in extrafile.readlines():
            line = re_comments.sub('', line).strip()
            if line == "":
                continue
            (package, maintainer) = line.split(None, 1)
            maintainers[package] = maintainer
            uploaders[package] = [maintainer]

    if Options["Print"]:
        for package in sorted(maintainers):
            print(format(package, maintainers[package]), end='')
    else:
        maintainer_file = open('Maintainers', 'w')
        uploader_file = open('Uploaders', 'w')
        for package in sorted(uploaders):
            maintainer_file.write(format(package, maintainers[package]))
            for uploader in uploaders[package]:
                uploader_file.write(format(package, uploader))
        uploader_file.close()
        maintainer_file.close()

        Logger.close()