Exemplo n.º 1
0
Arquivo: stats.py Projeto: Debian/dak
def number_of_packages():
    arches = {}
    arch_ids = {}
    suites = {}
    suite_ids = {}
    d = {}
    session = DBConn().session()
    # Build up suite mapping
    for i in session.query(Suite).all():
        suites[i.suite_id] = i.suite_name
        suite_ids[i.suite_name] = i.suite_id
    # Build up architecture mapping
    for i in session.query(Architecture).all():
        arches[i.arch_id] = i.arch_string
        arch_ids[i.arch_string] = i.arch_id
    # Pre-create the dictionary
    for suite_id in suites.keys():
        d[suite_id] = {}
        for arch_id in arches.keys():
            d[suite_id][arch_id] = 0
    # Get the raw data for binaries
    # Simultate 'GROUP by suite, architecture' with a dictionary
    # XXX: Why don't we just get the DB to do this?
    for i in session.execute("""SELECT suite, architecture, COUNT(suite)
                                FROM bin_associations
                           LEFT JOIN binaries ON bin = binaries.id
                            GROUP BY suite, architecture""").fetchall():
        d[i[0]][i[1]] = i[2]
    # Get the raw data for source
    arch_id = arch_ids["source"]
    for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall():
        (suite_id, count) = i
        d[suite_id][arch_id] = d[suite_id][arch_id] + count
    ## Print the results
    # Setup
    suite_list = suites.values()
    suite_id_list = []
    suite_arches = {}
    for suite in suite_list:
        suite_id = suite_ids[suite]
        suite_arches[suite_id] = {}
        for arch in get_suite_architectures(suite):
            suite_arches[suite_id][arch.arch_string] = ""
        suite_id_list.append(suite_id)
    output_list = [output_format(i) for i in suite_list]
    longest_suite = longest(output_list)
    arch_list = arches.values()
    arch_list.sort()
    longest_arch = longest(arch_list)
    # Header
    output = (" " * longest_arch) + " |"
    for suite in output_list:
        output = output + suite.center(longest_suite) + " |"
    output = output + "\n" + (len(output) * "-") + "\n"
    # per-arch data
    arch_list = arches.values()
    arch_list.sort()
    longest_arch = longest(arch_list)
    for arch in arch_list:
        arch_id = arch_ids[arch]
        output = output + arch.center(longest_arch) + " |"
        for suite_id in suite_id_list:
            if arch in suite_arches[suite_id]:
                count = "%d" % d[suite_id][arch_id]
            else:
                count = "-"
            output = output + count.rjust(longest_suite) + " |"
        output = output + "\n"
    print(output)
Exemplo n.º 2
0
def number_of_packages():
    arches = {}
    arch_ids = {}
    suites = {}
    suite_ids = {}
    d = {}
    session = DBConn().session()
    # Build up suite mapping
    for i in session.query(Suite).all():
        suites[i.suite_id] = i.suite_name
        suite_ids[i.suite_name] = i.suite_id
    # Build up architecture mapping
    for i in session.query(Architecture).all():
        arches[i.arch_id] = i.arch_string
        arch_ids[i.arch_string] = i.arch_id
    # Pre-create the dictionary
    for suite_id in suites.keys():
        d[suite_id] = {}
        for arch_id in arches.keys():
            d[suite_id][arch_id] = 0
    # Get the raw data for binaries
    # Simultate 'GROUP by suite, architecture' with a dictionary
    # XXX: Why don't we just get the DB to do this?
    for i in session.execute("""SELECT suite, architecture, COUNT(suite)
                                FROM bin_associations
                           LEFT JOIN binaries ON bin = binaries.id
                            GROUP BY suite, architecture""").fetchall():
        d[ i[0] ][ i[1] ] = i[2]
    # Get the raw data for source
    arch_id = arch_ids["source"]
    for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall():
        (suite_id, count) = i
        d[suite_id][arch_id] = d[suite_id][arch_id] + count
    ## Print the results
    # Setup
    suite_list = suites.values()
    suite_id_list = []
    suite_arches = {}
    for suite in suite_list:
        suite_id = suite_ids[suite]
        suite_arches[suite_id] = {}
        for arch in get_suite_architectures(suite):
            suite_arches[suite_id][arch.arch_string] = ""
        suite_id_list.append(suite_id)
    output_list = [ output_format(i) for i in suite_list ]
    longest_suite = longest(output_list)
    arch_list = arches.values()
    arch_list.sort()
    longest_arch = longest(arch_list)
    # Header
    output = (" "*longest_arch) + " |"
    for suite in output_list:
        output = output + suite.center(longest_suite)+" |"
    output = output + "\n"+(len(output)*"-")+"\n"
    # per-arch data
    arch_list = arches.values()
    arch_list.sort()
    longest_arch = longest(arch_list)
    for arch in arch_list:
        arch_id = arch_ids[arch]
        output = output + arch.center(longest_arch)+" |"
        for suite_id in suite_id_list:
            if suite_arches[suite_id].has_key(arch):
                count = "%d" % d[suite_id][arch_id]
            else:
                count = "-"
            output = output + count.rjust(longest_suite)+" |"
        output = output + "\n"
    print output
Exemplo n.º 3
0
def main():
    global Cnf, Options, Logger

    os.umask(0o002)

    Cnf = utils.get_conf()
    Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"),
                  ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
                  ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
                  ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
                  ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
                  ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
                  ('n', "no-act", "Generate-Index-Diffs::Options::NoAct"),
                  ('v', "verbose", "Generate-Index-Diffs::Options::Verbose"),
                ]
    suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
    Options = Cnf.subtree("Generate-Index-Diffs::Options")
    if "Help" in Options:
        usage()

    maxdiffs = Options.get("MaxDiffs::Default", "56")
    maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
    maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
    maxsources = Options.get("MaxDiffs::Sources", maxdiffs)

    # can only be set via config at the moment
    max_parallel = int(Options.get("MaxParallel", "8"))

    if "PatchName" not in Options:
        format = "%Y-%m-%d-%H%M.%S"
        Options["PatchName"] = time.strftime(format)

    session = DBConn().session()
    pending_tasks = []

    if not suites:
        query = session.query(Suite.suite_name)
        if Options.get('Archive'):
            archives = utils.split_args(Options['Archive'])
            query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
        suites = [s.suite_name for s in query]

    for suitename in suites:
        print("Processing: " + suitename)

        suiteobj = get_suite(suitename.lower(), session=session)

        # Use the canonical version of the suite name
        suite = suiteobj.suite_name

        if suiteobj.untouchable:
            print("Skipping: " + suite + " (untouchable)")
            continue

        skip_all = True
        if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all:
            skip_all = False

        architectures = get_suite_architectures(suite, skipall=skip_all, session=session)
        components = [c.component_name for c in session.query(Component.component_name)]

        suite_suffix = utils.suite_suffix(suitename)
        if components and suite_suffix:
            longsuite = suite + "/" + suite_suffix
        else:
            longsuite = suite

        merged_pdiffs = suiteobj.merged_pdiffs

        tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)

        # See if there are Translations which might need a new pdiff
        cwd = os.getcwd()
        for component in components:
            workpath = os.path.join(tree, component, "i18n")
            if os.path.isdir(workpath):
                os.chdir(workpath)
                for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
                    for entry in filenames:
                        if not re_includeinpdiff.match(entry):
                            continue
                        (fname, fext) = os.path.splitext(entry)
                        processfile = os.path.join(workpath, fname)
                        storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
                        coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs)
                        pending_tasks.append(coroutine)
        os.chdir(cwd)

        for archobj in architectures:
            architecture = archobj.arch_string

            if architecture == "source":
                longarch = architecture
                packages = "Sources"
                maxsuite = maxsources
            else:
                longarch = "binary-%s" % architecture
                packages = "Packages"
                maxsuite = maxpackages

            for component in components:
                # Process Contents
                file = "%s/%s/Contents-%s" % (tree, component, architecture)

                storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
                coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs)
                pending_tasks.append(coroutine)

                file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
                storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
                coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs)
                pending_tasks.append(coroutine)

    asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
Exemplo n.º 4
0
def main():
    global Cnf, Options, Logger

    os.umask(0o002)

    Cnf = utils.get_conf()
    Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
                  ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
                  ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
                  ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
                  ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
                  ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
                  ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
                ]
    suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
    Options = Cnf.subtree("Generate-Index-Diffs::Options")
    if Options.has_key("Help"): usage()

    maxdiffs = Options.get("MaxDiffs::Default", "56")
    maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
    maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
    maxsources = Options.get("MaxDiffs::Sources", maxdiffs)

    if not Options.has_key("PatchName"):
        format = "%Y-%m-%d-%H%M.%S"
        Options["PatchName"] = time.strftime( format )

    session = DBConn().session()

    if not suites:
        query = session.query(Suite.suite_name)
        if Options.get('Archive'):
            query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive'])
        suites = [ s.suite_name for s in query ]

    for suitename in suites:
        print "Processing: " + suitename

        suiteobj = get_suite(suitename.lower(), session=session)

        # Use the canonical version of the suite name
        suite = suiteobj.suite_name

        if suiteobj.untouchable:
            print "Skipping: " + suite + " (untouchable)"
            continue

        architectures = get_suite_architectures(suite, skipall=True, session=session)
        components = [ c.component_name for c in session.query(Component.component_name) ]

        suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
        if components and suite_suffix:
            longsuite = suite + "/" + suite_suffix
        else:
            longsuite = suite

        tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)

        # See if there are Translations which might need a new pdiff
        cwd = os.getcwd()
        for component in components:
            #print "DEBUG: Working on %s" % (component)
            workpath=os.path.join(tree, component, "i18n")
            if os.path.isdir(workpath):
                os.chdir(workpath)
                for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
                    for entry in filenames:
                        if not re_includeinpdiff.match(entry):
                            #print "EXCLUDING %s" % (entry)
                            continue
                        (fname, fext) = os.path.splitext(entry)
                        processfile=os.path.join(workpath, fname)
                        #print "Working: %s" % (processfile)
                        storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
                        #print "Storefile: %s" % (storename)
                        genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
        os.chdir(cwd)

        for archobj in architectures:
            architecture = archobj.arch_string

            for component in components:
                if architecture == "source":
                    longarch = architecture
                    packages = "Sources"
                    maxsuite = maxsources
                else:
                    longarch = "binary-%s"% (architecture)
                    packages = "Packages"
                    maxsuite = maxpackages
                    # Process Contents
                    file = "%s/%s/Contents-%s" % (tree, component, architecture)
                    storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
                    genchanges(Options, file + ".diff", storename, file, maxcontents)

                file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
                storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
                genchanges(Options, file + ".diff", storename, file, maxsuite)
Exemplo n.º 5
0
def main():
    global Cnf, Options, Logger

    os.umask(0o002)

    Cnf = utils.get_conf()
    Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
                  ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
                  ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
                  ('r', "rootdir", "Generate-Index-Diffs::Options::RootDir", "hasArg"),
                  ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
                  ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
                  ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
                ]
    suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
    Options = Cnf.subtree("Generate-Index-Diffs::Options")
    if Options.has_key("Help"): usage()

    maxdiffs = Options.get("MaxDiffs::Default", "56")
    maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
    maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
    maxsources = Options.get("MaxDiffs::Sources", maxdiffs)

    if not Options.has_key("PatchName"):
        format = "%Y-%m-%d-%H%M.%S"
        Options["PatchName"] = time.strftime( format )

    AptCnf = apt_pkg.newConfiguration()
    apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file())

    if Options.has_key("RootDir"):
        Cnf["Dir::Root"] = Options["RootDir"]

    if not suites:
        suites = Cnf.subtree("Suite").list()

    for suitename in suites:
        print "Processing: " + suitename
        SuiteBlock = Cnf.subtree("Suite::" + suitename)

        suiteobj = get_suite(suitename.lower())

        # Use the canonical version of the suite name
        suite = suiteobj.suite_name

        if suiteobj.untouchable:
            print "Skipping: " + suite + " (untouchable)"
            continue

        architectures = get_suite_architectures(suite, skipall=True)

        if SuiteBlock.has_key("Components"):
            components = SuiteBlock.value_list("Components")
        else:
            components = []

        suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
        if components and suite_suffix:
            longsuite = suite + "/" + suite_suffix
        else:
            longsuite = suite

        tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite))

        if AptCnf.has_key("tree::%s" % (tree)):
            sections = AptCnf["tree::%s::Sections" % (tree)].split()
        elif AptCnf.has_key("bindirectory::%s" % (tree)):
            sections = AptCnf["bindirectory::%s::Sections" % (tree)].split()
        else:
            aptcnf_filename = os.path.basename(utils.which_apt_conf_file())
            print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename)
            continue

        # See if there are Translations which might need a new pdiff
        cwd = os.getcwd()
        for component in sections:
            #print "DEBUG: Working on %s" % (component)
            workpath=os.path.join(Cnf["Dir::Root"], tree, component, "i18n")
            if os.path.isdir(workpath):
                os.chdir(workpath)
                for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
                    for entry in filenames:
                        if not re_includeinpdiff.match(entry):
                            #print "EXCLUDING %s" % (entry)
                            continue
                        (fname, fext) = os.path.splitext(entry)
                        processfile=os.path.join(workpath, fname)
                        #print "Working: %s" % (processfile)
                        storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
                        #print "Storefile: %s" % (storename)
                        genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
        os.chdir(cwd)

        for archobj in architectures:
            architecture = archobj.arch_string

            # use sections instead of components since dak.conf
            # treats "foo/bar main" as suite "foo", suitesuffix "bar" and
            # component "bar/main". suck.

            for component in sections:
                if architecture == "source":
                    longarch = architecture
                    packages = "Sources"
                    maxsuite = maxsources
                else:
                    longarch = "binary-%s"% (architecture)
                    packages = "Packages"
                    maxsuite = maxpackages
                    # Process Contents
                    file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component,
                            architecture)
                    storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
                    genchanges(Options, file + ".diff", storename, file, \
                      Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents))

                file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree,
                           component, longarch, packages)
                storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
                genchanges(Options, file + ".diff", storename, file, \
                  Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::%s" % (suite, packages), maxsuite))