def number_of_packages(): arches = {} arch_ids = {} suites = {} suite_ids = {} d = {} session = DBConn().session() # Build up suite mapping for i in session.query(Suite).all(): suites[i.suite_id] = i.suite_name suite_ids[i.suite_name] = i.suite_id # Build up architecture mapping for i in session.query(Architecture).all(): arches[i.arch_id] = i.arch_string arch_ids[i.arch_string] = i.arch_id # Pre-create the dictionary for suite_id in suites.keys(): d[suite_id] = {} for arch_id in arches.keys(): d[suite_id][arch_id] = 0 # Get the raw data for binaries # Simultate 'GROUP by suite, architecture' with a dictionary # XXX: Why don't we just get the DB to do this? for i in session.execute("""SELECT suite, architecture, COUNT(suite) FROM bin_associations LEFT JOIN binaries ON bin = binaries.id GROUP BY suite, architecture""").fetchall(): d[i[0]][i[1]] = i[2] # Get the raw data for source arch_id = arch_ids["source"] for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall(): (suite_id, count) = i d[suite_id][arch_id] = d[suite_id][arch_id] + count ## Print the results # Setup suite_list = suites.values() suite_id_list = [] suite_arches = {} for suite in suite_list: suite_id = suite_ids[suite] suite_arches[suite_id] = {} for arch in get_suite_architectures(suite): suite_arches[suite_id][arch.arch_string] = "" suite_id_list.append(suite_id) output_list = [output_format(i) for i in suite_list] longest_suite = longest(output_list) arch_list = arches.values() arch_list.sort() longest_arch = longest(arch_list) # Header output = (" " * longest_arch) + " |" for suite in output_list: output = output + suite.center(longest_suite) + " |" output = output + "\n" + (len(output) * "-") + "\n" # per-arch data arch_list = arches.values() arch_list.sort() longest_arch = longest(arch_list) for arch in arch_list: arch_id = arch_ids[arch] output = output + arch.center(longest_arch) + " |" for suite_id in suite_id_list: if arch in suite_arches[suite_id]: count = "%d" % d[suite_id][arch_id] else: count = "-" output = output + count.rjust(longest_suite) + " |" output = output + "\n" print(output)
def number_of_packages(): arches = {} arch_ids = {} suites = {} suite_ids = {} d = {} session = DBConn().session() # Build up suite mapping for i in session.query(Suite).all(): suites[i.suite_id] = i.suite_name suite_ids[i.suite_name] = i.suite_id # Build up architecture mapping for i in session.query(Architecture).all(): arches[i.arch_id] = i.arch_string arch_ids[i.arch_string] = i.arch_id # Pre-create the dictionary for suite_id in suites.keys(): d[suite_id] = {} for arch_id in arches.keys(): d[suite_id][arch_id] = 0 # Get the raw data for binaries # Simultate 'GROUP by suite, architecture' with a dictionary # XXX: Why don't we just get the DB to do this? for i in session.execute("""SELECT suite, architecture, COUNT(suite) FROM bin_associations LEFT JOIN binaries ON bin = binaries.id GROUP BY suite, architecture""").fetchall(): d[ i[0] ][ i[1] ] = i[2] # Get the raw data for source arch_id = arch_ids["source"] for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall(): (suite_id, count) = i d[suite_id][arch_id] = d[suite_id][arch_id] + count ## Print the results # Setup suite_list = suites.values() suite_id_list = [] suite_arches = {} for suite in suite_list: suite_id = suite_ids[suite] suite_arches[suite_id] = {} for arch in get_suite_architectures(suite): suite_arches[suite_id][arch.arch_string] = "" suite_id_list.append(suite_id) output_list = [ output_format(i) for i in suite_list ] longest_suite = longest(output_list) arch_list = arches.values() arch_list.sort() longest_arch = longest(arch_list) # Header output = (" "*longest_arch) + " |" for suite in output_list: output = output + suite.center(longest_suite)+" |" output = output + "\n"+(len(output)*"-")+"\n" # per-arch data arch_list = arches.values() arch_list.sort() longest_arch = longest(arch_list) for arch in arch_list: arch_id = arch_ids[arch] output = output + arch.center(longest_arch)+" |" for suite_id in suite_id_list: if suite_arches[suite_id].has_key(arch): count = "%d" % d[suite_id][arch_id] else: count = "-" output = output + count.rjust(longest_suite)+" |" output = output + "\n" print output
def main(): global Cnf, Options, Logger os.umask(0o002) Cnf = utils.get_conf() Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"), ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'), ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"), ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"), ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"), ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), ('n', "no-act", "Generate-Index-Diffs::Options::NoAct"), ('v', "verbose", "Generate-Index-Diffs::Options::Verbose"), ] suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Generate-Index-Diffs::Options") if "Help" in Options: usage() maxdiffs = Options.get("MaxDiffs::Default", "56") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) maxsources = Options.get("MaxDiffs::Sources", maxdiffs) # can only be set via config at the moment max_parallel = int(Options.get("MaxParallel", "8")) if "PatchName" not in Options: format = "%Y-%m-%d-%H%M.%S" Options["PatchName"] = time.strftime(format) session = DBConn().session() pending_tasks = [] if not suites: query = session.query(Suite.suite_name) if Options.get('Archive'): archives = utils.split_args(Options['Archive']) query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives)) suites = [s.suite_name for s in query] for suitename in suites: print("Processing: " + suitename) suiteobj = get_suite(suitename.lower(), session=session) # Use the canonical version of the suite name suite = suiteobj.suite_name if suiteobj.untouchable: print("Skipping: " + suite + " (untouchable)") continue skip_all = True if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all: skip_all = False architectures = get_suite_architectures(suite, skipall=skip_all, session=session) components = [c.component_name for c in session.query(Component.component_name)] suite_suffix = utils.suite_suffix(suitename) if components and suite_suffix: longsuite = suite + "/" + suite_suffix else: longsuite = suite merged_pdiffs = suiteobj.merged_pdiffs tree = os.path.join(suiteobj.archive.path, 'dists', longsuite) # See if there are Translations which might need a new pdiff cwd = os.getcwd() for component in components: workpath = os.path.join(tree, component, "i18n") if os.path.isdir(workpath): os.chdir(workpath) for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): for entry in filenames: if not re_includeinpdiff.match(entry): continue (fname, fext) = os.path.splitext(entry) processfile = os.path.join(workpath, fname) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname) coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs) pending_tasks.append(coroutine) os.chdir(cwd) for archobj in architectures: architecture = archobj.arch_string if architecture == "source": longarch = architecture packages = "Sources" maxsuite = maxsources else: longarch = "binary-%s" % architecture packages = "Packages" maxsuite = maxpackages for component in components: # Process Contents file = "%s/%s/Contents-%s" % (tree, component, architecture) storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs) pending_tasks.append(coroutine) file = "%s/%s/%s/%s" % (tree, component, longarch, packages) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture) coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs) pending_tasks.append(coroutine) asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
def main(): global Cnf, Options, Logger os.umask(0o002) Cnf = utils.get_conf() Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"), ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'), ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"), ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"), ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"), ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"), ] suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv) Options = Cnf.subtree("Generate-Index-Diffs::Options") if Options.has_key("Help"): usage() maxdiffs = Options.get("MaxDiffs::Default", "56") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) maxsources = Options.get("MaxDiffs::Sources", maxdiffs) if not Options.has_key("PatchName"): format = "%Y-%m-%d-%H%M.%S" Options["PatchName"] = time.strftime( format ) session = DBConn().session() if not suites: query = session.query(Suite.suite_name) if Options.get('Archive'): query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive']) suites = [ s.suite_name for s in query ] for suitename in suites: print "Processing: " + suitename suiteobj = get_suite(suitename.lower(), session=session) # Use the canonical version of the suite name suite = suiteobj.suite_name if suiteobj.untouchable: print "Skipping: " + suite + " (untouchable)" continue architectures = get_suite_architectures(suite, skipall=True, session=session) components = [ c.component_name for c in session.query(Component.component_name) ] suite_suffix = Cnf.find("Dinstall::SuiteSuffix") if components and suite_suffix: longsuite = suite + "/" + suite_suffix else: longsuite = suite tree = os.path.join(suiteobj.archive.path, 'dists', longsuite) # See if there are Translations which might need a new pdiff cwd = os.getcwd() for component in components: #print "DEBUG: Working on %s" % (component) workpath=os.path.join(tree, component, "i18n") if os.path.isdir(workpath): os.chdir(workpath) for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): for entry in filenames: if not re_includeinpdiff.match(entry): #print "EXCLUDING %s" % (entry) continue (fname, fext) = os.path.splitext(entry) processfile=os.path.join(workpath, fname) #print "Working: %s" % (processfile) storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname) #print "Storefile: %s" % (storename) genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs) os.chdir(cwd) for archobj in architectures: architecture = archobj.arch_string for component in components: if architecture == "source": longarch = architecture packages = "Sources" maxsuite = maxsources else: longarch = "binary-%s"% (architecture) packages = "Packages" maxsuite = maxpackages # Process Contents file = "%s/%s/Contents-%s" % (tree, component, architecture) storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) genchanges(Options, file + ".diff", storename, file, maxcontents) file = "%s/%s/%s/%s" % (tree, component, longarch, packages) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture) genchanges(Options, file + ".diff", storename, file, maxsuite)
def main(): global Cnf, Options, Logger os.umask(0o002) Cnf = utils.get_conf() Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"), ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"), ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"), ('r', "rootdir", "Generate-Index-Diffs::Options::RootDir", "hasArg"), ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"), ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"), ] suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv) Options = Cnf.subtree("Generate-Index-Diffs::Options") if Options.has_key("Help"): usage() maxdiffs = Options.get("MaxDiffs::Default", "56") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) maxsources = Options.get("MaxDiffs::Sources", maxdiffs) if not Options.has_key("PatchName"): format = "%Y-%m-%d-%H%M.%S" Options["PatchName"] = time.strftime( format ) AptCnf = apt_pkg.newConfiguration() apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file()) if Options.has_key("RootDir"): Cnf["Dir::Root"] = Options["RootDir"] if not suites: suites = Cnf.subtree("Suite").list() for suitename in suites: print "Processing: " + suitename SuiteBlock = Cnf.subtree("Suite::" + suitename) suiteobj = get_suite(suitename.lower()) # Use the canonical version of the suite name suite = suiteobj.suite_name if suiteobj.untouchable: print "Skipping: " + suite + " (untouchable)" continue architectures = get_suite_architectures(suite, skipall=True) if SuiteBlock.has_key("Components"): components = SuiteBlock.value_list("Components") else: components = [] suite_suffix = Cnf.find("Dinstall::SuiteSuffix") if components and suite_suffix: longsuite = suite + "/" + suite_suffix else: longsuite = suite tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite)) if AptCnf.has_key("tree::%s" % (tree)): sections = AptCnf["tree::%s::Sections" % (tree)].split() elif AptCnf.has_key("bindirectory::%s" % (tree)): sections = AptCnf["bindirectory::%s::Sections" % (tree)].split() else: aptcnf_filename = os.path.basename(utils.which_apt_conf_file()) print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename) continue # See if there are Translations which might need a new pdiff cwd = os.getcwd() for component in sections: #print "DEBUG: Working on %s" % (component) workpath=os.path.join(Cnf["Dir::Root"], tree, component, "i18n") if os.path.isdir(workpath): os.chdir(workpath) for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): for entry in filenames: if not re_includeinpdiff.match(entry): #print "EXCLUDING %s" % (entry) continue (fname, fext) = os.path.splitext(entry) processfile=os.path.join(workpath, fname) #print "Working: %s" % (processfile) storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname) #print "Storefile: %s" % (storename) genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs) os.chdir(cwd) for archobj in architectures: architecture = archobj.arch_string # use sections instead of components since dak.conf # treats "foo/bar main" as suite "foo", suitesuffix "bar" and # component "bar/main". suck. for component in sections: if architecture == "source": longarch = architecture packages = "Sources" maxsuite = maxsources else: longarch = "binary-%s"% (architecture) packages = "Packages" maxsuite = maxpackages # Process Contents file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component, architecture) storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) genchanges(Options, file + ".diff", storename, file, \ Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents)) file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree, component, longarch, packages) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture) genchanges(Options, file + ".diff", storename, file, \ Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::%s" % (suite, packages), maxsuite))