def number_of_packages():
arches = {}
arch_ids = {}
suites = {}
suite_ids = {}
d = {}
session = DBConn().session()
# Build up suite mapping
for i in session.query(Suite).all():
suites[i.suite_id] = i.suite_name
suite_ids[i.suite_name] = i.suite_id
# Build up architecture mapping
for i in session.query(Architecture).all():
arches[i.arch_id] = i.arch_string
arch_ids[i.arch_string] = i.arch_id
# Pre-create the dictionary
for suite_id in suites.keys():
d[suite_id] = {}
for arch_id in arches.keys():
d[suite_id][arch_id] = 0
# Get the raw data for binaries
# Simultate 'GROUP by suite, architecture' with a dictionary
# XXX: Why don't we just get the DB to do this?
for i in session.execute("""SELECT suite, architecture, COUNT(suite)
FROM bin_associations
LEFT JOIN binaries ON bin = binaries.id
GROUP BY suite, architecture""").fetchall():
d[i[0]][i[1]] = i[2]
# Get the raw data for source
arch_id = arch_ids["source"]
for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall():
(suite_id, count) = i
d[suite_id][arch_id] = d[suite_id][arch_id] + count
## Print the results
# Setup
suite_list = suites.values()
suite_id_list = []
suite_arches = {}
for suite in suite_list:
suite_id = suite_ids[suite]
suite_arches[suite_id] = {}
for arch in get_suite_architectures(suite):
suite_arches[suite_id][arch.arch_string] = ""
suite_id_list.append(suite_id)
output_list = [output_format(i) for i in suite_list]
longest_suite = longest(output_list)
arch_list = arches.values()
arch_list.sort()
longest_arch = longest(arch_list)
# Header
output = (" " * longest_arch) + " |"
for suite in output_list:
output = output + suite.center(longest_suite) + " |"
output = output + "\n" + (len(output) * "-") + "\n"
# per-arch data
arch_list = arches.values()
arch_list.sort()
longest_arch = longest(arch_list)
for arch in arch_list:
arch_id = arch_ids[arch]
output = output + arch.center(longest_arch) + " |"
for suite_id in suite_id_list:
if arch in suite_arches[suite_id]:
count = "%d" % d[suite_id][arch_id]
else:
count = "-"
output = output + count.rjust(longest_suite) + " |"
output = output + "\n"
print(output)
def number_of_packages():
arches = {}
arch_ids = {}
suites = {}
suite_ids = {}
d = {}
session = DBConn().session()
# Build up suite mapping
for i in session.query(Suite).all():
suites[i.suite_id] = i.suite_name
suite_ids[i.suite_name] = i.suite_id
# Build up architecture mapping
for i in session.query(Architecture).all():
arches[i.arch_id] = i.arch_string
arch_ids[i.arch_string] = i.arch_id
# Pre-create the dictionary
for suite_id in suites.keys():
d[suite_id] = {}
for arch_id in arches.keys():
d[suite_id][arch_id] = 0
# Get the raw data for binaries
# Simultate 'GROUP by suite, architecture' with a dictionary
# XXX: Why don't we just get the DB to do this?
for i in session.execute("""SELECT suite, architecture, COUNT(suite)
FROM bin_associations
LEFT JOIN binaries ON bin = binaries.id
GROUP BY suite, architecture""").fetchall():
d[ i[0] ][ i[1] ] = i[2]
# Get the raw data for source
arch_id = arch_ids["source"]
for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall():
(suite_id, count) = i
d[suite_id][arch_id] = d[suite_id][arch_id] + count
## Print the results
# Setup
suite_list = suites.values()
suite_id_list = []
suite_arches = {}
for suite in suite_list:
suite_id = suite_ids[suite]
suite_arches[suite_id] = {}
for arch in get_suite_architectures(suite):
suite_arches[suite_id][arch.arch_string] = ""
suite_id_list.append(suite_id)
output_list = [ output_format(i) for i in suite_list ]
longest_suite = longest(output_list)
arch_list = arches.values()
arch_list.sort()
longest_arch = longest(arch_list)
# Header
output = (" "*longest_arch) + " |"
for suite in output_list:
output = output + suite.center(longest_suite)+" |"
output = output + "\n"+(len(output)*"-")+"\n"
# per-arch data
arch_list = arches.values()
arch_list.sort()
longest_arch = longest(arch_list)
for arch in arch_list:
arch_id = arch_ids[arch]
output = output + arch.center(longest_arch)+" |"
for suite_id in suite_id_list:
if suite_arches[suite_id].has_key(arch):
count = "%d" % d[suite_id][arch_id]
else:
count = "-"
output = output + count.rjust(longest_suite)+" |"
output = output + "\n"
print output
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "no-act", "Generate-Index-Diffs::Options::NoAct"),
('v', "verbose", "Generate-Index-Diffs::Options::Verbose"),
]
suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if "Help" in Options:
usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
# can only be set via config at the moment
max_parallel = int(Options.get("MaxParallel", "8"))
if "PatchName" not in Options:
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime(format)
session = DBConn().session()
pending_tasks = []
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
archives = utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
suites = [s.suite_name for s in query]
for suitename in suites:
print("Processing: " + suitename)
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print("Skipping: " + suite + " (untouchable)")
continue
skip_all = True
if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all:
skip_all = False
architectures = get_suite_architectures(suite, skipall=skip_all, session=session)
components = [c.component_name for c in session.query(Component.component_name)]
suite_suffix = utils.suite_suffix(suitename)
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
merged_pdiffs = suiteobj.merged_pdiffs
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
workpath = os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
continue
(fname, fext) = os.path.splitext(entry)
processfile = os.path.join(workpath, fname)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs)
pending_tasks.append(coroutine)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s" % architecture
packages = "Packages"
maxsuite = maxpackages
for component in components:
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs)
pending_tasks.append(coroutine)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs)
pending_tasks.append(coroutine)
asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
]
suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if Options.has_key("Help"): usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
if not Options.has_key("PatchName"):
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime( format )
session = DBConn().session()
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive'])
suites = [ s.suite_name for s in query ]
for suitename in suites:
print "Processing: " + suitename
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
architectures = get_suite_architectures(suite, skipall=True, session=session)
components = [ c.component_name for c in session.query(Component.component_name) ]
suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
#print "DEBUG: Working on %s" % (component)
workpath=os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
#print "EXCLUDING %s" % (entry)
continue
(fname, fext) = os.path.splitext(entry)
processfile=os.path.join(workpath, fname)
#print "Working: %s" % (processfile)
storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
#print "Storefile: %s" % (storename)
genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
for component in components:
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s"% (architecture)
packages = "Packages"
maxsuite = maxpackages
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxcontents)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxsuite)
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('r', "rootdir", "Generate-Index-Diffs::Options::RootDir", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
]
suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if Options.has_key("Help"): usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
if not Options.has_key("PatchName"):
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime( format )
AptCnf = apt_pkg.newConfiguration()
apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file())
if Options.has_key("RootDir"):
Cnf["Dir::Root"] = Options["RootDir"]
if not suites:
suites = Cnf.subtree("Suite").list()
for suitename in suites:
print "Processing: " + suitename
SuiteBlock = Cnf.subtree("Suite::" + suitename)
suiteobj = get_suite(suitename.lower())
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
architectures = get_suite_architectures(suite, skipall=True)
if SuiteBlock.has_key("Components"):
components = SuiteBlock.value_list("Components")
else:
components = []
suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite))
if AptCnf.has_key("tree::%s" % (tree)):
sections = AptCnf["tree::%s::Sections" % (tree)].split()
elif AptCnf.has_key("bindirectory::%s" % (tree)):
sections = AptCnf["bindirectory::%s::Sections" % (tree)].split()
else:
aptcnf_filename = os.path.basename(utils.which_apt_conf_file())
print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename)
continue
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in sections:
#print "DEBUG: Working on %s" % (component)
workpath=os.path.join(Cnf["Dir::Root"], tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
#print "EXCLUDING %s" % (entry)
continue
(fname, fext) = os.path.splitext(entry)
processfile=os.path.join(workpath, fname)
#print "Working: %s" % (processfile)
storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
#print "Storefile: %s" % (storename)
genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
# use sections instead of components since dak.conf
# treats "foo/bar main" as suite "foo", suitesuffix "bar" and
# component "bar/main". suck.
for component in sections:
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s"% (architecture)
packages = "Packages"
maxsuite = maxpackages
# Process Contents
file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component,
architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, \
Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents))
file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree,
component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, \
Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::%s" % (suite, packages), maxsuite))