def get_defaultsuite(self):
from daklib.dbconn import get_suite
suitename = self.defaultsuitename
if not suitename:
return None
else:
return get_suite(suitename)
def get_defaultsuite(self):
from daklib.dbconn import get_suite
suitename = self.defaultsuitename
if not suitename:
return None
else:
return get_suite(suitename)
def main():
cnf = Config()
Arguments = [('h',"help","DEP11::Options::Help"),
('s',"suite","DEP11::Options::Suite", "HasArg"),
('e',"expire","DEP11::Options::ExpireCache"),
('h',"write-hints","DEP11::Options::WriteHints"),
]
for i in ["help", "suite", "ExpireCache"]:
if not cnf.has_key("DEP11::Options::%s" % (i)):
cnf["DEP11::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("DEP11::Options")
if Options["Help"]:
usage()
return
suitename = Options["Suite"]
if not suitename:
print("You need to specify a suite!")
sys.exit(1)
# check if we have some important config options set
if not cnf.has_key("Dir::MetaInfo"):
print("You need to specify a metadata export directory (Dir::MetaInfo)")
sys.exit(1)
if not cnf.has_key("DEP11::Url"):
print("You need to specify a metadata public web URL (DEP11::Url)")
sys.exit(1)
if not cnf.has_key("DEP11::IconSizes"):
print("You need to specify a list of allowed icon-sizes (DEP11::IconSizes)")
sys.exit(1)
if Options["WriteHints"] and not cnf.has_key("Dir::MetaInfoHints"):
print("You need to specify an export directory for DEP-11 hints files (Dir::MetaInfoHints)")
sys.exit(1)
logger = daklog.Logger('generate-metadata')
from daklib.dbconn import Component, DBConn, get_suite, Suite
session = DBConn().session()
suite = get_suite(suitename.lower(), session)
if Options["ExpireCache"]:
expire_dep11_data_cache(session, suitename, logger)
process_suite(session, suite, logger)
# export database content as Components-<arch>.xz YAML documents
write_component_files(session, suite, logger)
if Options["WriteHints"]:
write_hints_files(session, suite, logger)
# we're done
logger.close()
def main():
cnf = Config()
Arguments = [
('h', "help", "DEP11::Options::Help"),
('e', "expire", "DEP11::Options::ExpireCache"),
('s', "suite", "DEP11::Options::Suite", "HasArg"),
]
for i in ["help", "suite", "ExpireCache"]:
if not cnf.has_key("DEP11::Options::%s" % (i)):
cnf["DEP11::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("DEP11::Options")
if Options["Help"]:
usage()
return
suitename = Options["Suite"]
if not suitename:
print("You need to specify a suite!")
return
logger = daklog.Logger('generate-metadata')
from daklib.dbconn import Component, DBConn, get_suite, Suite
session = DBConn().session()
suite = get_suite(suitename.lower(), session)
if Options["ExpireCache"]:
expire_dep11_data_cache(session, suitename)
process_suite(session, suite, logger)
# export database content as Components-<arch>.xz YAML documents
write_component_files(suite)
# we're done
logger.close()
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
]
suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if Options.has_key("Help"): usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
if not Options.has_key("PatchName"):
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime( format )
session = DBConn().session()
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive'])
suites = [ s.suite_name for s in query ]
for suitename in suites:
print "Processing: " + suitename
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
architectures = get_suite_architectures(suite, skipall=True, session=session)
components = [ c.component_name for c in session.query(Component.component_name) ]
suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
#print "DEBUG: Working on %s" % (component)
workpath=os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
#print "EXCLUDING %s" % (entry)
continue
(fname, fext) = os.path.splitext(entry)
processfile=os.path.join(workpath, fname)
#print "Working: %s" % (processfile)
storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
#print "Storefile: %s" % (storename)
genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
for component in components:
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s"% (architecture)
packages = "Packages"
maxsuite = maxpackages
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxcontents)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxsuite)
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h', "help", "Generate-Packages-Sources::Options::Help"),
('a', 'archive',
'Generate-Packages-Sources::Options::Archive', 'HasArg'),
('s', "suite", "Generate-Packages-Sources::Options::Suite",
'HasArg'),
('f', "force", "Generate-Packages-Sources::Options::Force"),
('o', 'option', '', 'ArbItem')]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
import daklib.utils
if Options.has_key("Suite"):
suites = []
suite_names = daklib.utils.split_args(Options['Suite'])
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
archive_names = daklib.utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(
Archive.archive_name.in_(archive_names))
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log(msg)
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess received signal ', msg])
else:
logger.log(['E: ', msg])
# Lock tables so that nobody can change things underneath us
session.execute("LOCK TABLE src_associations IN SHARE MODE")
session.execute("LOCK TABLE bin_associations IN SHARE MODE")
for s in suites:
component_ids = [c.component_id for c in s.components]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar(
"Refusing to touch %s (untouchable and not forced)" %
s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c],
callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c],
callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages,
[s.suite_id, c, a.arch_id, 'deb'],
callback=parse_results)
pool.apply_async(generate_packages,
[s.suite_id, c, a.arch_id, 'udeb'],
callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
('a','archive','Generate-Packages-Sources::Options::Archive','HasArg'),
('s',"suite","Generate-Packages-Sources::Options::Suite"),
('f',"force","Generate-Packages-Sources::Options::Force"),
('o','option','','ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
if Options.has_key("Suite"):
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess recieved signal ', msg])
else:
logger.log(['E: ', msg])
for s in suites:
component_ids = [ c.component_id for c in s.components ]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c], callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=parse_results)
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "no-act", "Generate-Index-Diffs::Options::NoAct"),
('v', "verbose", "Generate-Index-Diffs::Options::Verbose"),
]
suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if "Help" in Options:
usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
# can only be set via config at the moment
max_parallel = int(Options.get("MaxParallel", "8"))
if "PatchName" not in Options:
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime(format)
session = DBConn().session()
pending_tasks = []
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
archives = utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
suites = [s.suite_name for s in query]
for suitename in suites:
print("Processing: " + suitename)
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print("Skipping: " + suite + " (untouchable)")
continue
skip_all = True
if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all:
skip_all = False
architectures = get_suite_architectures(suite, skipall=skip_all, session=session)
components = [c.component_name for c in session.query(Component.component_name)]
suite_suffix = utils.suite_suffix(suitename)
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
merged_pdiffs = suiteobj.merged_pdiffs
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
workpath = os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
continue
(fname, fext) = os.path.splitext(entry)
processfile = os.path.join(workpath, fname)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs)
pending_tasks.append(coroutine)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s" % architecture
packages = "Packages"
maxsuite = maxpackages
for component in components:
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs)
pending_tasks.append(coroutine)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs)
pending_tasks.append(coroutine)
asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('r', "rootdir", "Generate-Index-Diffs::Options::RootDir", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
]
suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if Options.has_key("Help"): usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
if not Options.has_key("PatchName"):
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime( format )
AptCnf = apt_pkg.newConfiguration()
apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file())
if Options.has_key("RootDir"):
Cnf["Dir::Root"] = Options["RootDir"]
if not suites:
suites = Cnf.subtree("Suite").list()
for suitename in suites:
print "Processing: " + suitename
SuiteBlock = Cnf.subtree("Suite::" + suitename)
suiteobj = get_suite(suitename.lower())
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
architectures = get_suite_architectures(suite, skipall=True)
if SuiteBlock.has_key("Components"):
components = SuiteBlock.value_list("Components")
else:
components = []
suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite))
if AptCnf.has_key("tree::%s" % (tree)):
sections = AptCnf["tree::%s::Sections" % (tree)].split()
elif AptCnf.has_key("bindirectory::%s" % (tree)):
sections = AptCnf["bindirectory::%s::Sections" % (tree)].split()
else:
aptcnf_filename = os.path.basename(utils.which_apt_conf_file())
print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename)
continue
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in sections:
#print "DEBUG: Working on %s" % (component)
workpath=os.path.join(Cnf["Dir::Root"], tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
#print "EXCLUDING %s" % (entry)
continue
(fname, fext) = os.path.splitext(entry)
processfile=os.path.join(workpath, fname)
#print "Working: %s" % (processfile)
storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
#print "Storefile: %s" % (storename)
genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
# use sections instead of components since dak.conf
# treats "foo/bar main" as suite "foo", suitesuffix "bar" and
# component "bar/main". suck.
for component in sections:
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s"% (architecture)
packages = "Packages"
maxsuite = maxpackages
# Process Contents
file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component,
architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, \
Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents))
file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree,
component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, \
Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::%s" % (suite, packages), maxsuite))