def main():
dak.examine_package.use_html = True
pool = DakProcessPool(processes=5)
session = DBConn().session()
upload_ids = [u.id for u in init(session)]
session.close()
for upload_id in upload_ids:
pool.apply_async(do_pkg, [upload_id], callback=result_callback)
pool.close()
#p.wait(timeout=600)
pool.join()
for htmlfile in htmlfiles_to_process:
with open(htmlfile, "w") as fd:
fd.write(timeout_str)
if pool.overall_status() != PROC_STATUS_SUCCESS:
raise Exception("Processing failed (code %s)" %
(pool.overall_status()))
files = set(os.listdir(cnf["Show-New::HTMLPath"]))
to_delete = [
x for x in files.difference(set(sources)) if x.endswith(".html")
]
for f in to_delete:
os.remove(os.path.join(cnf["Show-New::HTMLPath"], f))
def main():
cnf = Config()
Logger = daklog.Logger('generate-filelist')
Arguments = [('h', "help", "Filelist::Options::Help"),
('s', "suite", "Filelist::Options::Suite", "HasArg"),
('c', "component", "Filelist::Options::Component", "HasArg"),
('a', "architecture", "Filelist::Options::Architecture", "HasArg"),
('i', "incremental", "Filelist::Options::Incremental")]
session = DBConn().session()
query_suites = session.query(Suite)
suites = [suite.suite_name for suite in query_suites]
if not cnf.has_key('Filelist::Options::Suite'):
cnf['Filelist::Options::Suite'] = ','.join(suites).encode()
query_components = session.query(Component)
components = \
[component.component_name for component in query_components]
if not cnf.has_key('Filelist::Options::Component'):
cnf['Filelist::Options::Component'] = ','.join(components).encode()
query_architectures = session.query(Architecture)
architectures = \
[architecture.arch_string for architecture in query_architectures]
if not cnf.has_key('Filelist::Options::Architecture'):
cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode()
cnf['Filelist::Options::Help'] = ''
cnf['Filelist::Options::Incremental'] = ''
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Filelist::Options")
if Options['Help']:
usage()
pool = DakProcessPool()
query_suites = query_suites. \
filter(Suite.suite_name.in_(utils.split_args(Options['Suite'])))
query_components = query_components. \
filter(Component.component_name.in_(utils.split_args(Options['Component'])))
query_architectures = query_architectures. \
filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture'])))
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
Logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
Logger.log(['E: Subprocess recieved signal ', msg])
else:
Logger.log(['E: ', msg])
for suite in query_suites:
suite_id = suite.suite_id
for component in query_components:
component_id = component.component_id
for architecture in query_architectures:
architecture_id = architecture.arch_id
if architecture not in suite.architectures:
pass
elif architecture.arch_string == 'source':
pool.apply_async(writeSourceList,
(suite_id, component_id, Options['Incremental']), callback=parse_results)
elif architecture.arch_string == 'all':
pool.apply_async(writeAllList,
(suite_id, component_id, architecture_id, 'deb',
Options['Incremental']), callback=parse_results)
pool.apply_async(writeAllList,
(suite_id, component_id, architecture_id, 'udeb',
Options['Incremental']), callback=parse_results)
else: # arch any
pool.apply_async(writeBinaryList,
(suite_id, component_id, architecture_id, 'deb',
Options['Incremental']), callback=parse_results)
pool.apply_async(writeBinaryList,
(suite_id, component_id, architecture_id, 'udeb',
Options['Incremental']), callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
Logger.close()
sys.exit(pool.overall_status())
def main():
global Logger
cnf = Config()
for i in ["Help", "Suite", "Force", "Quiet"]:
key = "Generate-Releases::Options::%s" % i
if key not in cnf:
cnf[key] = ""
Arguments = [('h', "help", "Generate-Releases::Options::Help"),
('a', 'archive', 'Generate-Releases::Options::Archive', 'HasArg'),
('s', "suite", "Generate-Releases::Options::Suite"),
('f', "force", "Generate-Releases::Options::Force"),
('q', "quiet", "Generate-Releases::Options::Quiet"),
('o', 'option', '', 'ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Generate-Releases::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('generate-releases')
pool = DakProcessPool()
session = DBConn().session()
if Options["Suite"]:
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print("cannot find suite %s" % s)
Logger.log(['cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False) # noqa:E712
if 'Archive' in Options:
archive_names = utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(Archive.archive_name.in_(archive_names))
suites = query.all()
for s in suites:
# Setup a multiprocessing Pool. As many workers as we have CPU cores.
if s.untouchable and not Options["Force"]:
print("Skipping %s (untouchable)" % s.suite_name)
continue
if not Options["Quiet"]:
print("Processing %s" % s.suite_name)
Logger.log(['Processing release file for Suite: %s' % (s.suite_name)])
pool.apply_async(generate_helper, (s.suite_id, ))
# No more work will be added to our pool, close it and then wait for all to finish
pool.close()
pool.join()
retcode = pool.overall_status()
if retcode > 0:
# TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING
Logger.log(['Release file generation broken: %s' % (','.join([str(x[1]) for x in pool.results]))])
Logger.close()
sys.exit(retcode)
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h', "help", "Generate-Packages-Sources::Options::Help"),
('a', 'archive',
'Generate-Packages-Sources::Options::Archive', 'HasArg'),
('s', "suite", "Generate-Packages-Sources::Options::Suite",
'HasArg'),
('f', "force", "Generate-Packages-Sources::Options::Force"),
('o', 'option', '', 'ArbItem')]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
import daklib.utils
if Options.has_key("Suite"):
suites = []
suite_names = daklib.utils.split_args(Options['Suite'])
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
archive_names = daklib.utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(
Archive.archive_name.in_(archive_names))
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log(msg)
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess received signal ', msg])
else:
logger.log(['E: ', msg])
# Lock tables so that nobody can change things underneath us
session.execute("LOCK TABLE src_associations IN SHARE MODE")
session.execute("LOCK TABLE bin_associations IN SHARE MODE")
for s in suites:
component_ids = [c.component_id for c in s.components]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar(
"Refusing to touch %s (untouchable and not forced)" %
s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c],
callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c],
callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages,
[s.suite_id, c, a.arch_id, 'deb'],
callback=parse_results)
pool.apply_async(generate_packages,
[s.suite_id, c, a.arch_id, 'udeb'],
callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
('a','archive','Generate-Packages-Sources::Options::Archive','HasArg'),
('s',"suite","Generate-Packages-Sources::Options::Suite"),
('f',"force","Generate-Packages-Sources::Options::Force"),
('o','option','','ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
if Options.has_key("Suite"):
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess recieved signal ', msg])
else:
logger.log(['E: ', msg])
for s in suites:
component_ids = [ c.component_id for c in s.components ]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c], callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=parse_results)
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
def main ():
global Logger
cnf = Config()
for i in ["Help", "Suite", "Force", "Quiet"]:
if not cnf.has_key("Generate-Releases::Options::%s" % (i)):
cnf["Generate-Releases::Options::%s" % (i)] = ""
Arguments = [('h',"help","Generate-Releases::Options::Help"),
('a','archive','Generate-Releases::Options::Archive','HasArg'),
('s',"suite","Generate-Releases::Options::Suite"),
('f',"force","Generate-Releases::Options::Force"),
('q',"quiet","Generate-Releases::Options::Quiet"),
('o','option','','ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Generate-Releases::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('generate-releases')
pool = DakProcessPool()
session = DBConn().session()
if Options["Suite"]:
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "cannot find suite %s" % s
Logger.log(['cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
suites = query.all()
broken=[]
for s in suites:
# Setup a multiprocessing Pool. As many workers as we have CPU cores.
if s.untouchable and not Options["Force"]:
print "Skipping %s (untouchable)" % s.suite_name
continue
if not Options["Quiet"]:
print "Processing %s" % s.suite_name
Logger.log(['Processing release file for Suite: %s' % (s.suite_name)])
pool.apply_async(generate_helper, (s.suite_id, ))
# No more work will be added to our pool, close it and then wait for all to finish
pool.close()
pool.join()
retcode = pool.overall_status()
if retcode > 0:
# TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING
Logger.log(['Release file generation broken: %s' % (','.join([str(x[1]) for x in pool.results]))])
Logger.close()
sys.exit(retcode)
def main():
cnf = Config()
Logger = daklog.Logger('generate-filelist')
Arguments = [('h', "help", "Filelist::Options::Help"),
('s', "suite", "Filelist::Options::Suite", "HasArg"),
('c', "component", "Filelist::Options::Component", "HasArg"),
('a', "architecture", "Filelist::Options::Architecture",
"HasArg"),
('i', "incremental", "Filelist::Options::Incremental")]
session = DBConn().session()
query_suites = session.query(Suite)
suites = [suite.suite_name for suite in query_suites]
if not cnf.has_key('Filelist::Options::Suite'):
cnf['Filelist::Options::Suite'] = ','.join(suites).encode()
query_components = session.query(Component)
components = \
[component.component_name for component in query_components]
if not cnf.has_key('Filelist::Options::Component'):
cnf['Filelist::Options::Component'] = ','.join(components).encode()
query_architectures = session.query(Architecture)
architectures = \
[architecture.arch_string for architecture in query_architectures]
if not cnf.has_key('Filelist::Options::Architecture'):
cnf['Filelist::Options::Architecture'] = ','.join(
architectures).encode()
cnf['Filelist::Options::Help'] = ''
cnf['Filelist::Options::Incremental'] = ''
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Filelist::Options")
if Options['Help']:
usage()
pool = DakProcessPool()
query_suites = query_suites. \
filter(Suite.suite_name.in_(utils.split_args(Options['Suite'])))
query_components = query_components. \
filter(Component.component_name.in_(utils.split_args(Options['Component'])))
query_architectures = query_architectures. \
filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture'])))
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
Logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
Logger.log(['E: Subprocess recieved signal ', msg])
else:
Logger.log(['E: ', msg])
for suite in query_suites:
suite_id = suite.suite_id
for component in query_components:
component_id = component.component_id
for architecture in query_architectures:
architecture_id = architecture.arch_id
if architecture not in suite.architectures:
pass
elif architecture.arch_string == 'source':
pool.apply_async(
writeSourceList,
(suite_id, component_id, Options['Incremental']),
callback=parse_results)
elif architecture.arch_string == 'all':
pool.apply_async(writeAllList,
(suite_id, component_id, architecture_id,
'deb', Options['Incremental']),
callback=parse_results)
pool.apply_async(writeAllList,
(suite_id, component_id, architecture_id,
'udeb', Options['Incremental']),
callback=parse_results)
else: # arch any
pool.apply_async(writeBinaryList,
(suite_id, component_id, architecture_id,
'deb', Options['Incremental']),
callback=parse_results)
pool.apply_async(writeBinaryList,
(suite_id, component_id, architecture_id,
'udeb', Options['Incremental']),
callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
Logger.close()
sys.exit(pool.overall_status())
