def main():
cnf = Config()
arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
('j',"concurrency", "%s::%s" % (options_prefix,"Concurrency"),"HasArg"),
('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
]
args = apt_pkg.parse_commandline(cnf.Cnf, arguments,sys.argv)
num_threads = 1
if len(args) > 0:
usage()
if cnf.has_key("%s::%s" % (options_prefix,"Help")):
usage()
level=logging.INFO
if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
level=logging.ERROR
elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
level=logging.DEBUG
logging.basicConfig( level=level,
format='%(asctime)s %(levelname)s %(message)s',
stream = sys.stderr )
if Config().has_key( "%s::%s" %(options_prefix,"Concurrency")):
num_threads = int(Config()[ "%s::%s" %(options_prefix,"Concurrency")])
ImportKnownChanges(num_threads)
def sign_release_dir(suite, dirname):
cnf = Config()
if cnf.has_key("Dinstall::SigningKeyring"):
keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
if cnf.has_key("Dinstall::SigningPubKeyring"):
keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
arguments = "--no-options --batch --no-tty --armour"
relname = os.path.join(dirname, 'Release')
dest = os.path.join(dirname, 'Release.gpg')
if os.path.exists(dest):
os.unlink(dest)
inlinedest = os.path.join(dirname, 'InRelease')
if os.path.exists(inlinedest):
os.unlink(inlinedest)
# We can only use one key for inline signing so use the first one in
# the array for consistency
firstkey = True
for keyid in suite.signingkeys:
defkeyid = "--default-key %s" % keyid
os.system("gpg %s %s %s --detach-sign <%s >>%s" %
(keyring, defkeyid, arguments, relname, dest))
if firstkey:
os.system("gpg %s %s %s --clearsign <%s >>%s" %
(keyring, defkeyid, arguments, relname, inlinedest))
firstkey = False
def main():
cnf = Config()
arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
]
args = apt_pkg.parse_commandline(cnf.Cnf, arguments,sys.argv)
num_threads = 1
if len(args) > 0:
usage(1)
if cnf.has_key("%s::%s" % (options_prefix,"Help")):
usage(0)
level=logging.INFO
if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
level=logging.ERROR
elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
level=logging.DEBUG
logging.basicConfig( level=level,
format='%(asctime)s %(levelname)s %(message)s',
stream = sys.stderr )
ImportNewFiles()
def sign_release_dir(suite, dirname):
cnf = Config()
if cnf.has_key("Dinstall::SigningKeyring"):
keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
if cnf.has_key("Dinstall::SigningPubKeyring"):
keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
arguments = "--no-options --batch --no-tty --armour --personal-digest-preferences=SHA256"
relname = os.path.join(dirname, 'Release')
dest = os.path.join(dirname, 'Release.gpg')
if os.path.exists(dest):
os.unlink(dest)
inlinedest = os.path.join(dirname, 'InRelease')
if os.path.exists(inlinedest):
os.unlink(inlinedest)
defkeyid = ""
for keyid in suite.signingkeys or []:
defkeyid += "--local-user %s " % keyid
os.system("gpg %s %s %s --detach-sign <%s >>%s" %
(keyring, defkeyid, arguments, relname, dest))
os.system("gpg %s %s %s --clearsign <%s >>%s" %
(keyring, defkeyid, arguments, relname, inlinedest))
def sign_release_dir(suite, dirname):
cnf = Config()
if cnf.has_key("Dinstall::SigningKeyring"):
keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
if cnf.has_key("Dinstall::SigningPubKeyring"):
keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
arguments = "--no-options --batch --no-tty --armour --personal-digest-preferences=SHA256"
relname = os.path.join(dirname, 'Release')
dest = os.path.join(dirname, 'Release.gpg')
if os.path.exists(dest):
os.unlink(dest)
inlinedest = os.path.join(dirname, 'InRelease')
if os.path.exists(inlinedest):
os.unlink(inlinedest)
defkeyid=""
for keyid in suite.signingkeys or []:
defkeyid += "--local-user %s " % keyid
os.system("gpg %s %s %s --detach-sign <%s >>%s" %
(keyring, defkeyid, arguments, relname, dest))
os.system("gpg %s %s %s --clearsign <%s >>%s" %
(keyring, defkeyid, arguments, relname, inlinedest))
def main():
cnf = Config()
Arguments = [('h',"help","DEP11::Options::Help"),
('s',"suite","DEP11::Options::Suite", "HasArg"),
('e',"expire","DEP11::Options::ExpireCache"),
('h',"write-hints","DEP11::Options::WriteHints"),
]
for i in ["help", "suite", "ExpireCache"]:
if not cnf.has_key("DEP11::Options::%s" % (i)):
cnf["DEP11::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("DEP11::Options")
if Options["Help"]:
usage()
return
suitename = Options["Suite"]
if not suitename:
print("You need to specify a suite!")
sys.exit(1)
# check if we have some important config options set
if not cnf.has_key("Dir::MetaInfo"):
print("You need to specify a metadata export directory (Dir::MetaInfo)")
sys.exit(1)
if not cnf.has_key("DEP11::Url"):
print("You need to specify a metadata public web URL (DEP11::Url)")
sys.exit(1)
if not cnf.has_key("DEP11::IconSizes"):
print("You need to specify a list of allowed icon-sizes (DEP11::IconSizes)")
sys.exit(1)
if Options["WriteHints"] and not cnf.has_key("Dir::MetaInfoHints"):
print("You need to specify an export directory for DEP-11 hints files (Dir::MetaInfoHints)")
sys.exit(1)
logger = daklog.Logger('generate-metadata')
from daklib.dbconn import Component, DBConn, get_suite, Suite
session = DBConn().session()
suite = get_suite(suitename.lower(), session)
if Options["ExpireCache"]:
expire_dep11_data_cache(session, suitename, logger)
process_suite(session, suite, logger)
# export database content as Components-<arch>.xz YAML documents
write_component_files(session, suite, logger)
if Options["WriteHints"]:
write_hints_files(session, suite, logger)
# we're done
logger.close()
def main():
global Options, Logger, Sections, Priorities
cnf = Config()
session = DBConn().session()
Arguments = [('a', "automatic", "Process-New::Options::Automatic"),
('b', "no-binaries", "Process-New::Options::No-Binaries"),
('c', "comments", "Process-New::Options::Comments"),
('h', "help", "Process-New::Options::Help"),
('m', "manual-reject", "Process-New::Options::Manual-Reject",
"HasArg"), ('t', "trainee", "Process-New::Options::Trainee"),
('q', 'queue', 'Process-New::Options::Queue', 'HasArg'),
('n', "no-action", "Process-New::Options::No-Action")]
changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
for i in [
"automatic", "no-binaries", "comments", "help", "manual-reject",
"no-action", "version", "trainee"
]:
if not cnf.has_key("Process-New::Options::%s" % (i)):
cnf["Process-New::Options::%s" % (i)] = ""
queue_name = cnf.get('Process-New::Options::Queue', 'new')
new_queue = session.query(PolicyQueue).filter_by(
queue_name=queue_name).one()
if len(changes_files) == 0:
uploads = new_queue.uploads
else:
uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
.join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
Options = cnf.subtree("Process-New::Options")
if Options["Help"]:
usage()
if not Options["No-Action"]:
try:
Logger = daklog.Logger("process-new")
except CantOpenError as e:
Options["Trainee"] = "True"
Sections = Section_Completer(session)
Priorities = Priority_Completer(session)
readline.parse_and_bind("tab: complete")
if len(uploads) > 1:
sys.stderr.write("Sorting changes...\n")
uploads = sort_uploads(new_queue, uploads, session,
Options["No-Binaries"])
if Options["Comments"]:
show_new_comments(uploads, session)
else:
for upload in uploads:
do_pkg(upload, session)
end()
def init(session):
global cnf, Options
cnf = Config()
Arguments = [('h', "help", "Show-New::Options::Help"),
("p", "html-path", "Show-New::HTMLPath", "HasArg"),
('q', 'queue', 'Show-New::Options::Queue', 'HasArg')]
for i in ["help"]:
if not cnf.has_key("Show-New::Options::%s" % (i)):
cnf["Show-New::Options::%s" % (i)] = ""
changesnames = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Show-New::Options")
if Options["help"]:
usage()
queue_names = Options.find('Queue', 'new').split(',')
uploads = session.query(PolicyQueueUpload) \
.join(PolicyQueueUpload.policy_queue).filter(PolicyQueue.queue_name.in_(queue_names)) \
.join(PolicyQueueUpload.changes).order_by(DBChange.source)
if len(changesnames) > 0:
uploads = uploads.filter(DBChange.changesname.in_(changesnames))
return uploads
def init (self):
cnf = Config()
arguments = [('h', "help", "Update-DB::Options::Help")]
for i in [ "help" ]:
if not cnf.has_key("Update-DB::Options::%s" % (i)):
cnf["Update-DB::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, arguments, sys.argv)
options = cnf.subtree("Update-DB::Options")
if options["Help"]:
self.usage()
elif arguments:
utils.warn("dak update-db takes no arguments.")
self.usage(exit_code=1)
try:
if os.path.isdir(cnf["Dir::Lock"]):
lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
else:
utils.warn("Lock directory doesn't exist yet - not locking")
except IOError as e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
utils.fubar("Couldn't obtain lock; assuming another 'dak process-unchecked' is already running.")
self.update_db()
def do_update(self):
"""
Add missing PrimaryMirror field to archive table
"""
print __doc__
try:
cnf = Config()
c = self.db.cursor()
c.execute("ALTER TABLE archive ADD COLUMN primary_mirror TEXT")
c.execute("SELECT id, name FROM archive")
query = "UPDATE archive SET primary_mirror = %s WHERE id = %s"
for a_id, a_name in c.fetchall():
if cnf.has_key('Archive::%s::PrimaryMirror' % a_name):
primloc = cnf['Archive::%s::PrimaryMirror' % a_name]
print "Setting archive %s PrimaryMirror to %s" % (a_name, primloc)
c.execute(query, [primloc, a_id])
c.execute("UPDATE config SET value = '63' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError('Unable to apply sick update 63, rollback issued. Error message : %s' % (str(msg)))
def init(session):
global cnf, Options
cnf = Config()
Arguments = [
("h", "help", "Show-New::Options::Help"),
("p", "html-path", "Show-New::HTMLPath", "HasArg"),
("q", "queue", "Show-New::Options::Queue", "HasArg"),
]
for i in ["help"]:
if not cnf.has_key("Show-New::Options::%s" % (i)):
cnf["Show-New::Options::%s" % (i)] = ""
changesnames = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Show-New::Options")
if Options["help"]:
usage()
queue_names = Options.find("Queue", "new").split(",")
uploads = (
session.query(PolicyQueueUpload)
.join(PolicyQueueUpload.policy_queue)
.filter(PolicyQueue.queue_name.in_(queue_names))
.join(PolicyQueueUpload.changes)
.order_by(DBChange.source)
)
if len(changesnames) > 0:
uploads = uploads.filter(DBChange.changesname.in_(changesnames))
return uploads
def do_update(self):
"""
Add missing PrimaryMirror field to archive table
"""
print __doc__
try:
cnf = Config()
c = self.db.cursor()
c.execute("ALTER TABLE archive ADD COLUMN primary_mirror TEXT")
c.execute("SELECT id, name FROM archive")
query = "UPDATE archive SET primary_mirror = %s WHERE id = %s"
for a_id, a_name in c.fetchall():
if cnf.has_key('Archive::%s::PrimaryMirror' % a_name):
primloc = cnf['Archive::%s::PrimaryMirror' % a_name]
print "Setting archive %s PrimaryMirror to %s" % (a_name,
primloc)
c.execute(query, [primloc, a_id])
c.execute("UPDATE config SET value = '63' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
'Unable to apply sick update 63, rollback issued. Error message : %s'
% (str(msg)))
def main():
cnf = Config()
Arguments = [
('h', "help", "Copy-Installer::Options::Help"),
('s', "source", "Copy-Installer::Options::Source", "HasArg"),
('d', "destination", "Copy-Installer::Options::Destination", "HasArg"),
('n', "no-action", "Copy-Installer::Options::No-Action"),
]
for option in [ "help", "source", "destination", "no-action" ]:
if not cnf.has_key("Copy-Installer::Options::%s" % (option)):
cnf["Copy-Installer::Options::%s" % (option)] = ""
extra_arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Copy-Installer::Options")
if Options["Help"]:
usage()
if len(extra_arguments) != 1:
usage(1)
initializer = { "version": extra_arguments[0] }
if Options["Source"] != "":
initializer["source"] = Options["Source"]
if Options["Destination"] != "":
initializer["dest"] = Options["Destination"]
copier = InstallerCopier(**initializer)
print copier.get_message()
if Options["No-Action"]:
print 'Do nothing because --no-action has been set.'
else:
copier.do_copy()
print 'Installer has been copied successfully.'
def init(self):
cnf = Config()
arguments = [('h', "help", "Update-DB::Options::Help"),
("y", "yes", "Update-DB::Options::Yes")]
for i in ["help"]:
if not cnf.has_key("Update-DB::Options::%s" % (i)):
cnf["Update-DB::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, arguments, sys.argv)
options = cnf.subtree("Update-DB::Options")
if options["Help"]:
self.usage()
elif arguments:
utils.warn("dak update-db takes no arguments.")
self.usage(exit_code=1)
try:
if os.path.isdir(cnf["Dir::Lock"]):
lock_fd = os.open(
os.path.join(cnf["Dir::Lock"], 'dinstall.lock'),
os.O_RDWR | os.O_CREAT)
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
else:
utils.warn("Lock directory doesn't exist yet - not locking")
except IOError as e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[
e.errno] == 'EAGAIN':
utils.fubar(
"Couldn't obtain lock; assuming another 'dak process-unchecked' is already running."
)
self.update_db()
def main():
cnf = Config()
Arguments = [
('h', "help", "Copy-Installer::Options::Help"),
('s', "source", "Copy-Installer::Options::Source", "HasArg"),
('d', "destination", "Copy-Installer::Options::Destination", "HasArg"),
('n', "no-action", "Copy-Installer::Options::No-Action"),
]
for option in [ "help", "source", "destination", "no-action" ]:
if not cnf.has_key("Copy-Installer::Options::%s" % (option)):
cnf["Copy-Installer::Options::%s" % (option)] = ""
extra_arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Copy-Installer::Options")
if Options["Help"]:
usage()
if len(extra_arguments) != 1:
usage(1)
initializer = { "version": extra_arguments[0] }
if Options["Source"] != "":
initializer["source"] = Options["Source"]
if Options["Destination"] != "":
initializer["dest"] = Options["Destination"]
copier = InstallerCopier(**initializer)
print copier.get_message()
if Options["No-Action"]:
print 'Do nothing because --no-action has been set.'
else:
copier.do_copy()
print 'Installer has been copied successfully.'
def do_pkg(changes_full_path, session):
changes_dir = os.path.dirname(changes_full_path)
changes_file = os.path.basename(changes_full_path)
u = Upload()
u.pkg.changes_file = changes_file
(u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
u.load_changes(changes_file)
u.pkg.directory = changes_dir
u.update_subst()
u.logger = Logger
origchanges = os.path.abspath(u.pkg.changes_file)
# Try to get an included dsc
dsc = None
(status, _) = u.load_dsc()
if status:
dsc = u.pkg.dsc
cnf = Config()
bcc = "X-DAK: dak process-new"
if cnf.has_key("Dinstall::Bcc"):
u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
else:
u.Subst["__BCC__"] = bcc
files = u.pkg.files
u.check_distributions()
for deb_filename, f in files.items():
if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
u.binary_file_checks(deb_filename, session)
u.check_binary_against_db(deb_filename, session)
else:
u.source_file_checks(deb_filename, session)
u.check_source_against_db(deb_filename, session)
u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
try:
with lock_package(u.pkg.changes["source"]):
with clean_holding(u.pkg):
if not recheck(u, session):
return
new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session)
if byhand:
do_byhand(u, session)
elif new:
do_new(u, session)
else:
try:
check_daily_lock()
new_accept(u, Options["No-Action"], session)
except CantGetLockError:
print "Hello? Operator! Give me the number for 911!"
print "Dinstall in the locked area, cant process packages, come back later"
except AlreadyLockedError as e:
print "Seems to be locked by %s already, skipping..." % (e)
def main():
global Options
cnf = Config()
Arguments = [
('h', "help", "Auto-Decruft::Options::Help"),
('n', "dry-run", "Auto-Decruft::Options::Dry-Run"),
('d', "debug", "Auto-Decruft::Options::Debug"),
('s', "suite", "Auto-Decruft::Options::Suite", "HasArg"),
# The "\0" seems to be the only way to disable short options.
("\0", 'if-newer-version-in', "Auto-Decruft::Options::OtherSuite",
"HasArg"),
("\0", 'if-newer-version-in-rm-msg',
"Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg")
]
for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg"]:
if not cnf.has_key("Auto-Decruft::Options::%s" % (i)):
cnf["Auto-Decruft::Options::%s" % (i)] = ""
cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite",
"unstable")
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Auto-Decruft::Options")
if Options["Help"]:
usage()
debug = False
dryrun = False
if Options["Dry-Run"]:
dryrun = True
if Options["Debug"]:
debug = True
if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]:
utils.fubar(
"--if-newer-version-in requires --if-newer-version-in-rm-msg")
session = DBConn().session()
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
if Options["OtherSuite"]:
osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name
decruft_newer_version_in(osuite, suite_name, suite_id,
Options["OtherSuiteRMMsg"], session, dryrun)
if not dryrun:
session.commit()
def main():
global Options, Logger, Sections, Priorities
cnf = Config()
session = DBConn().session()
Arguments = [('a',"automatic","Process-New::Options::Automatic"),
('b',"no-binaries","Process-New::Options::No-Binaries"),
('c',"comments","Process-New::Options::Comments"),
('h',"help","Process-New::Options::Help"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('t',"trainee","Process-New::Options::Trainee"),
('n',"no-action","Process-New::Options::No-Action")]
for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
if not cnf.has_key("Process-New::Options::%s" % (i)):
cnf["Process-New::Options::%s" % (i)] = ""
changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
if len(changes_files) == 0:
new_queue = get_policy_queue('new', session );
changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
else:
changes_paths = [ os.path.abspath(j) for j in changes_files ]
Options = cnf.subtree("Process-New::Options")
if Options["Help"]:
usage()
if not Options["No-Action"]:
try:
Logger = daklog.Logger("process-new")
except CantOpenError as e:
Options["Trainee"] = "True"
Sections = Section_Completer(session)
Priorities = Priority_Completer(session)
readline.parse_and_bind("tab: complete")
if len(changes_paths) > 1:
sys.stderr.write("Sorting changes...\n")
changes_files = sort_changes(changes_paths, session, Options["No-Binaries"])
if Options["Comments"]:
show_new_comments(changes_files, session)
else:
for changes_file in changes_files:
changes_file = utils.validate_changes_file_arg(changes_file, 0)
if not changes_file:
continue
print "\n" + os.path.basename(changes_file)
do_pkg (changes_file, session)
end()
def main():
global Options, Logger, Sections, Priorities
cnf = Config()
session = DBConn().session()
Arguments = [('a',"automatic","Process-New::Options::Automatic"),
('b',"no-binaries","Process-New::Options::No-Binaries"),
('c',"comments","Process-New::Options::Comments"),
('h',"help","Process-New::Options::Help"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('t',"trainee","Process-New::Options::Trainee"),
('q','queue','Process-New::Options::Queue', 'HasArg'),
('n',"no-action","Process-New::Options::No-Action")]
changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
if not cnf.has_key("Process-New::Options::%s" % (i)):
cnf["Process-New::Options::%s" % (i)] = ""
queue_name = cnf.get('Process-New::Options::Queue', 'new')
new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
if len(changes_files) == 0:
uploads = new_queue.uploads
else:
uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
.join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
Options = cnf.subtree("Process-New::Options")
if Options["Help"]:
usage()
if not Options["No-Action"]:
try:
Logger = daklog.Logger("process-new")
except CantOpenError as e:
Options["Trainee"] = "True"
Sections = Section_Completer(session)
Priorities = Priority_Completer(session)
readline.parse_and_bind("tab: complete")
if len(uploads) > 1:
sys.stderr.write("Sorting changes...\n")
uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"])
if Options["Comments"]:
show_new_comments(uploads, session)
else:
for upload in uploads:
do_pkg (upload, session)
end()
def main():
global Options, Logger
cnf = Config()
session = DBConn().session()
Arguments = [('h', "help", "Process-Policy::Options::Help"),
('n', "no-action", "Process-Policy::Options::No-Action")]
for i in ["help", "no-action"]:
if not cnf.has_key("Process-Policy::Options::%s" % (i)):
cnf["Process-Policy::Options::%s" % (i)] = ""
queue_name = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
if len(queue_name) != 1:
print "E: Specify exactly one policy queue"
sys.exit(1)
queue_name = queue_name[0]
Options = cnf.subtree("Process-Policy::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger("process-policy")
if not Options["No-Action"]:
urgencylog = UrgencyLog()
with ArchiveTransaction() as transaction:
session = transaction.session
try:
pq = session.query(PolicyQueue).filter_by(
queue_name=queue_name).one()
except NoResultFound:
print "E: Cannot find policy queue %s" % queue_name
sys.exit(1)
commentsdir = os.path.join(pq.path, 'COMMENTS')
# The comments stuff relies on being in the right directory
os.chdir(pq.path)
do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK",
comment_reject, transaction)
do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK",
comment_accept, transaction)
do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK",
comment_accept, transaction)
remove_unreferenced_binaries(pq, transaction)
remove_unreferenced_sources(pq, transaction)
if not Options['No-Action']:
urgencylog.close()
def main ():
global db_files, waste, excluded
cnf = Config()
Arguments = [('h',"help","Check-Archive::Options::Help")]
for i in [ "help" ]:
if not cnf.has_key("Check-Archive::Options::%s" % (i)):
cnf["Check-Archive::Options::%s" % (i)] = ""
args = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Check-Archive::Options")
if Options["Help"]:
usage()
if len(args) < 1:
utils.warn("dak check-archive requires at least one argument")
usage(1)
elif len(args) > 1:
utils.warn("dak check-archive accepts only one argument")
usage(1)
mode = args[0].lower()
# Initialize DB
DBConn()
if mode == "checksums":
check_checksums()
elif mode == "files":
check_files()
elif mode == "dsc-syntax":
check_dscs()
elif mode == "missing-overrides":
check_override()
elif mode == "source-in-one-dir":
check_source_in_one_dir()
elif mode == "timestamps":
check_timestamps()
elif mode == "files-in-dsc":
check_files_in_dsc()
elif mode == "validate-indices":
check_indices_files_exist()
elif mode == "files-not-symlinks":
check_files_not_symlinks()
elif mode == "validate-builddeps":
check_build_depends()
elif mode == "add-missing-source-checksums":
add_missing_source_checksums()
elif mode == "fix-checksums":
fix_checksums()
else:
utils.warn("unknown mode '%s'" % (mode))
usage(1)
def main():
global db_files, waste, excluded
cnf = Config()
Arguments = [('h', "help", "Check-Archive::Options::Help")]
for i in ["help"]:
if not cnf.has_key("Check-Archive::Options::%s" % (i)):
cnf["Check-Archive::Options::%s" % (i)] = ""
args = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Check-Archive::Options")
if Options["Help"]:
usage()
if len(args) < 1:
utils.warn("dak check-archive requires at least one argument")
usage(1)
elif len(args) > 1:
utils.warn("dak check-archive accepts only one argument")
usage(1)
mode = args[0].lower()
# Initialize DB
DBConn()
if mode == "checksums":
check_checksums()
elif mode == "files":
check_files()
elif mode == "dsc-syntax":
check_dscs()
elif mode == "missing-overrides":
check_override()
elif mode == "source-in-one-dir":
check_source_in_one_dir()
elif mode == "timestamps":
check_timestamps()
elif mode == "files-in-dsc":
check_files_in_dsc()
elif mode == "validate-indices":
check_indices_files_exist()
elif mode == "files-not-symlinks":
check_files_not_symlinks()
elif mode == "validate-builddeps":
check_build_depends()
elif mode == "add-missing-source-checksums":
add_missing_source_checksums()
elif mode == "fix-checksums":
fix_checksums()
else:
utils.warn("unknown mode '%s'" % (mode))
usage(1)
def main():
global Options, Logger
cnf = Config()
session = DBConn().session()
Arguments = [
("h", "help", "Process-Policy::Options::Help"),
("n", "no-action", "Process-Policy::Options::No-Action"),
]
for i in ["help", "no-action"]:
if not cnf.has_key("Process-Policy::Options::%s" % (i)):
cnf["Process-Policy::Options::%s" % (i)] = ""
queue_name = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
if len(queue_name) != 1:
print "E: Specify exactly one policy queue"
sys.exit(1)
queue_name = queue_name[0]
Options = cnf.subtree("Process-Policy::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger("process-policy")
if not Options["No-Action"]:
urgencylog = UrgencyLog()
with ArchiveTransaction() as transaction:
session = transaction.session
try:
pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
except NoResultFound:
print "E: Cannot find policy queue %s" % queue_name
sys.exit(1)
commentsdir = os.path.join(pq.path, "COMMENTS")
# The comments stuff relies on being in the right directory
os.chdir(pq.path)
do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction)
do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction)
do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction)
remove_unreferenced_binaries(pq, transaction)
remove_unreferenced_sources(pq, transaction)
if not Options["No-Action"]:
urgencylog.close()
def main():
global Options, Logger
cnf = Config()
for i in ["Help", "Incoming", "No-Action", "Verbose"]:
if not cnf.has_key("Clean-Queues::Options::%s" % (i)):
cnf["Clean-Queues::Options::%s" % (i)] = ""
if not cnf.has_key("Clean-Queues::Options::Days"):
cnf["Clean-Queues::Options::Days"] = "14"
Arguments = [('h', "help", "Clean-Queues::Options::Help"),
('d', "days", "Clean-Queues::Options::Days", "IntLevel"),
('i', "incoming", "Clean-Queues::Options::Incoming",
"HasArg"),
('n', "no-action", "Clean-Queues::Options::No-Action"),
('v', "verbose", "Clean-Queues::Options::Verbose")]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Clean-Queues::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('clean-queues', Options['No-Action'])
init(cnf)
if Options["Verbose"]:
print "Processing incoming..."
flush_orphans()
reject = cnf["Dir::Reject"]
if os.path.exists(reject) and os.path.isdir(reject):
if Options["Verbose"]:
print "Processing reject directory..."
os.chdir(reject)
flush_old()
Logger.close()
def main ():
global Options, Logger
cnf = Config()
for i in ["Help", "Incoming", "No-Action", "Verbose" ]:
if not cnf.has_key("Clean-Queues::Options::%s" % (i)):
cnf["Clean-Queues::Options::%s" % (i)] = ""
if not cnf.has_key("Clean-Queues::Options::Days"):
cnf["Clean-Queues::Options::Days"] = "14"
Arguments = [('h',"help","Clean-Queues::Options::Help"),
('d',"days","Clean-Queues::Options::Days", "IntLevel"),
('i',"incoming","Clean-Queues::Options::Incoming", "HasArg"),
('n',"no-action","Clean-Queues::Options::No-Action"),
('v',"verbose","Clean-Queues::Options::Verbose")]
apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
Options = cnf.subtree("Clean-Queues::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('clean-queues', Options['No-Action'])
init(cnf)
if Options["Verbose"]:
print "Processing incoming..."
flush_orphans()
reject = cnf["Dir::Reject"]
if os.path.exists(reject) and os.path.isdir(reject):
if Options["Verbose"]:
print "Processing reject directory..."
os.chdir(reject)
flush_old()
Logger.close()
def main():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "Maximum" ]:
if not cnf.has_key("Clean-Suites::Options::%s" % (i)):
cnf["Clean-Suites::Options::%s" % (i)] = ""
Arguments = [('h',"help","Clean-Suites::Options::Help"),
('n',"no-action","Clean-Suites::Options::No-Action"),
('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Clean-Suites::Options")
if cnf["Clean-Suites::Options::Maximum"] != "":
try:
# Only use Maximum if it's an integer
max_delete = int(cnf["Clean-Suites::Options::Maximum"])
if max_delete < 1:
utils.fubar("If given, Maximum must be at least 1")
except ValueError as e:
utils.fubar("If given, Maximum must be an integer")
else:
max_delete = None
if Options["Help"]:
usage()
Logger = daklog.Logger("clean-suites", debug=Options["No-Action"])
session = DBConn().session()
now_date = datetime.now()
# Stay of execution; default to 1.5 days
soe = int(cnf.get('Clean-Suites::StayOfExecution', '129600'))
delete_date = now_date - timedelta(seconds=soe)
check_binaries(now_date, delete_date, max_delete, session)
clean_binaries(now_date, delete_date, max_delete, session)
check_sources(now_date, delete_date, max_delete, session)
check_files(now_date, delete_date, max_delete, session)
clean(now_date, delete_date, max_delete, session)
clean_maintainers(now_date, delete_date, max_delete, session)
clean_fingerprints(now_date, delete_date, max_delete, session)
clean_empty_directories(session)
Logger.close()
def main ():
global Options
cnf = Config()
Arguments = [('h',"help","Auto-Decruft::Options::Help"),
('n',"dry-run","Auto-Decruft::Options::Dry-Run"),
('d',"debug","Auto-Decruft::Options::Debug"),
('s',"suite","Auto-Decruft::Options::Suite","HasArg"),
# The "\0" seems to be the only way to disable short options.
("\0",'if-newer-version-in',"Auto-Decruft::Options::OtherSuite", "HasArg"),
("\0",'if-newer-version-in-rm-msg',"Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg")]
for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg"]:
if not cnf.has_key("Auto-Decruft::Options::%s" % (i)):
cnf["Auto-Decruft::Options::%s" % (i)] = ""
cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Auto-Decruft::Options")
if Options["Help"]:
usage()
debug = False
dryrun = False
if Options["Dry-Run"]:
dryrun = True
if Options["Debug"]:
debug = True
if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]:
utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg")
session = DBConn().session()
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
if Options["OtherSuite"]:
osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name
decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun)
if not dryrun:
session.commit()
def main():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "All"]:
if not cnf.has_key("Manage-Build-Queues::Options::%s" % (i)):
cnf["Manage-Build-Queues::Options::%s" % (i)] = ""
Arguments = [
("h", "help", "Manage-Build-Queues::Options::Help"),
("n", "no-action", "Manage-Build-Queues::Options::No-Action"),
("a", "all", "Manage-Build-Queues::Options::All"),
]
queue_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Manage-Build-Queues::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger("manage-build-queues", Options["No-Action"])
starttime = datetime.now()
session = DBConn().session()
if Options["All"]:
if len(queue_names) != 0:
print "E: Cannot use both -a and a queue_name"
sys.exit(1)
queues = session.query(BuildQueue).all()
else:
queues = []
for q in queue_names:
queue = get_build_queue(q.lower(), session)
if queue:
queues.append(queue)
else:
Logger.log(["cannot find queue %s" % q])
# For each given queue, look up object and call manage_queue
for q in queues:
Logger.log(["cleaning queue %s using datetime %s" % (q.queue_name, starttime)])
q.clean_and_update(starttime, Logger, dryrun=Options["No-Action"])
Logger.close()
def main():
cnf = Config()
Arguments = [('h', "help", "Make-Overrides::Options::Help")]
for i in ["help"]:
if not cnf.has_key("Make-Overrides::Options::%s" % (i)):
cnf["Make-Overrides::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Make-Overrides::Options")
if Options["Help"]:
usage()
d = DBConn()
session = d.session()
for suite in session.query(Suite).filter(Suite.overrideprocess == True):
if suite.untouchable:
print "Skipping %s as it is marked as untouchable" % suite.suite_name
continue
sys.stderr.write("Processing %s...\n" % (suite.suite_name))
override_suite = suite.overridecodename or suite.codename
for component in session.query(Component).all():
for otype in session.query(OverrideType).all():
otype_name = otype.overridetype
cname = component.component_name
# TODO: Stick suffix info in database (or get rid of it)
if otype_name == "deb":
suffix = ""
elif otype_name == "udeb":
if cname == "contrib":
continue # Ick2
suffix = ".debian-installer"
elif otype_name == "dsc":
suffix = ".src"
else:
utils.fubar("Don't understand OverrideType %s" %
otype.overridetype)
cname = cname.replace('/', '_')
filename = os.path.join(
cnf["Dir::Override"],
"override.%s.%s%s" % (override_suite, cname, suffix))
output_file = utils.open_file(filename, 'w')
do_list(output_file, suite, component, otype, session)
output_file.close()
def main():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "All"]:
if not cnf.has_key("Manage-Build-Queues::Options::%s" % (i)):
cnf["Manage-Build-Queues::Options::%s" % (i)] = ""
Arguments = [('h', "help", "Manage-Build-Queues::Options::Help"),
('n', "no-action", "Manage-Build-Queues::Options::No-Action"),
('a', "all", "Manage-Build-Queues::Options::All")]
queue_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Manage-Build-Queues::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('manage-build-queues', Options['No-Action'])
starttime = datetime.now()
session = DBConn().session()
with ArchiveTransaction() as transaction:
session = transaction.session
if Options['All']:
if len(queue_names) != 0:
print "E: Cannot use both -a and a queue name"
sys.exit(1)
queues = session.query(BuildQueue)
else:
queues = session.query(BuildQueue).filter(
BuildQueue.queue_name.in_(queue_names))
for q in queues:
Logger.log([
'cleaning queue %s using datetime %s' %
(q.queue_name, starttime)
])
clean(q, transaction, now=starttime)
if not Options['No-Action']:
transaction.commit()
else:
transaction.rollback()
Logger.close()
def main():
global Options, Logger
cnf = Config()
session = DBConn().session()
Arguments = [('h',"help","Process-Policy::Options::Help"),
('n',"no-action","Process-Policy::Options::No-Action")]
for i in ["help", "no-action"]:
if not cnf.has_key("Process-Policy::Options::%s" % (i)):
cnf["Process-Policy::Options::%s" % (i)] = ""
queue_name = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
if len(queue_name) != 1:
print "E: Specify exactly one policy queue"
sys.exit(1)
queue_name = queue_name[0]
Options = cnf.subtree("Process-Policy::Options")
if Options["Help"]:
usage()
if not Options["No-Action"]:
try:
Logger = daklog.Logger("process-policy")
except CantOpenError as e:
Logger = None
# Find policy queue
session.query(PolicyQueue)
try:
pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
except NoResultFound:
print "E: Cannot find policy queue %s" % queue_name
sys.exit(1)
commentsdir = os.path.join(pq.path, 'COMMENTS')
# The comments stuff relies on being in the right directory
os.chdir(pq.path)
do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, session)
do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, session)
do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, session)
def process_buildinfos(upload):
cnf = Config()
if not cnf.has_key('Dir::BuildinfoArchive'):
return
target_dir = os.path.join(
cnf['Dir::BuildinfoArchive'],
datetime.datetime.now().strftime('%Y/%m/%d'),
)
for f in upload.changes.buildinfo_files:
src = os.path.join(upload.directory, f.filename)
dst = utils.find_next_free(os.path.join(target_dir, f.filename))
Logger.log(["Archiving", f.filename])
upload.transaction.fs.copy(src, dst, mode=0o644)
def main ():
cnf = Config()
Arguments = [('h',"help","Make-Overrides::Options::Help")]
for i in [ "help" ]:
if not cnf.has_key("Make-Overrides::Options::%s" % (i)):
cnf["Make-Overrides::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Make-Overrides::Options")
if Options["Help"]:
usage()
d = DBConn()
session = d.session()
for suite in session.query(Suite).filter(Suite.overrideprocess==True):
if suite.untouchable:
print "Skipping %s as it is marked as untouchable" % suite.suite_name
continue
sys.stderr.write("Processing %s...\n" % (suite.suite_name))
override_suite = suite.overridecodename or suite.codename
for component in session.query(Component).all():
for otype in session.query(OverrideType).all():
otype_name = otype.overridetype
cname = component.component_name
# TODO: Stick suffix info in database (or get rid of it)
if otype_name == "deb":
suffix = ""
elif otype_name == "udeb":
if cname == "contrib":
continue # Ick2
suffix = ".debian-installer"
elif otype_name == "dsc":
suffix = ".src"
else:
utils.fubar("Don't understand OverrideType %s" % otype.overridetype)
cname = cname.replace('/', '_')
filename = os.path.join(cnf["Dir::Override"], "override.%s.%s%s" % (override_suite, cname, suffix))
output_file = utils.open_file(filename, 'w')
do_list(output_file, suite, component, otype, session)
output_file.close()
def main ():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "All"]:
if not cnf.has_key("Manage-Build-Queues::Options::%s" % (i)):
cnf["Manage-Build-Queues::Options::%s" % (i)] = ""
Arguments = [('h',"help","Manage-Build-Queues::Options::Help"),
('n',"no-action","Manage-Build-Queues::Options::No-Action"),
('a',"all","Manage-Build-Queues::Options::All")]
queue_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Manage-Build-Queues::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('manage-build-queues', Options['No-Action'])
starttime = datetime.now()
session = DBConn().session()
with ArchiveTransaction() as transaction:
session = transaction.session
if Options['All']:
if len(queue_names) != 0:
print "E: Cannot use both -a and a queue name"
sys.exit(1)
queues = session.query(BuildQueue)
else:
queues = session.query(BuildQueue).filter(BuildQueue.queue_name.in_(queue_names))
for q in queues:
Logger.log(['cleaning queue %s using datetime %s' % (q.queue_name, starttime)])
clean(q, transaction, now=starttime)
if not Options['No-Action']:
transaction.commit()
else:
transaction.rollback()
Logger.close()
def main():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "All"]:
if not cnf.has_key("Manage-Debug-Suites::Options::%s" % (i)):
cnf["Manage-Debug-Suites::Options::%s" % (i)] = ""
Arguments = [('h', "help", "Manage-Debug-Suites::Options::Help"),
('n', "no-action", "Manage-Debug-Suites::Options::No-Action"),
('a', "all", "Manage-Debug-Suites::Options::All")]
debug_suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Manage-Debug-Suites::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('manage-debug-suites', Options['No-Action'])
with ArchiveTransaction() as transaction:
session = transaction.session
if Options['All']:
if len(debug_suite_names) != 0:
print "E: Cannot use both -a and a queue name"
sys.exit(1)
raise Exception("Not yet implemented.")
else:
debug_suites = session.query(Suite).filter(
Suite.suite_name.in_(debug_suite_names))
for debug_suite in debug_suites:
Logger.log(
['cleaning debug suite {0}'.format(debug_suite.suite_name)])
clean(debug_suite, transaction)
if not Options['No-Action']:
transaction.commit()
else:
transaction.rollback()
Logger.close()
def main ():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "All"]:
if not cnf.has_key("Manage-Debug-Suites::Options::%s" % (i)):
cnf["Manage-Debug-Suites::Options::%s" % (i)] = ""
Arguments = [('h',"help","Manage-Debug-Suites::Options::Help"),
('n',"no-action","Manage-Debug-Suites::Options::No-Action"),
('a',"all","Manage-Debug-Suites::Options::All")]
debug_suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Manage-Debug-Suites::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('manage-debug-suites', Options['No-Action'])
with ArchiveTransaction() as transaction:
session = transaction.session
if Options['All']:
if len(debug_suite_names) != 0:
print "E: Cannot use both -a and a queue name"
sys.exit(1)
raise Exception("Not yet implemented.")
else:
debug_suites = session.query(Suite).filter(Suite.suite_name.in_(debug_suite_names))
for debug_suite in debug_suites:
Logger.log(['cleaning debug suite {0}'.format(debug_suite.suite_name)])
clean(debug_suite, transaction)
if not Options['No-Action']:
transaction.commit()
else:
transaction.rollback()
Logger.close()
def comment_reject(changes_file, srcqueue, comments, session):
u = Upload()
u.pkg.changes_file = changes_file
u.load_changes(changes_file)
u.update_subst()
u.rejects.append(comments)
cnf = Config()
bcc = "X-DAK: dak process-policy"
if cnf.has_key("Dinstall::Bcc"):
u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
else:
u.Subst["__BCC__"] = bcc
if not Options["No-Action"]:
u.do_reject(manual=0, reject_message='\n'.join(u.rejects))
u.pkg.remove_known_changes(session=session)
session.commit()
print " REJECT"
Logger.log(["Policy Queue REJECT: %s: %s" % (srcqueue.queue_name, u.pkg.changes_file)])
def do_update(self):
"""
Add suite options for overrides and control-suite to DB
"""
print __doc__
try:
cnf = Config()
c = self.db.cursor()
c.execute("ALTER TABLE suite ADD COLUMN overrideprocess BOOLEAN NOT NULL DEFAULT FALSE")
c.execute("COMMENT ON COLUMN suite.overrideprocess IS %s", ['If true, check-overrides will process the suite by default'])
c.execute("ALTER TABLE suite ADD COLUMN overrideorigin TEXT DEFAULT NULL")
c.execute("COMMENT ON COLUMN suite.overrideprocess IS %s", ['If NOT NULL, check-overrides will take missing overrides from the named suite'])
# Migrate config file values into database
if cnf.has_key("Check-Overrides::OverrideSuites"):
for suitename in cnf.subtree("Check-Overrides::OverrideSuites").list():
if cnf.get("Check-Overrides::OverrideSuites::%s::Process" % suitename, "0") == "1":
print "Marking %s to have overrides processed automatically" % suitename.lower()
c.execute("UPDATE suite SET overrideprocess = TRUE WHERE suite_name = %s", [suitename.lower()])
originsuite = cnf.get("Check-Overrides::OverrideSuites::%s::OriginSuite" % suitename, '')
if originsuite != '':
print "Setting %s to use %s as origin for overrides" % (suitename.lower(), originsuite.lower())
c.execute("UPDATE suite SET overrideorigin = %s WHERE suite_name = %s", [originsuite.lower(), suitename.lower()])
c.execute("ALTER TABLE suite ADD COLUMN allowcsset BOOLEAN NOT NULL DEFAULT FALSE")
c.execute("COMMENT ON COLUMN suite.allowcsset IS %s", ['Allow control-suite to be used with the --set option without forcing'])
# Import historical hard-coded values
c.execute("UPDATE suite SET allowcsset = TRUE WHERE suite_name IN ('testing', 'squeeze-updates')")
c.execute("UPDATE config SET value = '70' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError('Unable to apply sick update 70, rollback issued. Error message : %s' % (str(msg)))
def main():
cnf = Config()
Arguments = [
('h', "help", "DEP11::Options::Help"),
('e', "expire", "DEP11::Options::ExpireCache"),
('s', "suite", "DEP11::Options::Suite", "HasArg"),
]
for i in ["help", "suite", "ExpireCache"]:
if not cnf.has_key("DEP11::Options::%s" % (i)):
cnf["DEP11::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("DEP11::Options")
if Options["Help"]:
usage()
return
suitename = Options["Suite"]
if not suitename:
print("You need to specify a suite!")
return
logger = daklog.Logger('generate-metadata')
from daklib.dbconn import Component, DBConn, get_suite, Suite
session = DBConn().session()
suite = get_suite(suitename.lower(), session)
if Options["ExpireCache"]:
expire_dep11_data_cache(session, suitename)
process_suite(session, suite, logger)
# export database content as Components-<arch>.xz YAML documents
write_component_files(suite)
# we're done
logger.close()
def init(session):
global cnf, Options
cnf = Config()
Arguments = [('h',"help","Show-New::Options::Help"),
("p","html-path","Show-New::HTMLPath","HasArg")]
for i in ["help"]:
if not cnf.has_key("Show-New::Options::%s" % (i)):
cnf["Show-New::Options::%s" % (i)] = ""
changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
if len(changes_files) == 0:
new_queue = get_policy_queue('new', session );
changes_files = utils.get_changes_files(new_queue.path)
Options = cnf.subtree("Show-New::Options")
if Options["help"]:
usage()
return changes_files
def main():
global Options, Logger
cnf = Config()
session = DBConn().session()
Arguments = [('h',"help","Archive-Dedup-Pool::Options::Help")]
apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
for i in ["help"]:
if not cnf.has_key("Archive-Dedup-Pool::Options::%s" % (i)):
cnf["Archive-Dedup-Pool::Options::%s" % (i)] = ""
Options = cnf.subtree("Archive-Dedup-Pool::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger("archive-dedup-pool")
dedup(session)
Logger.close()
def main():
cnf = Config()
Arguments = [('a', "architecture", "Ls::Options::Architecture", "HasArg"),
('b', "binarytype", "Ls::Options::BinaryType", "HasArg"),
('c', "component", "Ls::Options::Component", "HasArg"),
('f', "format", "Ls::Options::Format", "HasArg"),
('g', "greaterorequal", "Ls::Options::GreaterOrEqual"),
('G', "greaterthan", "Ls::Options::GreaterThan"),
('r', "regex", "Ls::Options::Regex"),
('s', "suite", "Ls::Options::Suite", "HasArg"),
('S', "source-and-binary", "Ls::Options::Source-And-Binary"),
('h', "help", "Ls::Options::Help")]
for i in [
"architecture", "binarytype", "component", "format",
"greaterorequal", "greaterthan", "regex", "suite",
"source-and-binary", "help"
]:
if not cnf.has_key("Ls::Options::%s" % (i)):
cnf["Ls::Options::%s" % (i)] = ""
packages = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Ls::Options")
if Options["Help"]:
usage()
if not packages:
utils.fubar("need at least one package name as an argument.")
session = DBConn().session()
# If cron.daily is running; warn the user that our output might seem strange
if os.path.exists(os.path.join(cnf["Dir::Lock"], "daily.lock")):
utils.warn(
"Archive maintenance is in progress; database inconsistencies are possible."
)
# Handle buildd maintenance helper options
if Options["GreaterOrEqual"] or Options["GreaterThan"]:
if Options["GreaterOrEqual"] and Options["GreaterThan"]:
utils.fubar(
"-g/--greaterorequal and -G/--greaterthan are mutually exclusive."
)
if not Options["Suite"]:
Options["Suite"] = "unstable"
# Parse -a/--architecture, -c/--component and -s/--suite
(con_suites, con_architectures, con_components, check_source) = \
utils.parse_args(Options)
if Options["BinaryType"]:
if Options["BinaryType"] != "udeb" and Options["BinaryType"] != "deb":
utils.fubar("Invalid binary type. 'udeb' and 'deb' recognised.")
con_bintype = "AND b.type = '%s'" % (Options["BinaryType"])
# REMOVE ME TRAMP
if Options["BinaryType"] == "udeb":
check_source = 0
else:
con_bintype = ""
if Options["Regex"]:
comparison_operator = "~"
else:
comparison_operator = "="
if Options["Source-And-Binary"]:
new_packages = []
for package in packages:
q = session.execute(
"SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s :package %s"
% (comparison_operator, con_suites), {'package': package})
new_packages.extend([i[0] for i in q.fetchall()])
if package not in new_packages:
new_packages.append(package)
packages = new_packages
results = 0
for package in packages:
q = session.execute(
"""
SELECT b.package, b.version, a.arch_string, su.suite_name, c.name, m.name
FROM binaries b, architecture a, suite su, bin_associations ba,
files f, files_archive_map af, component c, maintainer m
WHERE b.package %s :package AND a.id = b.architecture AND su.id = ba.suite
AND b.id = ba.bin AND b.file = f.id AND af.file_id = f.id AND su.archive_id = af.archive_id
AND af.component_id = c.id AND b.maintainer = m.id %s %s %s
""" % (comparison_operator, con_suites, con_architectures, con_bintype),
{'package': package})
ql = q.fetchall()
if check_source:
q = session.execute(
"""
SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name
FROM source s, suite su, src_associations sa, files f, files_archive_map af,
component c, maintainer m
WHERE s.source %s :package AND su.id = sa.suite AND s.id = sa.source
AND s.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id
AND s.maintainer = m.id %s
""" % (comparison_operator, con_suites), {'package': package})
if not Options["Architecture"] or con_architectures:
ql.extend(q.fetchall())
else:
ql = q.fetchall()
d = {}
highver = {}
for i in ql:
results += 1
(pkg, version, architecture, suite, component, maintainer) = i
if component != "main":
suite = "%s/%s" % (suite, component)
if not d.has_key(pkg):
d[pkg] = {}
highver.setdefault(pkg, "")
if not d[pkg].has_key(version):
d[pkg][version] = {}
if apt_pkg.version_compare(version, highver[pkg]) > 0:
highver[pkg] = version
if not d[pkg][version].has_key(suite):
d[pkg][version][suite] = []
d[pkg][version][suite].append(architecture)
packages = d.keys()
packages.sort()
# Calculate optimal column sizes
sizes = [10, 13, 10]
for pkg in packages:
versions = d[pkg].keys()
for version in versions:
suites = d[pkg][version].keys()
for suite in suites:
sizes[0] = max(sizes[0], len(pkg))
sizes[1] = max(sizes[1], len(version))
sizes[2] = max(sizes[2], len(suite))
fmt = "%%%is | %%%is | %%%is | " % tuple(sizes)
for pkg in packages:
versions = d[pkg].keys()
versions.sort(apt_pkg.version_compare)
for version in versions:
suites = d[pkg][version].keys()
suites.sort()
for suite in suites:
arches = d[pkg][version][suite]
arches.sort(utils.arch_compare_sw)
if Options["Format"] == "": #normal
sys.stdout.write(fmt % (pkg, version, suite))
sys.stdout.write(", ".join(arches))
sys.stdout.write('\n')
elif Options["Format"] in ["control-suite", "heidi"]:
for arch in arches:
sys.stdout.write("%s %s %s\n" %
(pkg, version, arch))
if Options["GreaterOrEqual"]:
print "\n%s (>= %s)" % (pkg, highver[pkg])
if Options["GreaterThan"]:
print "\n%s (>> %s)" % (pkg, highver[pkg])
if not results:
sys.exit(1)
def main():
global Options
cnf = Config()
Arguments = [
('h', "help", "Rm::Options::Help"),
('a', "architecture", "Rm::Options::Architecture", "HasArg"),
('b', "binary", "Rm::Options::Binary"),
('B', "binary-only", "Rm::Options::Binary-Only"),
('c', "component", "Rm::Options::Component", "HasArg"),
('C', "carbon-copy", "Rm::Options::Carbon-Copy",
"HasArg"), # Bugs to Cc
('d', "done", "Rm::Options::Done", "HasArg"), # Bugs fixed
('D', "do-close", "Rm::Options::Do-Close"),
('R', "rdep-check", "Rm::Options::Rdep-Check"),
(
'm', "reason", "Rm::Options::Reason", "HasArg"
), # Hysterical raisins; -m is old-dinstall option for rejection reason
('n', "no-action", "Rm::Options::No-Action"),
('p', "partial", "Rm::Options::Partial"),
('s', "suite", "Rm::Options::Suite", "HasArg"),
('S', "source-only", "Rm::Options::Source-Only"),
]
for i in [
"architecture", "binary", "binary-only", "carbon-copy",
"component", "done", "help", "no-action", "partial", "rdep-check",
"reason", "source-only", "Do-Close"
]:
if not cnf.has_key("Rm::Options::%s" % (i)):
cnf["Rm::Options::%s" % (i)] = ""
if not cnf.has_key("Rm::Options::Suite"):
cnf["Rm::Options::Suite"] = "unstable"
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Rm::Options")
if Options["Help"]:
usage()
session = DBConn().session()
# Sanity check options
if not arguments:
utils.fubar("need at least one package name as an argument.")
if Options["Architecture"] and Options["Source-Only"]:
utils.fubar(
"can't use -a/--architecture and -S/--source-only options simultaneously."
)
if ((Options["Binary"] and Options["Source-Only"])
or (Options["Binary"] and Options["Binary-Only"])
or (Options["Binary-Only"] and Options["Source-Only"])):
utils.fubar(
"Only one of -b/--binary, -B/--binary-only and -S/--source-only can be used."
)
if Options.has_key("Carbon-Copy") and not Options.has_key("Done"):
utils.fubar(
"can't use -C/--carbon-copy without also using -d/--done option.")
if Options["Architecture"] and not Options["Partial"]:
utils.warn("-a/--architecture implies -p/--partial.")
Options["Partial"] = "true"
if Options["Do-Close"] and not Options["Done"]:
utils.fubar("No.")
if (Options["Do-Close"] and (Options["Binary"] or Options["Binary-Only"]
or Options["Source-Only"])):
utils.fubar("No.")
# Force the admin to tell someone if we're not doing a 'dak
# cruft-report' inspired removal (or closing a bug, which counts
# as telling someone).
if not Options["No-Action"] and not Options["Carbon-Copy"] \
and not Options["Done"] and Options["Reason"].find("[auto-cruft]") == -1:
utils.fubar(
"Need a -C/--carbon-copy if not closing a bug and not doing a cruft removal."
)
# Process -C/--carbon-copy
#
# Accept 3 types of arguments (space separated):
# 1) a number - assumed to be a bug number, i.e. [email protected]
# 2) the keyword 'package' - cc's [email protected] for every argument
# 3) contains a '@' - assumed to be an email address, used unmodified
#
carbon_copy = []
for copy_to in utils.split_args(Options.get("Carbon-Copy")):
if copy_to.isdigit():
if cnf.has_key("Dinstall::BugServer"):
carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
else:
utils.fubar(
"Asked to send mail to #%s in BTS but Dinstall::BugServer is not configured"
% copy_to)
elif copy_to == 'package':
for package in arguments:
if cnf.has_key("Dinstall::PackagesServer"):
carbon_copy.append(package + "@" +
cnf["Dinstall::PackagesServer"])
if cnf.has_key("Dinstall::TrackingServer"):
carbon_copy.append(package + "@" +
cnf["Dinstall::TrackingServer"])
elif '@' in copy_to:
carbon_copy.append(copy_to)
else:
utils.fubar(
"Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address."
% (copy_to))
if Options["Binary"]:
field = "b.package"
else:
field = "s.source"
con_packages = "AND %s IN (%s)" % (field, ", ".join(
[repr(i) for i in arguments]))
(con_suites, con_architectures, con_components, check_source) = \
utils.parse_args(Options)
# Additional suite checks
suite_ids_list = []
whitelists = []
suites = utils.split_args(Options["Suite"])
suites_list = utils.join_with_commas_and(suites)
if not Options["No-Action"]:
for suite in suites:
s = get_suite(suite, session=session)
if s is not None:
suite_ids_list.append(s.suite_id)
whitelists.append(s.mail_whitelist)
if suite in ("oldstable", "stable"):
print "**WARNING** About to remove from the (old)stable suite!"
print "This should only be done just prior to a (point) release and not at"
print "any other time."
game_over()
elif suite == "testing":
print "**WARNING About to remove from the testing suite!"
print "There's no need to do this normally as removals from unstable will"
print "propogate to testing automagically."
game_over()
# Additional architecture checks
if Options["Architecture"] and check_source:
utils.warn("'source' in -a/--argument makes no sense and is ignored.")
# Don't do dependency checks on multiple suites
if Options["Rdep-Check"] and len(suites) > 1:
utils.fubar(
"Reverse dependency check on multiple suites is not implemented.")
to_remove = []
maintainers = {}
# We have 3 modes of package selection: binary, source-only, binary-only
# and source+binary.
# XXX: TODO: This all needs converting to use placeholders or the object
# API. It's an SQL injection dream at the moment
if Options["Binary"]:
# Removal by binary package name
q = session.execute(
"SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, files_archive_map af, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s %s"
% (con_packages, con_suites, con_components, con_architectures))
to_remove.extend(q)
else:
# Source-only
if not Options["Binary-Only"]:
q = session.execute(
"SELECT s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, archive, files f, files_archive_map af, component c WHERE sa.source = s.id AND sa.suite = su.id AND archive.id = su.archive_id AND s.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s"
% (con_packages, con_suites, con_components))
to_remove.extend(q)
if not Options["Source-Only"]:
# Source + Binary
q = session.execute(
"""
SELECT b.package, b.version, a.arch_string, b.id, b.maintainer
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN architecture a ON b.architecture = a.id
JOIN suite su ON ba.suite = su.id
JOIN archive ON archive.id = su.archive_id
JOIN files_archive_map af ON b.file = af.file_id AND af.archive_id = archive.id
JOIN component c ON af.component_id = c.id
JOIN source s ON b.source = s.id
JOIN src_associations sa ON s.id = sa.source AND sa.suite = su.id
WHERE TRUE %s %s %s %s""" %
(con_packages, con_suites, con_components, con_architectures))
to_remove.extend(q)
if not to_remove:
print "Nothing to do."
sys.exit(0)
# If we don't have a reason; spawn an editor so the user can add one
# Write the rejection email out as the <foo>.reason file
if not Options["Reason"] and not Options["No-Action"]:
(fd, temp_filename) = utils.temp_filename()
editor = os.environ.get("EDITOR", "vi")
result = os.system("%s %s" % (editor, temp_filename))
if result != 0:
utils.fubar("vi invocation failed for `%s'!" % (temp_filename),
result)
temp_file = utils.open_file(temp_filename)
for line in temp_file.readlines():
Options["Reason"] += line
temp_file.close()
os.unlink(temp_filename)
# Generate the summary of what's to be removed
d = {}
for i in to_remove:
package = i[0]
version = i[1]
architecture = i[2]
maintainer = i[4]
maintainers[maintainer] = ""
if not d.has_key(package):
d[package] = {}
if not d[package].has_key(version):
d[package][version] = []
if architecture not in d[package][version]:
d[package][version].append(architecture)
maintainer_list = []
for maintainer_id in maintainers.keys():
maintainer_list.append(get_maintainer(maintainer_id).name)
summary = ""
removals = d.keys()
removals.sort()
for package in removals:
versions = d[package].keys()
versions.sort(apt_pkg.version_compare)
for version in versions:
d[package][version].sort(utils.arch_compare_sw)
summary += "%10s | %10s | %s\n" % (package, version, ", ".join(
d[package][version]))
print "Will remove the following packages from %s:" % (suites_list)
print
print summary
print "Maintainer: %s" % ", ".join(maintainer_list)
if Options["Done"]:
print "Will also close bugs: " + Options["Done"]
if carbon_copy:
print "Will also send CCs to: " + ", ".join(carbon_copy)
if Options["Do-Close"]:
print "Will also close associated bug reports."
print
print "------------------- Reason -------------------"
print Options["Reason"]
print "----------------------------------------------"
print
if Options["Rdep-Check"]:
arches = utils.split_args(Options["Architecture"])
reverse_depends_check(removals, suites[0], arches, session)
# If -n/--no-action, drop out here
if Options["No-Action"]:
sys.exit(0)
print "Going to remove the packages now."
game_over()
# Do the actual deletion
print "Deleting...",
sys.stdout.flush()
try:
bugs = utils.split_args(Options["Done"])
remove(session,
Options["Reason"],
suites,
to_remove,
partial=Options["Partial"],
components=utils.split_args(Options["Component"]),
done_bugs=bugs,
carbon_copy=carbon_copy,
close_related_bugs=Options["Do-Close"])
except ValueError as ex:
utils.fubar(ex.message)
else:
print "done."
def main ():
cnf = Config()
Arguments = [('n', "no-action", "Import-Users-From-Passwd::Options::No-Action"),
('q', "quiet", "Import-Users-From-Passwd::Options::Quiet"),
('v', "verbose", "Import-Users-From-Passwd::Options::Verbose"),
('h', "help", "Import-Users-From-Passwd::Options::Help")]
for i in [ "no-action", "quiet", "verbose", "help" ]:
if not cnf.has_key("Import-Users-From-Passwd::Options::%s" % (i)):
cnf["Import-Users-From-Passwd::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Import-Users-From-Passwd::Options")
if Options["Help"]:
usage()
elif arguments:
utils.warn("dak import-users-from-passwd takes no non-option arguments.")
usage(1)
session = DBConn().session()
valid_gid = cnf.get("Import-Users-From-Passwd::ValidGID", "")
if valid_gid:
debiangrp = grp.getgrnam(valid_gid).gr_mem
else:
debiangrp = []
passwd_unames = {}
for entry in pwd.getpwall():
uname = entry[0]
if uname not in debiangrp:
if Options["Verbose"]:
print "Skipping %s (Not in group %s)." % (uname, valid_gid)
continue
passwd_unames[uname] = ""
postgres_unames = {}
q = session.execute("SELECT usename FROM pg_user")
for i in q.fetchall():
uname = i[0]
postgres_unames[uname] = ""
known_postgres_unames = {}
for i in cnf.get("Import-Users-From-Passwd::KnownPostgres","").split(","):
uname = i.strip()
known_postgres_unames[uname] = ""
keys = postgres_unames.keys()
keys.sort()
for uname in keys:
if not passwd_unames.has_key(uname) and not known_postgres_unames.has_key(uname):
print "I: Deleting %s from Postgres, no longer in passwd or list of known Postgres users" % (uname)
q = session.execute('DROP USER "%s"' % (uname))
keys = passwd_unames.keys()
keys.sort()
safe_name = re.compile('^[A-Za-z0-9]+$')
for uname in keys:
if not postgres_unames.has_key(uname):
if not Options["Quiet"]:
print "Creating %s user in Postgres." % (uname)
if not Options["No-Action"]:
if safe_name.match(uname):
# NB: I never figured out how to use a bind parameter for this query
# XXX: Fix this as it looks like a potential SQL injection attack to me
# (hence the safe_name match we do)
try:
q = session.execute('CREATE USER "%s"' % (uname))
session.commit()
except Exception as e:
utils.warn("Could not create user %s (%s)" % (uname, str(e)))
session.rollback()
else:
print "NOT CREATING USER %s. Doesn't match safety regex" % uname
session.commit()
def main():
global Logger, Options, sections, priorities
cnf = Config()
Arguments = [('h', "help", "Check-Overrides::Options::Help"),
('n', "no-action", "Check-Overrides::Options::No-Action")]
for i in ["help", "no-action"]:
if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
cnf["Check-Overrides::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Check-Overrides::Options")
if Options["Help"]:
usage()
session = DBConn().session()
# init sections, priorities:
# We need forward and reverse
sections = get_sections(session)
for name, entry in sections.items():
sections[entry] = name
priorities = get_priorities(session)
for name, entry in priorities.items():
priorities[entry] = name
if not Options["No-Action"]:
Logger = daklog.Logger("check-overrides")
else:
Logger = daklog.Logger("check-overrides", 1)
for suite in session.query(Suite).filter(Suite.overrideprocess == True):
originosuite = None
originremark = ''
if suite.overrideorigin is not None:
originosuite = get_suite(suite.overrideorigin, session)
if originosuite is None:
utils.fubar(
"%s has an override origin suite of %s but it doesn't exist!"
% (suite.suite_name, suite.overrideorigin))
originosuite = originosuite.suite_name
originremark = " taking missing from %s" % originosuite
print "Processing %s%s..." % (suite.suite_name, originremark)
# Get a list of all suites that use the override file of 'suite.suite_name' as
# well as the suite
ocodename = suite.codename
suiteids = [
x.suite_id for x in session.query(Suite).filter(
Suite.overridecodename == ocodename).all()
]
if suite.suite_id not in suiteids:
suiteids.append(suite.suite_id)
if len(suiteids) < 1:
utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
for component in session.query(Component).all():
# It is crucial for the dsc override creation based on binary
# overrides that 'dsc' goes first
component_name = component.component_name
otypes = ['dsc']
for ot in session.query(OverrideType):
if ot.overridetype == 'dsc':
continue
otypes.append(ot.overridetype)
for otype in otypes:
print "Processing %s [%s - %s]" \
% (suite.suite_name, component_name, otype)
sys.stdout.flush()
process(suite.suite_name, suiteids, originosuite,
component_name, otype, session)
Logger.close()
def main():
cnf = Config()
Logger = daklog.Logger('generate-filelist')
Arguments = [('h', "help", "Filelist::Options::Help"),
('s', "suite", "Filelist::Options::Suite", "HasArg"),
('c', "component", "Filelist::Options::Component", "HasArg"),
('a', "architecture", "Filelist::Options::Architecture", "HasArg"),
('i', "incremental", "Filelist::Options::Incremental")]
session = DBConn().session()
query_suites = session.query(Suite)
suites = [suite.suite_name for suite in query_suites]
if not cnf.has_key('Filelist::Options::Suite'):
cnf['Filelist::Options::Suite'] = ','.join(suites).encode()
query_components = session.query(Component)
components = \
[component.component_name for component in query_components]
if not cnf.has_key('Filelist::Options::Component'):
cnf['Filelist::Options::Component'] = ','.join(components).encode()
query_architectures = session.query(Architecture)
architectures = \
[architecture.arch_string for architecture in query_architectures]
if not cnf.has_key('Filelist::Options::Architecture'):
cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode()
cnf['Filelist::Options::Help'] = ''
cnf['Filelist::Options::Incremental'] = ''
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Filelist::Options")
if Options['Help']:
usage()
pool = DakProcessPool()
query_suites = query_suites. \
filter(Suite.suite_name.in_(utils.split_args(Options['Suite'])))
query_components = query_components. \
filter(Component.component_name.in_(utils.split_args(Options['Component'])))
query_architectures = query_architectures. \
filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture'])))
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
Logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
Logger.log(['E: Subprocess recieved signal ', msg])
else:
Logger.log(['E: ', msg])
for suite in query_suites:
suite_id = suite.suite_id
for component in query_components:
component_id = component.component_id
for architecture in query_architectures:
architecture_id = architecture.arch_id
if architecture not in suite.architectures:
pass
elif architecture.arch_string == 'source':
pool.apply_async(writeSourceList,
(suite_id, component_id, Options['Incremental']), callback=parse_results)
elif architecture.arch_string == 'all':
pool.apply_async(writeAllList,
(suite_id, component_id, architecture_id, 'deb',
Options['Incremental']), callback=parse_results)
pool.apply_async(writeAllList,
(suite_id, component_id, architecture_id, 'udeb',
Options['Incremental']), callback=parse_results)
else: # arch any
pool.apply_async(writeBinaryList,
(suite_id, component_id, architecture_id, 'deb',
Options['Incremental']), callback=parse_results)
pool.apply_async(writeBinaryList,
(suite_id, component_id, architecture_id, 'udeb',
Options['Incremental']), callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
Logger.close()
sys.exit(pool.overall_status())
def main():
cnf = Config()
Arguments = [
("h", "help", "Override::Options::Help"),
("c", "check", "Override::Options::Check"),
("d", "done", "Override::Options::Done", "HasArg"),
("n", "no-action", "Override::Options::No-Action"),
("s", "suite", "Override::Options::Suite", "HasArg"),
]
for i in ["help", "check", "no-action"]:
if not cnf.has_key("Override::Options::%s" % (i)):
cnf["Override::Options::%s" % (i)] = ""
if not cnf.has_key("Override::Options::Suite"):
cnf["Override::Options::Suite"] = "unstable"
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Override::Options")
if Options["Help"]:
usage()
session = DBConn().session()
if not arguments:
utils.fubar("package name is a required argument.")
package = arguments.pop(0)
suite_name = Options["Suite"]
if arguments and len(arguments) > 2:
utils.fubar("Too many arguments")
suite = get_suite(suite_name, session)
if suite is None:
utils.fubar("Unknown suite '{0}'".format(suite_name))
if arguments and len(arguments) == 1:
# Determine if the argument is a priority or a section...
arg = arguments.pop()
q = session.execute(
"""
SELECT ( SELECT COUNT(*) FROM section WHERE section = :arg ) AS secs,
( SELECT COUNT(*) FROM priority WHERE priority = :arg ) AS prios
""",
{"arg": arg},
)
r = q.fetchall()
if r[0][0] == 1:
arguments = (arg, ".")
elif r[0][1] == 1:
arguments = (".", arg)
else:
utils.fubar("%s is not a valid section or priority" % (arg))
# Retrieve current section/priority...
oldsection, oldsourcesection, oldpriority = None, None, None
for packagetype in ["source", "binary"]:
eqdsc = "!="
if packagetype == "source":
eqdsc = "="
q = session.execute(
"""
SELECT priority.priority AS prio, section.section AS sect, override_type.type AS type
FROM override, priority, section, suite, override_type
WHERE override.priority = priority.id
AND override.type = override_type.id
AND override_type.type %s 'dsc'
AND override.section = section.id
AND override.package = :package
AND override.suite = suite.id
AND suite.suite_name = :suite_name
"""
% (eqdsc),
{"package": package, "suite_name": suite_name},
)
if q.rowcount == 0:
continue
if q.rowcount > 1:
utils.fubar("%s is ambiguous. Matches %d packages" % (package, q.rowcount))
r = q.fetchone()
if packagetype == "binary":
oldsection = r[1]
oldpriority = r[0]
else:
oldsourcesection = r[1]
oldpriority = "source"
if not oldpriority and not oldsourcesection:
utils.fubar("Unable to find package %s" % (package))
if oldsection and oldsourcesection and oldsection != oldsourcesection:
# When setting overrides, both source & binary will become the same section
utils.warn("Source is in section '%s' instead of '%s'" % (oldsourcesection, oldsection))
if not oldsection:
oldsection = oldsourcesection
if not arguments:
print "%s is in section '%s' at priority '%s'" % (package, oldsection, oldpriority)
sys.exit(0)
# At this point, we have a new section and priority... check they're valid...
newsection, newpriority = arguments
if newsection == ".":
newsection = oldsection
if newpriority == ".":
newpriority = oldpriority
s = get_section(newsection, session)
if s is None:
utils.fubar("Supplied section %s is invalid" % (newsection))
newsecid = s.section_id
p = get_priority(newpriority, session)
if p is None:
utils.fubar("Supplied priority %s is invalid" % (newpriority))
newprioid = p.priority_id
if newpriority == oldpriority and newsection == oldsection:
print "I: Doing nothing"
sys.exit(0)
if oldpriority == "source" and newpriority != "source":
utils.fubar("Trying to change priority of a source-only package")
if Options["Check"] and newpriority != oldpriority:
check_override_compliance(package, p, suite.archive.path, suite_name, cnf, session)
# If we're in no-action mode
if Options["No-Action"]:
if newpriority != oldpriority:
print "I: Would change priority from %s to %s" % (oldpriority, newpriority)
if newsection != oldsection:
print "I: Would change section from %s to %s" % (oldsection, newsection)
if Options.has_key("Done"):
print "I: Would also close bug(s): %s" % (Options["Done"])
sys.exit(0)
if newpriority != oldpriority:
print "I: Will change priority from %s to %s" % (oldpriority, newpriority)
if newsection != oldsection:
print "I: Will change section from %s to %s" % (oldsection, newsection)
if not Options.has_key("Done"):
pass
# utils.warn("No bugs to close have been specified. Noone will know you have done this.")
else:
print "I: Will close bug(s): %s" % (Options["Done"])
game_over()
Logger = daklog.Logger("override")
dsc_otype_id = get_override_type("dsc").overridetype_id
# We're already in a transaction
# We're in "do it" mode, we have something to do... do it
if newpriority != oldpriority:
session.execute(
"""
UPDATE override
SET priority = :newprioid
WHERE package = :package
AND override.type != :otypedsc
AND suite = (SELECT id FROM suite WHERE suite_name = :suite_name)""",
{"newprioid": newprioid, "package": package, "otypedsc": dsc_otype_id, "suite_name": suite_name},
)
Logger.log(["changed priority", package, oldpriority, newpriority])
if newsection != oldsection:
q = session.execute(
"""
UPDATE override
SET section = :newsecid
WHERE package = :package
AND suite = (SELECT id FROM suite WHERE suite_name = :suite_name)""",
{"newsecid": newsecid, "package": package, "suite_name": suite_name},
)
Logger.log(["changed section", package, oldsection, newsection])
session.commit()
if Options.has_key("Done"):
if not cnf.has_key("Dinstall::BugServer"):
utils.warn("Asked to send Done message but Dinstall::BugServer is not configured")
Logger.close()
return
Subst = {}
Subst["__OVERRIDE_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
bcc = []
if cnf.find("Dinstall::Bcc") != "":
bcc.append(cnf["Dinstall::Bcc"])
if bcc:
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst["__BCC__"] = "X-Filler: 42"
if cnf.has_key("Dinstall::PackagesServer"):
Subst["__CC__"] = "Cc: " + package + "@" + cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override"
else:
Subst["__CC__"] = "X-DAK: dak override"
Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst["__WHOAMI__"] = utils.whoami()
Subst["__SOURCE__"] = package
summary = "Concerning package %s...\n" % (package)
summary += "Operating on the %s suite\n" % (suite_name)
if newpriority != oldpriority:
summary += "Changed priority from %s to %s\n" % (oldpriority, newpriority)
if newsection != oldsection:
summary += "Changed section from %s to %s\n" % (oldsection, newsection)
Subst["__SUMMARY__"] = summary
template = os.path.join(cnf["Dir::Templates"], "override.bug-close")
for bug in utils.split_args(Options["Done"]):
Subst["__BUG_NUMBER__"] = bug
mail_message = utils.TemplateSubst(Subst, template)
utils.send_mail(mail_message)
Logger.log(["closed bug", bug])
Logger.close()
def main():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h', "help", "Cruft-Report::Options::Help"),
('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
('w', "wanna-build-dump",
"Cruft-Report::Options::Wanna-Build-Dump", "HasArg")]
for i in ["help", "Rdep-Check"]:
if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
cnf["Cruft-Report::Options::%s" % (i)] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite",
"unstable")
if not cnf.has_key("Cruft-Report::Options::Mode"):
cnf["Cruft-Report::Options::Mode"] = "daily"
if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu"
]
elif Options["Mode"] == "full":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu", "dubious nbs", "bnb", "bms", "anais"
]
elif Options["Mode"] == "bdo":
checks = ["nbs", "obsolete source"]
else:
utils.warn(
"%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood."
% (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path,
suite_name, component)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
(filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
sources = utils.open_file(temp_filename)
Sources = apt_pkg.TagFile(sources)
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [i.strip() for i in binaries.split(',')]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if not bins_in_suite.has_key(binary):
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list, source,
session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
sources.close()
os.unlink(temp_filename)
# Checks based on the Packages files
check_components = components[:]
if suite_name != "staging":
check_components.append('main/debian-installer')
for component in check_components:
architectures = [
a.arch_string for a in get_suite_architectures(
suite_name, skipsrc=True, skipall=True, session=session)
]
for architecture in architectures:
if component == 'main/debian-installer' and re.match(
"kfreebsd", architecture):
continue
filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (
suite.archive.path, suite_name, component, architecture)
# apt_pkg.TagFile needs a real file handle
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput(
"gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
if "nfu" in checks:
nfu_packages.setdefault(architecture, [])
nfu_entries = parse_nfu(architecture)
packages = utils.open_file(temp_filename)
Packages = apt_pkg.TagFile(packages)
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if bin2source.has_key(package) and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if not bin_pkgs.has_key(package):
nbs.setdefault(source, {})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append(
(package, version, source_versions[source]))
packages.close()
os.unlink(temp_filename)
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
for source in nbs.keys():
for package in nbs[source].keys():
versions = nbs[source][package].keys()
versions.sort(apt_pkg.version_compare)
latest_version = versions.pop()
source_version = source_versions.get(source, "0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id,
session)
if "nviu" in checks:
do_newer_version('chromodoris', 'staging', 'NVIU', session)
# FIXME: Not used in Tanglu
#if "nvit" in checks:
# do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
###
if Options["Mode"] == "full":
print "=" * 75
print
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print "Unbuilt binary packages"
print "-----------------------"
print
keys = bin_not_built.keys()
keys.sort()
for source in keys:
binaries = bin_not_built[source].keys()
binaries.sort()
print " o %s: %s" % (source, ", ".join(binaries))
print
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print "Architecture Not Allowed In Source"
print "----------------------------------"
print anais_output
print
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)
def main():
global Logger
cnf = Config()
Arguments = [('a', "add", "Control-Overrides::Options::Add"),
('c', "component", "Control-Overrides::Options::Component",
"HasArg"), ('h', "help", "Control-Overrides::Options::Help"),
('l', "list", "Control-Overrides::Options::List"),
('q', "quiet", "Control-Overrides::Options::Quiet"),
('s', "suite", "Control-Overrides::Options::Suite", "HasArg"),
('S', "set", "Control-Overrides::Options::Set"),
('C', "change", "Control-Overrides::Options::Change"),
('n', "no-action", "Control-Overrides::Options::No-Action"),
('t', "type", "Control-Overrides::Options::Type", "HasArg")]
# Default arguments
for i in ["add", "help", "list", "quiet", "set", "change", "no-action"]:
if not cnf.has_key("Control-Overrides::Options::%s" % (i)):
cnf["Control-Overrides::Options::%s" % (i)] = ""
if not cnf.has_key("Control-Overrides::Options::Component"):
cnf["Control-Overrides::Options::Component"] = "main"
if not cnf.has_key("Control-Overrides::Options::Suite"):
cnf["Control-Overrides::Options::Suite"] = "unstable"
if not cnf.has_key("Control-Overrides::Options::Type"):
cnf["Control-Overrides::Options::Type"] = "deb"
file_list = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
if cnf["Control-Overrides::Options::Help"]:
usage()
session = DBConn().session()
mode = None
for i in ["add", "list", "set", "change"]:
if cnf["Control-Overrides::Options::%s" % (i)]:
if mode:
utils.fubar("Can not perform more than one action at once.")
mode = i
# Need an action...
if mode is None:
utils.fubar("No action specified.")
(suite, component, otype) = (cnf["Control-Overrides::Options::Suite"],
cnf["Control-Overrides::Options::Component"],
cnf["Control-Overrides::Options::Type"])
if mode == "list":
list_overrides(suite, component, otype, session)
else:
if get_suite(suite).untouchable:
utils.fubar("%s: suite is untouchable" % suite)
action = True
if cnf["Control-Overrides::Options::No-Action"]:
utils.warn("In No-Action Mode")
action = False
Logger = daklog.Logger("control-overrides", mode)
if file_list:
for f in file_list:
process_file(utils.open_file(f), suite, component, otype, mode,
action, session)
else:
process_file(sys.stdin, suite, component, otype, mode, action,
session)
Logger.close()
def main():
global Options, Logger
cnf = Config()
summarystats = SummaryStats()
Arguments = [('a', "automatic", "Dinstall::Options::Automatic"),
('h', "help", "Dinstall::Options::Help"),
('n', "no-action", "Dinstall::Options::No-Action"),
('p', "no-lock", "Dinstall::Options::No-Lock"),
('s', "no-mail", "Dinstall::Options::No-Mail"),
('d', "directory", "Dinstall::Options::Directory", "HasArg")]
for i in [
"automatic", "help", "no-action", "no-lock", "no-mail", "version",
"directory"
]:
if not cnf.has_key("Dinstall::Options::%s" % (i)):
cnf["Dinstall::Options::%s" % (i)] = ""
changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Dinstall::Options")
if Options["Help"]:
usage()
# -n/--dry-run invalidates some other options which would involve things happening
if Options["No-Action"]:
Options["Automatic"] = ""
# Check that we aren't going to clash with the daily cron job
if not Options["No-Action"] and os.path.exists(
"%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
utils.fubar("Archive maintenance in progress. Try again later.")
# Obtain lock if not in no-action mode and initialize the log
if not Options["No-Action"]:
lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'),
os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[
e.errno] == 'EAGAIN':
utils.fubar(
"Couldn't obtain lock; assuming another 'dak process-upload' is already running."
)
else:
raise
# Initialise UrgencyLog() - it will deal with the case where we don't
# want to log urgencies
urgencylog = UrgencyLog()
Logger = daklog.Logger("process-upload", Options["No-Action"])
# If we have a directory flag, use it to find our files
if cnf["Dinstall::Options::Directory"] != "":
# Note that we clobber the list of files we were given in this case
# so warn if the user has done both
if len(changes_files) > 0:
utils.warn(
"Directory provided so ignoring files given on command line")
changes_files = utils.get_changes_files(
cnf["Dinstall::Options::Directory"])
Logger.log([
"Using changes files from directory",
cnf["Dinstall::Options::Directory"],
len(changes_files)
])
elif not len(changes_files) > 0:
utils.fubar("No changes files given and no directory specified")
else:
Logger.log(
["Using changes files from command-line",
len(changes_files)])
process_changes(changes_files)
if summarystats.accept_count:
sets = "set"
if summarystats.accept_count > 1:
sets = "sets"
print "Installed %d package %s, %s." % (
summarystats.accept_count, sets,
utils.size_type(int(summarystats.accept_bytes)))
Logger.log(
["total", summarystats.accept_count, summarystats.accept_bytes])
if summarystats.reject_count:
sets = "set"
if summarystats.reject_count > 1:
sets = "sets"
print "Rejected %d package %s." % (summarystats.reject_count, sets)
Logger.log(["rejected", summarystats.reject_count])
if not Options["No-Action"]:
urgencylog.close()
Logger.close()
def main():
global Logger
cnf = Config()
for i in ["Help", "Suite", "Force", "Quiet"]:
if not cnf.has_key("Generate-Releases::Options::%s" % (i)):
cnf["Generate-Releases::Options::%s" % (i)] = ""
Arguments = [('h', "help", "Generate-Releases::Options::Help"),
('a', 'archive',
'Generate-Releases::Options::Archive', 'HasArg'),
('s', "suite", "Generate-Releases::Options::Suite"),
('f', "force", "Generate-Releases::Options::Force"),
('q', "quiet", "Generate-Releases::Options::Quiet"),
('o', 'option', '', 'ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Generate-Releases::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger('generate-releases')
pool = DakProcessPool()
session = DBConn().session()
if Options["Suite"]:
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "cannot find suite %s" % s
Logger.log(['cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
query = query.join(Suite.archive).filter(
Archive.archive_name == Options['Archive'])
suites = query.all()
broken = []
for s in suites:
# Setup a multiprocessing Pool. As many workers as we have CPU cores.
if s.untouchable and not Options["Force"]:
print "Skipping %s (untouchable)" % s.suite_name
continue
if not Options["Quiet"]:
print "Processing %s" % s.suite_name
Logger.log(['Processing release file for Suite: %s' % (s.suite_name)])
pool.apply_async(generate_helper, (s.suite_id, ))
# No more work will be added to our pool, close it and then wait for all to finish
pool.close()
pool.join()
retcode = pool.overall_status()
if retcode > 0:
# TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING
Logger.log([
'Release file generation broken: %s' %
(','.join([str(x[1]) for x in pool.results]))
])
Logger.close()
sys.exit(retcode)
def main ():
global Options
cnf = Config()
Arguments = [('h',"help","Rm::Options::Help"),
('A','no-arch-all-rdeps','Rm::Options::NoArchAllRdeps'),
('a',"architecture","Rm::Options::Architecture", "HasArg"),
('b',"binary", "Rm::Options::Binary"),
('B',"binary-only", "Rm::Options::Binary-Only"),
('c',"component", "Rm::Options::Component", "HasArg"),
('C',"carbon-copy", "Rm::Options::Carbon-Copy", "HasArg"), # Bugs to Cc
('d',"done","Rm::Options::Done", "HasArg"), # Bugs fixed
('D',"do-close","Rm::Options::Do-Close"),
('R',"rdep-check", "Rm::Options::Rdep-Check"),
('m',"reason", "Rm::Options::Reason", "HasArg"), # Hysterical raisins; -m is old-dinstall option for rejection reason
('n',"no-action","Rm::Options::No-Action"),
('p',"partial", "Rm::Options::Partial"),
('s',"suite","Rm::Options::Suite", "HasArg"),
('S',"source-only", "Rm::Options::Source-Only"),
]
for i in [ 'NoArchAllRdeps',
"architecture", "binary", "binary-only", "carbon-copy", "component",
"done", "help", "no-action", "partial", "rdep-check", "reason",
"source-only", "Do-Close" ]:
if not cnf.has_key("Rm::Options::%s" % (i)):
cnf["Rm::Options::%s" % (i)] = ""
if not cnf.has_key("Rm::Options::Suite"):
cnf["Rm::Options::Suite"] = "unstable"
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Rm::Options")
if Options["Help"]:
usage()
session = DBConn().session()
# Sanity check options
if not arguments:
utils.fubar("need at least one package name as an argument.")
if Options["Architecture"] and Options["Source-Only"]:
utils.fubar("can't use -a/--architecture and -S/--source-only options simultaneously.")
if ((Options["Binary"] and Options["Source-Only"])
or (Options["Binary"] and Options["Binary-Only"])
or (Options["Binary-Only"] and Options["Source-Only"])):
utils.fubar("Only one of -b/--binary, -B/--binary-only and -S/--source-only can be used.")
if Options.has_key("Carbon-Copy") and not Options.has_key("Done"):
utils.fubar("can't use -C/--carbon-copy without also using -d/--done option.")
if Options["Architecture"] and not Options["Partial"]:
utils.warn("-a/--architecture implies -p/--partial.")
Options["Partial"] = "true"
if Options["Do-Close"] and not Options["Done"]:
utils.fubar("No.")
if (Options["Do-Close"]
and (Options["Binary"] or Options["Binary-Only"] or Options["Source-Only"])):
utils.fubar("No.")
# Force the admin to tell someone if we're not doing a 'dak
# cruft-report' inspired removal (or closing a bug, which counts
# as telling someone).
if not Options["No-Action"] and not Options["Carbon-Copy"] \
and not Options["Done"] and Options["Reason"].find("[auto-cruft]") == -1:
utils.fubar("Need a -C/--carbon-copy if not closing a bug and not doing a cruft removal.")
# Process -C/--carbon-copy
#
# Accept 3 types of arguments (space separated):
# 1) a number - assumed to be a bug number, i.e. [email protected]
# 2) the keyword 'package' - cc's [email protected] for every argument
# 3) contains a '@' - assumed to be an email address, used unmodified
#
carbon_copy = []
for copy_to in utils.split_args(Options.get("Carbon-Copy")):
if copy_to.isdigit():
if cnf.has_key("Dinstall::BugServer"):
carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
else:
utils.fubar("Asked to send mail to #%s in BTS but Dinstall::BugServer is not configured" % copy_to)
elif copy_to == 'package':
for package in arguments:
if cnf.has_key("Dinstall::PackagesServer"):
carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"])
if cnf.has_key("Dinstall::TrackingServer"):
carbon_copy.append(package + "@" + cnf["Dinstall::TrackingServer"])
elif '@' in copy_to:
carbon_copy.append(copy_to)
else:
utils.fubar("Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address." % (copy_to))
if Options["Binary"]:
field = "b.package"
else:
field = "s.source"
con_packages = "AND %s IN (%s)" % (field, ", ".join([ repr(i) for i in arguments ]))
(con_suites, con_architectures, con_components, check_source) = \
utils.parse_args(Options)
# Additional suite checks
suite_ids_list = []
whitelists = []
suites = utils.split_args(Options["Suite"])
suites_list = utils.join_with_commas_and(suites)
if not Options["No-Action"]:
for suite in suites:
s = get_suite(suite, session=session)
if s is not None:
suite_ids_list.append(s.suite_id)
whitelists.append(s.mail_whitelist)
if suite in ("oldstable", "stable"):
print "**WARNING** About to remove from the (old)stable suite!"
print "This should only be done just prior to a (point) release and not at"
print "any other time."
game_over()
elif suite == "testing":
print "**WARNING About to remove from the testing suite!"
print "There's no need to do this normally as removals from unstable will"
print "propogate to testing automagically."
game_over()
# Additional architecture checks
if Options["Architecture"] and check_source:
utils.warn("'source' in -a/--argument makes no sense and is ignored.")
# Don't do dependency checks on multiple suites
if Options["Rdep-Check"] and len(suites) > 1:
utils.fubar("Reverse dependency check on multiple suites is not implemented.")
to_remove = []
maintainers = {}
# We have 3 modes of package selection: binary, source-only, binary-only
# and source+binary.
# XXX: TODO: This all needs converting to use placeholders or the object
# API. It's an SQL injection dream at the moment
if Options["Binary"]:
# Removal by binary package name
q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, files_archive_map af, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
to_remove.extend(q)
else:
# Source-only
if not Options["Binary-Only"]:
q = session.execute("SELECT s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, archive, files f, files_archive_map af, component c WHERE sa.source = s.id AND sa.suite = su.id AND archive.id = su.archive_id AND s.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s" % (con_packages, con_suites, con_components))
to_remove.extend(q)
if not Options["Source-Only"]:
# Source + Binary
q = session.execute("""
SELECT b.package, b.version, a.arch_string, b.id, b.maintainer
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN architecture a ON b.architecture = a.id
JOIN suite su ON ba.suite = su.id
JOIN archive ON archive.id = su.archive_id
JOIN files_archive_map af ON b.file = af.file_id AND af.archive_id = archive.id
JOIN component c ON af.component_id = c.id
JOIN source s ON b.source = s.id
JOIN src_associations sa ON s.id = sa.source AND sa.suite = su.id
WHERE TRUE %s %s %s %s""" % (con_packages, con_suites, con_components, con_architectures))
to_remove.extend(q)
if not to_remove:
print "Nothing to do."
sys.exit(0)
# If we don't have a reason; spawn an editor so the user can add one
# Write the rejection email out as the <foo>.reason file
if not Options["Reason"] and not Options["No-Action"]:
(fd, temp_filename) = utils.temp_filename()
editor = os.environ.get("EDITOR","vi")
result = os.system("%s %s" % (editor, temp_filename))
if result != 0:
utils.fubar ("vi invocation failed for `%s'!" % (temp_filename), result)
temp_file = utils.open_file(temp_filename)
for line in temp_file.readlines():
Options["Reason"] += line
temp_file.close()
os.unlink(temp_filename)
# Generate the summary of what's to be removed
d = {}
for i in to_remove:
package = i[0]
version = i[1]
architecture = i[2]
maintainer = i[4]
maintainers[maintainer] = ""
if not d.has_key(package):
d[package] = {}
if not d[package].has_key(version):
d[package][version] = []
if architecture not in d[package][version]:
d[package][version].append(architecture)
maintainer_list = []
for maintainer_id in maintainers.keys():
maintainer_list.append(get_maintainer(maintainer_id).name)
summary = ""
removals = d.keys()
removals.sort()
for package in removals:
versions = d[package].keys()
versions.sort(apt_pkg.version_compare)
for version in versions:
d[package][version].sort(utils.arch_compare_sw)
summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
print "Will remove the following packages from %s:" % (suites_list)
print
print summary
print "Maintainer: %s" % ", ".join(maintainer_list)
if Options["Done"]:
print "Will also close bugs: "+Options["Done"]
if carbon_copy:
print "Will also send CCs to: " + ", ".join(carbon_copy)
if Options["Do-Close"]:
print "Will also close associated bug reports."
print
print "------------------- Reason -------------------"
print Options["Reason"]
print "----------------------------------------------"
print
if Options["Rdep-Check"]:
arches = utils.split_args(Options["Architecture"])
include_arch_all = Options['NoArchAllRdeps'] == ''
reverse_depends_check(removals, suites[0], arches, session, include_arch_all=include_arch_all)
# If -n/--no-action, drop out here
if Options["No-Action"]:
sys.exit(0)
print "Going to remove the packages now."
game_over()
# Do the actual deletion
print "Deleting...",
sys.stdout.flush()
try:
bugs = utils.split_args(Options["Done"])
remove(session, Options["Reason"], suites, to_remove,
partial=Options["Partial"], components=utils.split_args(Options["Component"]),
done_bugs=bugs, carbon_copy=carbon_copy, close_related_bugs=Options["Do-Close"]
)
except ValueError as ex:
utils.fubar(ex.message)
else:
print "done."
def main():
global Options, Logger
cnf = Config()
for i in ["Help", "No-Action", "Maximum"]:
if not cnf.has_key("Clean-Suites::Options::%s" % (i)):
cnf["Clean-Suites::Options::%s" % (i)] = ""
Arguments = [('h', "help", "Clean-Suites::Options::Help"),
('a', 'archive', 'Clean-Suites::Options::Archive', 'HasArg'),
('n', "no-action", "Clean-Suites::Options::No-Action"),
('m', "maximum", "Clean-Suites::Options::Maximum", "HasArg")]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Clean-Suites::Options")
if cnf["Clean-Suites::Options::Maximum"] != "":
try:
# Only use Maximum if it's an integer
max_delete = int(cnf["Clean-Suites::Options::Maximum"])
if max_delete < 1:
utils.fubar("If given, Maximum must be at least 1")
except ValueError as e:
utils.fubar("If given, Maximum must be an integer")
else:
max_delete = None
if Options["Help"]:
usage()
program = "clean-suites"
if Options['No-Action']:
program = "clean-suites (no action)"
Logger = daklog.Logger(program, debug=Options["No-Action"])
session = DBConn().session()
archives = None
if 'Archive' in Options:
archive_names = Options['Archive'].split(',')
archives = session.query(Archive).filter(
Archive.archive_name.in_(archive_names)).all()
if len(archives) == 0:
utils.fubar('Unknown archive.')
now_date = datetime.now()
set_archive_delete_dates(now_date, session)
check_binaries(now_date, session)
clean_binaries(now_date, session)
check_sources(now_date, session)
check_files(now_date, session)
clean(now_date, archives, max_delete, session)
clean_maintainers(now_date, session)
clean_fingerprints(now_date, session)
clean_empty_directories(session)
session.rollback()
Logger.close()
def main ():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h',"help","Cruft-Report::Options::Help"),
('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
('R',"rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s',"suite","Cruft-Report::Options::Suite","HasArg"),
('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
for i in [ "help", "Rdep-Check" ]:
if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
cnf["Cruft-Report::Options::%s" % (i)] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
if not cnf.has_key("Cruft-Report::Options::Mode"):
cnf["Cruft-Report::Options::Mode"] = "daily"
if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu" ]
elif Options["Mode"] == "full":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
elif Options["Mode"] == "bdo":
checks = [ "nbs", "obsolete source" ]
else:
utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path, suite_name, component)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
sources = utils.open_file(temp_filename)
Sources = apt_pkg.TagFile(sources)
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [ i.strip() for i in binaries.split(',') ]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if not bins_in_suite.has_key(binary):
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list, source, session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
sources.close()
os.unlink(temp_filename)
# Checks based on the Packages files
check_components = components[:]
if suite_name != "experimental":
check_components.append('main/debian-installer');
for component in check_components:
architectures = [ a.arch_string for a in get_suite_architectures(suite_name,
skipsrc=True, skipall=True,
session=session) ]
for architecture in architectures:
if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
continue
filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (suite.archive.path, suite_name, component, architecture)
# apt_pkg.TagFile needs a real file handle
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
if "nfu" in checks:
nfu_packages.setdefault(architecture,[])
nfu_entries = parse_nfu(architecture)
packages = utils.open_file(temp_filename)
Packages = apt_pkg.TagFile(packages)
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if bin2source.has_key(package) and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if not bin_pkgs.has_key(package):
nbs.setdefault(source,{})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append((package,version,source_versions[source]))
packages.close()
os.unlink(temp_filename)
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
for source in nbs.keys():
for package in nbs[source].keys():
versions = nbs[source][package].keys()
versions.sort(apt_pkg.version_compare)
latest_version = versions.pop()
source_version = source_versions.get(source,"0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
if "nviu" in checks:
do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nvit" in checks:
do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
###
if Options["Mode"] == "full":
print "="*75
print
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print "Unbuilt binary packages"
print "-----------------------"
print
keys = bin_not_built.keys()
keys.sort()
for source in keys:
binaries = bin_not_built[source].keys()
binaries.sort()
print " o %s: %s" % (source, ", ".join(binaries))
print
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print "Architecture Not Allowed In Source"
print "----------------------------------"
print anais_output
print
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)
def main ():
cnf = Config()
Arguments = [('a', "architecture", "Ls::Options::Architecture", "HasArg"),
('b', "binarytype", "Ls::Options::BinaryType", "HasArg"),
('c', "component", "Ls::Options::Component", "HasArg"),
('f', "format", "Ls::Options::Format", "HasArg"),
('g', "greaterorequal", "Ls::Options::GreaterOrEqual"),
('G', "greaterthan", "Ls::Options::GreaterThan"),
('r', "regex", "Ls::Options::Regex"),
('s', "suite", "Ls::Options::Suite", "HasArg"),
('S', "source-and-binary", "Ls::Options::Source-And-Binary"),
('h', "help", "Ls::Options::Help")]
for i in [ "architecture", "binarytype", "component", "format",
"greaterorequal", "greaterthan", "regex", "suite",
"source-and-binary", "help" ]:
if not cnf.has_key("Ls::Options::%s" % (i)):
cnf["Ls::Options::%s" % (i)] = ""
packages = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Ls::Options")
if Options["Help"]:
usage()
if not packages:
utils.fubar("need at least one package name as an argument.")
# If cron.daily is running; warn the user that our output might seem strange
if os.path.exists(os.path.join(cnf["Dir::Lock"], "daily.lock")):
utils.warn("Archive maintenance is in progress; database inconsistencies are possible.")
# Handle buildd maintenance helper options
if Options["GreaterOrEqual"] or Options["GreaterThan"]:
if Options["GreaterOrEqual"] and Options["GreaterThan"]:
utils.fubar("-g/--greaterorequal and -G/--greaterthan are mutually exclusive.")
if not Options["Suite"]:
Options["Suite"] = "unstable"
kwargs = dict()
if Options["Regex"]:
kwargs['regex'] = True
if Options["Source-And-Binary"]:
kwargs['source_and_binary'] = True
if Options["Suite"]:
kwargs['suites'] = utils.split_args(Options['Suite'])
if Options["Architecture"]:
kwargs['architectures'] = utils.split_args(Options['Architecture'])
if Options['BinaryType']:
kwargs['binary_types'] = utils.split_args(Options['BinaryType'])
if Options['Component']:
kwargs['components'] = utils.split_args(Options['Component'])
if Options['Format']:
kwargs['format'] = Options['Format']
if Options['GreaterOrEqual']:
kwargs['highest'] = '>='
elif Options['GreaterThan']:
kwargs['highest'] = '>>'
for line in list_packages(packages, **kwargs):
print(line)
def main():
cnf = Config()
Arguments = [
('h', "help", "Override::Options::Help"),
('c', "check", "Override::Options::Check"),
('d', "done", "Override::Options::Done", "HasArg"),
('n', "no-action", "Override::Options::No-Action"),
('s', "suite", "Override::Options::Suite", "HasArg"),
]
for i in ["help", "check", "no-action"]:
if not cnf.has_key("Override::Options::%s" % (i)):
cnf["Override::Options::%s" % (i)] = ""
if not cnf.has_key("Override::Options::Suite"):
cnf["Override::Options::Suite"] = "unstable"
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Override::Options")
if Options["Help"]:
usage()
session = DBConn().session()
if not arguments:
utils.fubar("package name is a required argument.")
package = arguments.pop(0)
suite_name = Options["Suite"]
if arguments and len(arguments) > 2:
utils.fubar("Too many arguments")
suite = get_suite(suite_name, session)
if suite is None:
utils.fubar("Unknown suite '{0}'".format(suite_name))
if arguments and len(arguments) == 1:
# Determine if the argument is a priority or a section...
arg = arguments.pop()
q = session.execute(
"""
SELECT ( SELECT COUNT(*) FROM section WHERE section = :arg ) AS secs,
( SELECT COUNT(*) FROM priority WHERE priority = :arg ) AS prios
""", {'arg': arg})
r = q.fetchall()
if r[0][0] == 1:
arguments = (arg, ".")
elif r[0][1] == 1:
arguments = (".", arg)
else:
utils.fubar("%s is not a valid section or priority" % (arg))
# Retrieve current section/priority...
oldsection, oldsourcesection, oldpriority = None, None, None
for packagetype in ['source', 'binary']:
eqdsc = '!='
if packagetype == 'source':
eqdsc = '='
q = session.execute(
"""
SELECT priority.priority AS prio, section.section AS sect, override_type.type AS type
FROM override, priority, section, suite, override_type
WHERE override.priority = priority.id
AND override.type = override_type.id
AND override_type.type %s 'dsc'
AND override.section = section.id
AND override.package = :package
AND override.suite = suite.id
AND suite.suite_name = :suite_name
""" % (eqdsc), {
'package': package,
'suite_name': suite_name
})
if q.rowcount == 0:
continue
if q.rowcount > 1:
utils.fubar("%s is ambiguous. Matches %d packages" %
(package, q.rowcount))
r = q.fetchone()
if packagetype == 'binary':
oldsection = r[1]
oldpriority = r[0]
else:
oldsourcesection = r[1]
oldpriority = 'source'
if not oldpriority and not oldsourcesection:
utils.fubar("Unable to find package %s" % (package))
if oldsection and oldsourcesection and oldsection != oldsourcesection:
# When setting overrides, both source & binary will become the same section
utils.warn("Source is in section '%s' instead of '%s'" %
(oldsourcesection, oldsection))
if not oldsection:
oldsection = oldsourcesection
if not arguments:
print "%s is in section '%s' at priority '%s'" % (package, oldsection,
oldpriority)
sys.exit(0)
# At this point, we have a new section and priority... check they're valid...
newsection, newpriority = arguments
if newsection == ".":
newsection = oldsection
if newpriority == ".":
newpriority = oldpriority
s = get_section(newsection, session)
if s is None:
utils.fubar("Supplied section %s is invalid" % (newsection))
newsecid = s.section_id
p = get_priority(newpriority, session)
if p is None:
utils.fubar("Supplied priority %s is invalid" % (newpriority))
newprioid = p.priority_id
if newpriority == oldpriority and newsection == oldsection:
print "I: Doing nothing"
sys.exit(0)
if oldpriority == 'source' and newpriority != 'source':
utils.fubar("Trying to change priority of a source-only package")
if Options["Check"] and newpriority != oldpriority:
check_override_compliance(package, p, suite.archive.path, suite_name,
cnf, session)
# If we're in no-action mode
if Options["No-Action"]:
if newpriority != oldpriority:
print "I: Would change priority from %s to %s" % (oldpriority,
newpriority)
if newsection != oldsection:
print "I: Would change section from %s to %s" % (oldsection,
newsection)
if Options.has_key("Done"):
print "I: Would also close bug(s): %s" % (Options["Done"])
sys.exit(0)
if newpriority != oldpriority:
print "I: Will change priority from %s to %s" % (oldpriority,
newpriority)
if newsection != oldsection:
print "I: Will change section from %s to %s" % (oldsection, newsection)
if not Options.has_key("Done"):
pass
#utils.warn("No bugs to close have been specified. Noone will know you have done this.")
else:
print "I: Will close bug(s): %s" % (Options["Done"])
game_over()
Logger = daklog.Logger("override")
dsc_otype_id = get_override_type('dsc').overridetype_id
# We're already in a transaction
# We're in "do it" mode, we have something to do... do it
if newpriority != oldpriority:
session.execute(
"""
UPDATE override
SET priority = :newprioid
WHERE package = :package
AND override.type != :otypedsc
AND suite = (SELECT id FROM suite WHERE suite_name = :suite_name)""",
{
'newprioid': newprioid,
'package': package,
'otypedsc': dsc_otype_id,
'suite_name': suite_name
})
Logger.log(["changed priority", package, oldpriority, newpriority])
if newsection != oldsection:
q = session.execute(
"""
UPDATE override
SET section = :newsecid
WHERE package = :package
AND suite = (SELECT id FROM suite WHERE suite_name = :suite_name)""",
{
'newsecid': newsecid,
'package': package,
'suite_name': suite_name
})
Logger.log(["changed section", package, oldsection, newsection])
session.commit()
if Options.has_key("Done"):
if not cnf.has_key("Dinstall::BugServer"):
utils.warn(
"Asked to send Done message but Dinstall::BugServer is not configured"
)
Logger.close()
return
Subst = {}
Subst["__OVERRIDE_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
bcc = []
if cnf.find("Dinstall::Bcc") != "":
bcc.append(cnf["Dinstall::Bcc"])
if bcc:
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst["__BCC__"] = "X-Filler: 42"
if cnf.has_key("Dinstall::PackagesServer"):
Subst["__CC__"] = "Cc: " + package + "@" + cnf[
"Dinstall::PackagesServer"] + "\nX-DAK: dak override"
else:
Subst["__CC__"] = "X-DAK: dak override"
Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst["__WHOAMI__"] = utils.whoami()
Subst["__SOURCE__"] = package
summary = "Concerning package %s...\n" % (package)
summary += "Operating on the %s suite\n" % (suite_name)
if newpriority != oldpriority:
summary += "Changed priority from %s to %s\n" % (oldpriority,
newpriority)
if newsection != oldsection:
summary += "Changed section from %s to %s\n" % (oldsection,
newsection)
Subst["__SUMMARY__"] = summary
template = os.path.join(cnf["Dir::Templates"], "override.bug-close")
for bug in utils.split_args(Options["Done"]):
Subst["__BUG_NUMBER__"] = bug
mail_message = utils.TemplateSubst(Subst, template)
utils.send_mail(mail_message)
Logger.log(["closed bug", bug])
Logger.close()
def main():
cnf = Config()
Arguments = [('h',"help","Make-Maintainers::Options::Help"),
('a',"archive","Make-Maintainers::Options::Archive",'HasArg'),
('s',"source","Make-Maintainers::Options::Source"),
('p',"print","Make-Maintainers::Options::Print")]
for i in ["Help", "Source", "Print" ]:
if not cnf.has_key("Make-Maintainers::Options::%s" % (i)):
cnf["Make-Maintainers::Options::%s" % (i)] = ""
extra_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Make-Maintainers::Options")
if Options["Help"] or not Options.get('Archive'):
usage()
Logger = daklog.Logger('make-maintainers')
session = DBConn().session()
archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
# dictionary packages to maintainer names
maintainers = dict()
# dictionary packages to list of uploader names
uploaders = dict()
source_query = session.query(DBSource).from_statement('''
select distinct on (source.source) source.* from source
join src_associations sa on source.id = sa.source
join suite on sa.suite = suite.id
where suite.archive_id = :archive_id
order by source.source, source.version desc''') \
.params(archive_id=archive.archive_id)
binary_query = session.query(DBBinary).from_statement('''
select distinct on (binaries.package) binaries.* from binaries
join bin_associations ba on binaries.id = ba.bin
join suite on ba.suite = suite.id
where suite.archive_id = :archive_id
order by binaries.package, binaries.version desc''') \
.params(archive_id=archive.archive_id)
Logger.log(['sources'])
for source in source_query:
maintainers[source.source] = source.maintainer.name
uploaders[source.source] = uploader_list(source)
if not Options["Source"]:
Logger.log(['binaries'])
for binary in binary_query:
if binary.package not in maintainers:
maintainers[binary.package] = binary.maintainer.name
uploaders[binary.package] = uploader_list(binary.source)
Logger.log(['files'])
# Process any additional Maintainer files (e.g. from pseudo
# packages)
for filename in extra_files:
extrafile = utils.open_file(filename)
for line in extrafile.readlines():
line = re_comments.sub('', line).strip()
if line == "":
continue
(package, maintainer) = line.split(None, 1)
maintainers[package] = maintainer
uploaders[package] = [maintainer]
if Options["Print"]:
for package in sorted(maintainers):
sys.stdout.write(format(package, maintainers[package]))
else:
maintainer_file = open('Maintainers', 'w')
uploader_file = open('Uploaders', 'w')
for package in sorted(uploaders):
maintainer_file.write(format(package, maintainers[package]))
for uploader in uploaders[package]:
uploader_file.write(format(package, uploader))
uploader_file.close()
maintainer_file.close()
Logger.close()
def main():
global Options
cnf = Config()
Arguments = [
("h", "help", "Import-Keyring::Options::Help"),
("L", "import-ldap-users", "Import-Keyring::Options::Import-Ldap-Users"),
("U", "generate-users", "Import-Keyring::Options::Generate-Users", "HasArg"),
("l", "list-uids", "Import-Keyring::Options::List-UIDs", "HasArg"),
("n", "no-action", "Import-Keyring::Options::No-Action"),
]
for i in ["help", "report-changes", "generate-users", "import-ldap-users", "list-uids", "no-action"]:
if not cnf.has_key("Import-Keyring::Options::%s" % (i)):
cnf["Import-Keyring::Options::%s" % (i)] = ""
keyring_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
### Parse options
Options = cnf.subtree("Import-Keyring::Options")
if Options["Help"]:
usage()
### Initialise
session = DBConn().session()
if Options["List-UIDs"]:
list_uids(session, Options["List-UIDs"])
sys.exit(0)
if len(keyring_names) != 1:
usage(1)
### Keep track of changes made
changes = [] # (uid, changes strings)
### Cache all the existing fingerprint entries
db_fin_info = get_fingerprint_info(session)
### Parse the keyring
keyringname = keyring_names[0]
keyring = get_keyring(keyringname, session)
if not keyring:
print "E: Can't load keyring %s from database" % keyringname
sys.exit(1)
keyring.load_keys(keyringname)
### Generate new uid entries if they're needed (from LDAP or the keyring)
if Options["Generate-Users"]:
format = Options["Generate-Users"]
(desuid_byname, desuid_byid) = keyring.generate_users_from_keyring(Options["Generate-Users"], session)
elif Options["Import-Ldap-Users"]:
(desuid_byname, desuid_byid) = keyring.import_users_from_ldap(session)
else:
(desuid_byname, desuid_byid) = ({}, {})
### Cache all the existing uid entries
(db_uid_byname, db_uid_byid) = get_uid_info(session)
### Update full names of applicable users
for keyid in desuid_byid.keys():
uid = (keyid, desuid_byid[keyid][0])
name = desuid_byid[keyid][1]
oname = db_uid_byid[keyid][1]
if name and oname != name:
changes.append((uid[1], "Full name: %s" % (name)))
session.execute("UPDATE uid SET name = :name WHERE id = :keyid", {"name": name, "keyid": keyid})
# The fingerprint table (fpr) points to a uid and a keyring.
# If the uid is being decided here (ldap/generate) we set it to it.
# Otherwise, if the fingerprint table already has a uid (which we've
# cached earlier), we preserve it.
# Otherwise we leave it as None
fpr = {}
for z in keyring.keys.keys():
keyid = db_uid_byname.get(keyring.keys[z].get("uid", None), [None])[0]
if keyid == None:
keyid = db_fin_info.get(keyring.keys[z]["fingerprints"][0], [None])[0]
for y in keyring.keys[z]["fingerprints"]:
fpr[y] = (keyid, keyring.keyring_id)
# For any keys that used to be in this keyring, disassociate them.
# We don't change the uid, leaving that for historical info; if
# the id should change, it'll be set when importing another keyring.
for f, (u, fid, kr) in db_fin_info.iteritems():
if kr != keyring.keyring_id:
continue
if f in fpr:
continue
changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s" % (f)))
session.execute(
"""UPDATE fingerprint
SET keyring = NULL,
source_acl_id = NULL,
binary_acl_id = NULL,
binary_reject = TRUE
WHERE id = :fprid""",
{"fprid": fid},
)
session.execute("""DELETE FROM binary_acl_map WHERE fingerprint_id = :fprid""", {"fprid": fid})
# For the keys in this keyring, add/update any fingerprints that've
# changed.
for f in fpr:
newuid = fpr[f][0]
newuiduid = db_uid_byid.get(newuid, [None])[0]
(olduid, oldfid, oldkid) = db_fin_info.get(f, [-1, -1, -1])
if olduid == None:
olduid = -1
if oldkid == None:
oldkid = -1
if oldfid == -1:
changes.append((newuiduid, "Added key: %s" % (f)))
fp = Fingerprint()
fp.fingerprint = f
fp.keyring_id = keyring.keyring_id
if newuid:
fp.uid_id = newuid
fp.binary_acl_id = keyring.default_binary_acl_id
fp.source_acl_id = keyring.default_source_acl_id
fp.default_binary_reject = keyring.default_binary_reject
session.add(fp)
session.flush()
for k in keyring.keyring_acl_map:
ba = BinaryACLMap()
ba.fingerprint_id = fp.fingerprint_id
ba.architecture_id = k.architecture_id
session.add(ba)
session.flush()
else:
if newuid and olduid != newuid and olduid == -1:
changes.append((newuiduid, "Linked key: %s" % f))
changes.append((newuiduid, " (formerly unowned)"))
session.execute("UPDATE fingerprint SET uid = :uid WHERE id = :fpr", {"uid": newuid, "fpr": oldfid})
# Don't move a key from a keyring with a higher priority to a lower one
if oldkid != keyring.keyring_id:
movekey = False
if oldkid == -1:
movekey = True
else:
try:
oldkeyring = session.query(Keyring).filter_by(keyring_id=oldkid).one()
except NotFoundError:
print "ERROR: Cannot find old keyring with id %s" % oldkid
sys.exit(1)
if oldkeyring.priority < keyring.priority:
movekey = True
# Only change the keyring if it won't result in a loss of permissions
if movekey:
session.execute("""DELETE FROM binary_acl_map WHERE fingerprint_id = :fprid""", {"fprid": oldfid})
session.execute(
"""UPDATE fingerprint
SET keyring = :keyring,
source_acl_id = :source_acl_id,
binary_acl_id = :binary_acl_id,
binary_reject = :binary_reject
WHERE id = :fpr""",
{
"keyring": keyring.keyring_id,
"source_acl_id": keyring.default_source_acl_id,
"binary_acl_id": keyring.default_binary_acl_id,
"binary_reject": keyring.default_binary_reject,
"fpr": oldfid,
},
)
session.flush()
for k in keyring.keyring_acl_map:
ba = BinaryACLMap()
ba.fingerprint_id = oldfid
ba.architecture_id = k.architecture_id
session.add(ba)
session.flush()
else:
print "Key %s exists in both %s and %s keyrings. Not demoting." % (
f,
oldkeyring.keyring_name,
keyring.keyring_name,
)
# All done!
if Options["No-Action"]:
session.rollback()
else:
session.commit()
# Print a summary
changesd = {}
for (k, v) in changes:
if k not in changesd:
changesd[k] = ""
changesd[k] += " %s\n" % (v)
keys = changesd.keys()
keys.sort()
for k in keys:
print "%s\n%s\n" % (k, changesd[k])
def update_db(self):
# Ok, try and find the configuration table
print "Determining dak database revision ..."
cnf = Config()
logger = Logger('update-db')
modules = []
try:
# Build a connect string
if cnf.has_key("DB::Service"):
connect_str = "service=%s" % cnf["DB::Service"]
else:
connect_str = "dbname=%s" % (cnf["DB::Name"])
if cnf.has_key("DB::Host") and cnf["DB::Host"] != '':
connect_str += " host=%s" % (cnf["DB::Host"])
if cnf.has_key("DB::Port") and cnf["DB::Port"] != '-1':
connect_str += " port=%d" % (int(cnf["DB::Port"]))
self.db = psycopg2.connect(connect_str)
db_role = cnf.get("DB::Role")
if db_role is not None:
self.db.cursor().execute('SET ROLE "{}"'.format(db_role))
except Exception as e:
print "FATAL: Failed connect to database (%s)" % str(e)
sys.exit(1)
database_revision = int(self.get_db_rev())
logger.log(
['transaction id before update: %s' % self.get_transaction_id()])
if database_revision == -1:
print "dak database schema predates update-db."
print ""
print "This script will attempt to upgrade it to the lastest, but may fail."
print "Please make sure you have a database backup handy. If you don't, press Ctrl-C now!"
print ""
print "Continuing in five seconds ..."
time.sleep(5)
print ""
print "Attempting to upgrade pre-zero database to zero"
self.update_db_to_zero()
database_revision = 0
dbfiles = glob(
os.path.join(os.path.dirname(__file__), 'dakdb/update*.py'))
required_database_schema = max(
map(int, findall('update(\d+).py', " ".join(dbfiles))))
print "dak database schema at %d" % database_revision
print "dak version requires schema %d" % required_database_schema
if database_revision < required_database_schema:
print "\nUpdates to be applied:"
for i in range(database_revision, required_database_schema):
i += 1
dakdb = __import__("dakdb", globals(), locals(),
['update' + str(i)])
update_module = getattr(dakdb, "update" + str(i))
print "Update %d: %s" % (
i, next(s for s in update_module.__doc__.split("\n") if s))
modules.append((update_module, i))
if not Config().find_b("Update-DB::Options::Yes", False):
prompt = "\nUpdate database? (y/N) "
answer = utils.our_raw_input(prompt)
if answer.upper() != 'Y':
sys.exit(0)
else:
print "no updates required"
logger.log(["no updates required"])
sys.exit(0)
for module in modules:
(update_module, i) = module
try:
update_module.do_update(self)
message = "updated database schema from %d to %d" % (
database_revision, i)
print message
logger.log([message])
except DBUpdateError as e:
# Seems the update did not work.
print "Was unable to update database schema from %d to %d." % (
database_revision, i)
print "The error message received was %s" % (e)
logger.log(["DB Schema upgrade failed"])
logger.close()
utils.fubar("DB Schema upgrade failed")
database_revision += 1
logger.close()
