def parse_prod(logdate):
global stats
global users
maildate = ''.join([x[-2:] for x in logdate.split('-')])
mailarchive = join(utils.get_conf()['Dir::Base'], 'mail/archive',
'mail-%s.xz' % maildate)
if not isfile(mailarchive):
return
(fd, tmpfile) = utils.temp_filename(utils.get_conf()['Dir::TempPath'])
system('xzcat %s > %s' % (mailarchive, tmpfile))
for message in mbox(tmpfile):
if (message['subject']
and message['subject'].startswith('Comments regarding')):
try:
member = users[' '.join(message['From'].split()[:-1])]
except KeyError:
continue
ts = mktime_tz(parsedate_tz(message['date']))
timestamp = datetime.fromtimestamp(ts).strftime("%Y%m%d%H%M%S")
date = parse_timestamp(timestamp)
if date not in stats:
stats[date] = {'stats': {'NEW': 0, 'ACCEPT': 0,
'REJECT': 0, 'PROD': 0}, 'members': {}}
if member not in stats[date]['members']:
stats[date]['members'][member] = {'ACCEPT': 0, 'REJECT': 0,
'PROD': 0}
if member not in stats['history']['members']:
stats['history']['members'][member] = {'ACCEPT': 0,
'REJECT': 0, 'PROD': 0}
stats[date]['stats']['PROD'] += 1
stats[date]['members'][member]['PROD'] += 1
stats['history']['stats']['PROD'] += 1
stats['history']['members'][member]['PROD'] += 1
unlink(tmpfile)
def init():
global Cnf, Options
Cnf = utils.get_conf()
Arguments = [('h', "help", "Show-Deferred::Options::Help"),
("p", "link-path", "Show-Deferred::LinkPath", "HasArg"),
("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"),
('r', "rrd", "Show-Deferred::Options::Rrd", "HasArg")]
args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
for i in ["help"]:
key = "Show-Deferred::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
for i, j in [("DeferredQueue", "--deferred-queue")]:
key = "Show-Deferred::%s" % i
if key not in Cnf:
print("""%s is mandatory.
set via config file or command-line option %s""" % (key, j), file=sys.stderr)
Options = Cnf.subtree("Show-Deferred::Options")
if Options["help"]:
usage()
# Initialise database connection
DBConn()
return args
def main ():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Stats::Options::Help")]
for i in [ "help" ]:
if not Cnf.has_key("Stats::Options::%s" % (i)):
Cnf["Stats::Options::%s" % (i)] = ""
args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Stats::Options")
if Options["Help"]:
usage()
if len(args) < 1:
utils.warn("dak stats requires a MODE argument")
usage(1)
elif len(args) > 1:
utils.warn("dak stats accepts only one MODE argument")
usage(1)
mode = args[0].lower()
if mode == "arch-space":
per_arch_space_use()
elif mode == "pkg-nums":
number_of_packages()
elif mode == "daily-install":
daily_install_stats()
else:
utils.warn("unknown mode '%s'" % (mode))
usage(1)
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Queue-Report::Options::Help"),
('n',"new","Queue-Report::Options::New"),
('8','822',"Queue-Report::Options::822"),
('s',"sort","Queue-Report::Options::Sort", "HasArg"),
('a',"age","Queue-Report::Options::Age", "HasArg"),
('r',"rrd","Queue-Report::Options::Rrd", "HasArg"),
('d',"directories","Queue-Report::Options::Directories", "HasArg")]
for i in [ "help" ]:
if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
Cnf["Queue-Report::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Queue-Report::Options")
if Options["Help"]:
usage()
if Cnf.has_key("Queue-Report::Options::New"):
header()
queue_names = []
if Cnf.has_key("Queue-Report::Options::Directories"):
for i in Cnf["Queue-Report::Options::Directories"].split(","):
queue_names.append(i)
elif Cnf.has_key("Queue-Report::Directories"):
queue_names = Cnf.value_list("Queue-Report::Directories")
else:
queue_names = [ "byhand", "new" ]
if Cnf.has_key("Queue-Report::Options::Rrd"):
rrd_dir = Cnf["Queue-Report::Options::Rrd"]
elif Cnf.has_key("Dir::Rrd"):
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
f = None
if Cnf.has_key("Queue-Report::Options::822"):
# Open the report file
f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w")
session = DBConn().session()
for queue_name in queue_names:
queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).first()
if queue is not None:
process_queue(queue, f, rrd_dir)
else:
utils.warn("Cannot find queue %s" % queue_name)
if Cnf.has_key("Queue-Report::Options::822"):
f.close()
if Cnf.has_key("Queue-Report::Options::New"):
footer()
def do_update(self):
""" Execute the DB update """
print "Lets make suite_architecture table use sane values"
Cnf = get_conf()
query = "INSERT into suite_architectures (suite, architecture) VALUES (%s, %s)" #: Update query
try:
c = self.db.cursor()
c.execute("DELETE FROM suite_architectures;")
c.execute("SELECT id, arch_string FROM architecture;")
a=c.fetchall()
for arch in a:
archs[arch[1]]=arch[0]
c.execute("SELECT id,suite_name FROM suite")
s=c.fetchall()
for suite in s:
suites[suite[1]]=suite[0]
for suite in Cnf.subtree("Suite").list():
print "Processing suite %s" % (suite)
architectures = Cnf.subtree("Suite::" + suite).value_list("Architectures")
suite = suite.lower()
for arch in architectures:
c.execute(query, [suites[suite], archs[arch]])
c.execute("UPDATE config SET value = '4' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError("Unable to apply sanity to suite_architecture table, rollback issued. Error message : %s" % (str(msg)))
def show_config(command):
args = [str(x) for x in command]
cnf = utils.get_conf()
die_arglen(args, 2, "E: config needs at least a command")
mode = args[1].lower()
if mode == 'db':
connstr = ""
if cnf.has_key("DB::Service"):
# Service mode
connstr = "postgresql://service=%s" % cnf["DB::Service"]
elif cnf.has_key("DB::Host"):
# TCP/IP
connstr = "postgres://%s" % cnf["DB::Host"]
if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
connstr += ":%s" % cnf["DB::Port"]
connstr += "/%s" % cnf["DB::Name"]
else:
# Unix Socket
connstr = "postgres:///%s" % cnf["DB::Name"]
if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
connstr += "?port=%s" % cnf["DB::Port"]
print connstr
elif mode == 'db-shell':
e = []
if cnf.has_key("DB::Service"):
e.append('PGSERVICE')
print "PGSERVICE=%s" % cnf["DB::Service"]
if cnf.has_key("DB::Name"):
e.append('PGDATABASE')
print "PGDATABASE=%s" % cnf["DB::Name"]
if cnf.has_key("DB::Host"):
print "PGHOST=%s" % cnf["DB::Host"]
e.append('PGHOST')
if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1":
print "PGPORT=%s" % cnf["DB::Port"]
e.append('PGPORT')
print "export " + " ".join(e)
elif mode == 'get':
print cnf.get(args[2])
else:
session = DBConn().session()
try:
o = session.query(DBConfig).filter_by(name = mode).one()
print o.value
except NoResultFound:
print "W: option '%s' not set" % mode
def architecture(command):
args = [str(x) for x in command]
Cnf = utils.get_conf()
d = DBConn()
die_arglen(args, 2, "E: architecture needs at least a command")
mode = args[1].lower()
if mode == 'list':
__architecture_list(d, args)
elif mode == 'add':
__architecture_add(d, args)
elif mode == 'rm':
__architecture_rm(d, args)
else:
die("E: architecture command unknown")
def main():
Cnf = utils.get_conf()
Arguments = [('h','help','Make-Changelog::Options::Help'),
('a','archive','Make-Changelog::Options::Archive','HasArg'),
('s','suite','Make-Changelog::Options::Suite','HasArg'),
('b','base-suite','Make-Changelog::Options::Base-Suite','HasArg'),
('n','binnmu','Make-Changelog::Options::binNMU'),
('e','export','Make-Changelog::Options::export'),
('p','progress','Make-Changelog::Options::progress')]
for i in ['help', 'suite', 'base-suite', 'binnmu', 'export', 'progress']:
if not Cnf.has_key('Make-Changelog::Options::%s' % (i)):
Cnf['Make-Changelog::Options::%s' % (i)] = ''
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree('Make-Changelog::Options')
suite = Cnf['Make-Changelog::Options::Suite']
base_suite = Cnf['Make-Changelog::Options::Base-Suite']
binnmu = Cnf['Make-Changelog::Options::binNMU']
export = Cnf['Make-Changelog::Options::export']
progress = Cnf['Make-Changelog::Options::progress']
if Options['help'] or not (suite and base_suite) and not export:
usage()
for s in suite, base_suite:
if not export and not get_suite(s):
utils.fubar('Invalid suite "%s"' % s)
session = DBConn().session()
if export:
archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
exportpath = archive.changelog
if exportpath:
export_files(session, archive, exportpath, progress)
generate_export_filelist(exportpath)
else:
utils.fubar('No changelog export path defined')
elif binnmu:
display_changes(get_binary_uploads(suite, base_suite, session), 3)
else:
display_changes(get_source_uploads(suite, base_suite, session), 2)
session.commit()
def do_update(self):
print "Add constraints to src_uploaders"
Cnf = get_conf()
try:
c = self.db.cursor()
# Deal with out-of-date src_uploaders entries
c.execute("DELETE FROM src_uploaders WHERE source NOT IN (SELECT id FROM source)")
c.execute("DELETE FROM src_uploaders WHERE maintainer NOT IN (SELECT id FROM maintainer)")
# Add constraints
c.execute("ALTER TABLE src_uploaders ADD CONSTRAINT src_uploaders_maintainer FOREIGN KEY (maintainer) REFERENCES maintainer(id) ON DELETE CASCADE")
c.execute("ALTER TABLE src_uploaders ADD CONSTRAINT src_uploaders_source FOREIGN KEY (source) REFERENCES source(id) ON DELETE CASCADE")
c.execute("UPDATE config SET value = '10' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError("Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
def suite(command):
args = [str(x) for x in command]
Cnf = utils.get_conf()
d = DBConn()
die_arglen(args, 2, "E: suite needs at least a command")
mode = args[1].lower()
if mode == 'list':
__suite_list(d, args)
elif mode == 'show':
__suite_show(d, args)
elif mode == 'add':
__suite_add(d, args, False)
elif mode == 'add-all-arches':
__suite_add(d, args, True)
else:
die("E: suite command unknown")
def main():
Cnf = utils.get_conf()
cnf = Config()
Arguments = [('h','help','Make-Changelog::Options::Help'),
('s','suite','Make-Changelog::Options::Suite','HasArg'),
('b','base-suite','Make-Changelog::Options::Base-Suite','HasArg'),
('n','binnmu','Make-Changelog::Options::binNMU'),
('e','export','Make-Changelog::Options::export')]
for i in ['help', 'suite', 'base-suite', 'binnmu', 'export']:
if not Cnf.has_key('Make-Changelog::Options::%s' % (i)):
Cnf['Make-Changelog::Options::%s' % (i)] = ''
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree('Make-Changelog::Options')
suite = Cnf['Make-Changelog::Options::Suite']
base_suite = Cnf['Make-Changelog::Options::Base-Suite']
binnmu = Cnf['Make-Changelog::Options::binNMU']
export = Cnf['Make-Changelog::Options::export']
if Options['help'] or not (suite and base_suite) and not export:
usage()
for s in suite, base_suite:
if not export and not get_suite(s):
utils.fubar('Invalid suite "%s"' % s)
session = DBConn().session()
if export:
if cnf.exportpath:
exportpath = os.path.join(Cnf['Dir::Export'], cnf.exportpath)
export_files(session, Cnf['Dir::Pool'], exportpath)
else:
utils.fubar('No changelog export path defined')
elif binnmu:
display_changes(get_binary_uploads(suite, base_suite, session), 3)
else:
display_changes(get_source_uploads(suite, base_suite, session), 2)
session.commit()
def main():
global Cnf
global users
Cnf = utils.get_conf()
Arguments = [('h', "help", "Stats::Options::Help")]
for i in ["help"]:
key = "Stats::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Stats::Options")
if Options["Help"]:
usage()
if len(args) < 1:
utils.warn("dak stats requires a MODE argument")
usage(1)
elif len(args) > 1:
if args[0].lower() != "new":
utils.warn("dak stats accepts only one MODE argument")
usage(1)
elif args[0].lower() == "new":
utils.warn("new MODE requires an output file")
usage(1)
mode = args[0].lower()
if mode == "arch-space":
per_arch_space_use()
elif mode == "pkg-nums":
number_of_packages()
elif mode == "daily-install":
daily_install_stats()
elif mode == "new":
users = utils.get_users_from_ldap()
new_stats(Cnf["Dir::Log"], args[1])
else:
utils.warn("unknown mode '%s'" % (mode))
usage(1)
def init():
"""
Initialize. Sets up database connection, parses commandline arguments.
@attention: This function may run B{within sudo}
"""
global Cnf, Options
apt_pkg.init()
Cnf = utils.get_conf()
Arguments = [
("a", "automatic", "Edit-Transitions::Options::Automatic"),
("h", "help", "Edit-Transitions::Options::Help"),
("e", "edit", "Edit-Transitions::Options::Edit"),
("i", "import", "Edit-Transitions::Options::Import", "HasArg"),
("c", "check", "Edit-Transitions::Options::Check"),
("s", "sudo", "Edit-Transitions::Options::Sudo"),
("n", "no-action", "Edit-Transitions::Options::No-Action"),
]
for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]:
if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)):
Cnf["Edit-Transitions::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Edit-Transitions::Options")
if Options["help"]:
usage()
username = utils.getusername()
if username != "dak":
print "Non-dak user: %s" % username
Options["sudo"] = "y"
# Initialise DB connection
DBConn()
def do_update(self):
print "pending_contents should distinguish by arch"
Cnf = get_conf()
try:
c = self.db.cursor()
c.execute("DELETE FROM pending_content_associations")
c.execute("""ALTER TABLE pending_content_associations
ADD COLUMN architecture integer NOT NULL""")
c.execute("""ALTER TABLE ONLY pending_content_associations
ADD CONSTRAINT pending_content_assiciations_arch
FOREIGN KEY (architecture)
REFERENCES architecture(id)
ON DELETE CASCADE""")
c.execute("UPDATE config SET value = '9' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError("Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
def suite_component(command):
args = [str(x) for x in command]
Cnf = utils.get_conf()
d = DBConn()
die_arglen(args, 2, "E: suite-component needs at least a command")
mode = args[1].lower()
if mode == 'list':
__suite_component_list(d, args)
elif mode == 'list-component':
__suite_component_listcomponent(d, args)
elif mode == 'list-suite':
__suite_component_listsuite(d, args)
elif mode == 'add':
__suite_component_add(d, args)
# elif mode == 'rm':
# __suite_architecture_rm(d, args)
else:
die("E: suite-component command unknown")
def main():
"""
for now, we just dump a list of commands that could be sent for
[email protected]
"""
global Cnf
Cnf = utils.get_conf()
for arg in arguments:
opt = "BtsCategorize::Options::%s" % arg[1]
if opt not in Cnf:
Cnf[opt] = ""
packages = apt_pkg.parse_commandline(Cnf, arguments, sys.argv)
Options = Cnf.subtree('BtsCategorize::Options')
if Options["Help"]:
usage()
sys.exit(0)
if Options["Quiet"]:
level = logging.ERROR
elif Options["Verbose"]:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level,
format='%(asctime)s %(levelname)s %(message)s',
stream=sys.stderr)
body = BugClassifier().email_text()
if body:
send_email(body, Options["Simulate"])
else:
log.info("nothing to do")
def init():
"""
Initialize. Sets up database connection, parses commandline arguments.
@attention: This function may run B{within sudo}
"""
global Cnf, Options
apt_pkg.init()
Cnf = utils.get_conf()
Arguments = [('a', "automatic", "Edit-Transitions::Options::Automatic"),
('h', "help", "Edit-Transitions::Options::Help"),
('e', "edit", "Edit-Transitions::Options::Edit"),
('i', "import", "Edit-Transitions::Options::Import", "HasArg"),
('c', "check", "Edit-Transitions::Options::Check"),
('s', "sudo", "Edit-Transitions::Options::Sudo"),
('n', "no-action", "Edit-Transitions::Options::No-Action")]
for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]:
key = "Edit-Transitions::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Edit-Transitions::Options")
if Options["help"]:
usage()
username = utils.getusername()
if username != "dak":
print("Non-dak user: %s" % username)
Options["sudo"] = "y"
# Initialise DB connection
DBConn()
def do_update(self):
print "pending_contents should distinguish by arch"
Cnf = get_conf()
try:
c = self.db.cursor()
c.execute("DELETE FROM pending_content_associations")
c.execute("""ALTER TABLE pending_content_associations
ADD COLUMN architecture integer NOT NULL""")
c.execute("""ALTER TABLE ONLY pending_content_associations
ADD CONSTRAINT pending_content_assiciations_arch
FOREIGN KEY (architecture)
REFERENCES architecture(id)
ON DELETE CASCADE""")
c.execute("UPDATE config SET value = '9' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
"Unable to apply suite config updates, rollback issued. Error message : %s"
% (str(msg)))
def suite(command):
args = [str(x) for x in command]
Cnf = utils.get_conf()
d = DBConn()
die_arglen(args, 2, "E: suite needs at least a command")
mode = args[1].lower()
if mode == 'list':
__suite_list(d, args)
elif mode == 'show':
__suite_show(d, args)
elif mode == 'rm':
__suite_rm(d, args)
elif mode == 'add':
__suite_add(d, args, False)
elif mode == 'add-all-arches':
__suite_add(d, args, True)
elif mode == 'add-build-queue':
__suite_add_build_queue(d, args)
else:
die("E: suite command unknown")
def show_keyring(command):
args = [str(x) for x in command]
cnf = utils.get_conf()
die_arglen(args, 2, "E: keyring needs at least a command")
mode = args[1].lower()
d = DBConn()
q = d.session().query(Keyring).filter(Keyring.active == True)
if mode == 'list-all':
pass
elif mode == 'list-binary':
q = q.join(Keyring.acl).filter(ACL.allow_source == False)
elif mode == 'list-source':
q = q.join(Keyring.acl).filter(ACL.allow_source == True)
else:
die("E: keyring command unknown")
for k in q.all():
print k.keyring_name
def show_keyring(command):
args = [str(x) for x in command]
cnf = utils.get_conf()
die_arglen(args, 2, "E: keyring needs at least a command")
mode = args[1].lower()
d = DBConn()
q = d.session().query(Keyring).filter(Keyring.active == True) # noqa:E712
if mode == 'list-all':
pass
elif mode == 'list-binary':
q = q.join(Keyring.acl).filter(ACL.allow_source == False) # noqa:E712
elif mode == 'list-source':
q = q.join(Keyring.acl).filter(ACL.allow_source == True) # noqa:E712
else:
die("E: keyring command unknown")
for k in q.all():
print(k.keyring_name)
def main():
"""Initial setup of an archive."""
global Cnf
Cnf = utils.get_conf()
arguments = [('h', "help", "Init-Dirs::Options::Help")]
for i in ["help"]:
if not Cnf.has_key("Init-Dirs::Options::%s" % (i)):
Cnf["Init-Dirs::Options::%s" % (i)] = ""
d = DBConn()
arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv)
options = Cnf.subtree("Init-Dirs::Options")
if options["Help"]:
usage()
elif arguments:
utils.warn("dak init-dirs takes no arguments.")
usage(exit_code=1)
create_directories()
def main ():
"""Initial setup of an archive."""
global Cnf
Cnf = utils.get_conf()
arguments = [('h', "help", "Init-Dirs::Options::Help")]
for i in [ "help" ]:
if not Cnf.has_key("Init-Dirs::Options::%s" % (i)):
Cnf["Init-Dirs::Options::%s" % (i)] = ""
d = DBConn()
arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv)
options = Cnf.subtree("Init-Dirs::Options")
if options["Help"]:
usage()
elif arguments:
utils.warn("dak init-dirs takes no arguments.")
usage(exit_code=1)
create_directories()
def version_check(command):
args = [str(x) for x in command]
Cnf = utils.get_conf()
d = DBConn()
die_arglen(args, 2, "E: version-check needs at least a command")
mode = args[1].lower()
if mode == 'list':
__version_check_list(d)
elif mode == 'list-suite':
if len(args) != 3:
die("E: version-check list-suite needs a single parameter")
__version_check_list_suite(d, args[2])
elif mode == 'add':
if len(args) != 5:
die("E: version-check add needs three parameters")
__version_check_add(d, args[2], args[3], args[4])
elif mode == 'rm':
if len(args) != 5:
die("E: version-check rm needs three parameters")
__version_check_rm(d, args[2], args[3], args[4])
else:
die("E: version-check command unknown")
def main():
"""Perform administrative work on the dak database"""
global dryrun
Cnf = utils.get_conf()
arguments = [('h', "help", "Admin::Options::Help"),
('n', "dry-run", "Admin::Options::Dry-Run")]
for i in ["help", "dry-run"]:
if not Cnf.has_key("Admin::Options::%s" % (i)):
Cnf["Admin::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv)
options = Cnf.subtree("Admin::Options")
if options["Help"] or len(arguments) < 1:
usage()
if options["Dry-Run"]:
dryrun = True
subcommand = str(arguments[0])
if subcommand in dispatch.keys():
dispatch[subcommand](arguments)
else:
die("E: Unknown command")
def main():
Cnf = utils.get_conf()
count = 0
move_date = int(time.time())
os.chdir(Cnf["Dir::Done"])
files = glob.glob("%s/*" % (Cnf["Dir::Done"]))
for filename in files:
if os.path.isfile(filename):
filemtime = os.stat(filename)[stat.ST_MTIME]
if filemtime > move_date:
continue
mtime = time.gmtime(filemtime)
dirname = time.strftime("%Y/%m/%d", mtime)
if not os.path.exists(dirname):
print "Creating: %s" % (dirname)
os.makedirs(dirname)
dest = dirname + '/' + os.path.basename(filename)
if os.path.exists(dest):
utils.warn("%s already exists." % (dest))
continue
print "Move: %s -> %s" % (filename, dest)
os.rename(filename, dest)
count = count + 1
print "Moved %d files." % (count)
def version_check(command):
args = [str(x) for x in command]
Cnf = utils.get_conf()
d = DBConn()
die_arglen(args, 2, "E: version-check needs at least a command")
mode = args[1].lower()
if mode == 'list':
__version_check_list(d)
elif mode == 'list-suite':
if len(args) != 3:
die("E: version-check list-suite needs a single parameter")
__version_check_list_suite(d, args[2])
elif mode == 'add':
if len(args) != 5:
die("E: version-check add needs three parameters")
__version_check_add(d, args[2], args[3], args[4])
elif mode == 'rm':
if len(args) != 5:
die("E: version-check rm needs three parameters")
__version_check_rm(d, args[2], args[3], args[4])
else:
die("E: version-check command unknown")
def main():
"""Perform administrative work on the dak database"""
global dryrun
Cnf = utils.get_conf()
arguments = [('h', "help", "Admin::Options::Help"),
('n', "dry-run", "Admin::Options::Dry-Run")]
for i in [ "help", "dry-run" ]:
if not Cnf.has_key("Admin::Options::%s" % (i)):
Cnf["Admin::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv)
options = Cnf.subtree("Admin::Options")
if options["Help"] or len(arguments) < 1:
usage()
if options["Dry-Run"]:
dryrun = True
subcommand = str(arguments[0])
if subcommand in dispatch.keys():
dispatch[subcommand](arguments)
else:
die("E: Unknown command")
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Queue-Report::Options::Help"),
('n',"new","Queue-Report::Options::New"),
('8','822',"Queue-Report::Options::822"),
('s',"sort","Queue-Report::Options::Sort", "HasArg"),
('a',"age","Queue-Report::Options::Age", "HasArg"),
('r',"rrd","Queue-Report::Options::Rrd", "HasArg"),
('d',"directories","Queue-Report::Options::Directories", "HasArg")]
for i in [ "help" ]:
if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
Cnf["Queue-Report::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Queue-Report::Options")
if Options["Help"]:
usage()
if Cnf.has_key("Queue-Report::Options::New"):
header()
# Initialize db so we can get the NEW comments
dbconn = DBConn()
queue_names = [ ]
if Cnf.has_key("Queue-Report::Options::Directories"):
for i in Cnf["Queue-Report::Options::Directories"].split(","):
queue_names.append(i)
elif Cnf.has_key("Queue-Report::Directories"):
queue_names = Cnf.value_list("Queue-Report::Directories")
else:
queue_names = [ "byhand", "new" ]
if Cnf.has_key("Queue-Report::Options::Rrd"):
rrd_dir = Cnf["Queue-Report::Options::Rrd"]
elif Cnf.has_key("Dir::Rrd"):
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
f = None
if Cnf.has_key("Queue-Report::Options::822"):
# Open the report file
f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w")
session = dbconn.session()
for queue_name in queue_names:
queue = get_policy_queue(queue_name, session)
if queue:
directory = os.path.abspath(queue.path)
changes_files = glob.glob("%s/*.changes" % (directory))
process_changes_files(changes_files, os.path.basename(directory), f, rrd_dir)
else:
utils.warn("Cannot find queue %s" % queue_name)
if Cnf.has_key("Queue-Report::Options::822"):
f.close()
if Cnf.has_key("Queue-Report::Options::New"):
footer()
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h', "help", "Graph::Options::Help"),
('x', "extra-rrd", "Graph::Options::Extra-Rrd", "HasArg"),
('r', "rrd", "Graph::Options::Rrd", "HasArg"),
('i', "images", "Graph::Options::Images", "HasArg"),
('n', "names", "Graph::Options::Names", "HasArg")]
for i in ["help"]:
key = "Graph::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Graph::Options")
if Options["Help"]:
usage()
names = []
if "Graph::Options::Names" in Cnf:
for i in Cnf["Graph::Options::Names"].split(","):
names.append(i)
elif "Graph::Names" in Cnf:
names = Cnf.value_list("Graph::Names")
else:
names = default_names
extra_rrdtool_args = []
if "Graph::Options::Extra-Rrd" in Cnf:
for i in Cnf["Graph::Options::Extra-Rrd"].split(","):
f = open(i)
extra_rrdtool_args.extend(f.read().strip().split("\n"))
f.close()
elif "Graph::Extra-Rrd" in Cnf:
for i in Cnf.value_list("Graph::Extra-Rrd"):
f = open(i)
extra_rrdtool_args.extend(f.read().strip().split("\n"))
f.close()
if "Graph::Options::Rrd" in Cnf:
rrd_dir = Cnf["Graph::Options::Rrd"]
elif "Dir::Rrd" in Cnf:
rrd_dir = Cnf["Dir::Rrd"]
else:
print("No directory to read RRD files from\n", file=sys.stderr)
sys.exit(1)
if "Graph::Options::Images" in Cnf:
image_dir = Cnf["Graph::Options::Images"]
else:
print("No directory to write graph images to\n", file=sys.stderr)
sys.exit(1)
for name in names:
stdargs = [rrd_dir, image_dir, name, extra_rrdtool_args]
graph(*(stdargs + ['day', 'day', 'now-1d']))
graph(*(stdargs + ['week', 'week', 'now-1w']))
graph(*(stdargs + ['month', 'month', 'now-1m']))
graph(*(stdargs + ['year', 'year', 'now-1y']))
graph(*(stdargs + ['5years', '5 years', 'now-5y', True]))
graph(*(stdargs + ['10years', '10 years', 'now-10y', True]))
def main():
global Cnf
keyrings = None
Cnf = utils.get_conf()
Arguments = [
('h', "help", "Add-User::Options::Help"),
('k', "key", "Add-User::Options::Key", "HasArg"),
('u', "user", "Add-User::Options::User", "HasArg"),
]
for i in ["help"]:
key = "Add-User::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Add-User::Options")
if Options["help"]:
usage()
session = DBConn().session()
if not keyrings:
keyrings = get_active_keyring_paths()
cmd = [
"gpg", "--with-colons", "--no-secmem-warning",
"--no-auto-check-trustdb", "--with-fingerprint", "--no-default-keyring"
]
cmd.extend(utils.gpg_keyring_args(keyrings).split())
cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]])
output = subprocess.check_output(cmd).rstrip()
m = re_gpg_fingerprint_colon.search(output)
if not m:
print(output)
utils.fubar(
"0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"],
utils.prefix_multi_line_string(output, " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ", "")
uid = ""
if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]:
uid = Cnf["Add-User::Options::User"]
name = Cnf["Add-User::Options::User"]
else:
u = re_user_address.search(output)
if not u:
print(output)
utils.fubar(
"0x%s: (2) No userid found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"],
utils.prefix_multi_line_string(output, " [GPG output:] ")))
uid = u.group(1)
n = re_user_name.search(output)
name = n.group(1)
# Look for all email addresses on the key.
emails = []
for line in output.split('\n'):
e = re_user_mails.search(line)
if not e:
continue
emails.append(e.group(2))
print("0x%s -> %s <%s> -> %s -> %s" %
(Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key))
prompt = "Add user %s with above data (y/N) ? " % (uid)
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
# Create an account for the user?
summary = ""
# Now add user to the database.
# Note that we provide a session, so we're responsible for committing
uidobj = get_or_set_uid(uid, session=session)
uid_id = uidobj.uid_id
session.commit()
# Lets add user to the email-whitelist file if its configured.
if "Dinstall::MailWhiteList" in Cnf and Cnf[
"Dinstall::MailWhiteList"] != "":
f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
for mail in emails:
f.write(mail + '\n')
f.close()
print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" %
(uid, uid_id, name, primary_key))
# Should we send mail to the newly added user?
if Cnf.find_b("Add-User::SendEmail"):
mail = name + "<" + emails[0] + ">"
Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
Subst["__PRIMARY_KEY__"] = primary_key
Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
Subst["__SUMMARY__"] = summary
new_add_message = utils.TemplateSubst(
Subst, Cnf["Dir::Templates"] + "/add-user.added")
utils.send_mail(new_add_message)
else:
uid = None
def do_update(self):
print "Moving some of the suite config into the DB"
Cnf = get_conf()
try:
c = self.db.cursor()
c.execute("ALTER TABLE suite ADD COLUMN untouchable BOOLEAN NOT NULL DEFAULT FALSE;")
query = "UPDATE suite SET untouchable = TRUE WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
untouchable = Cnf.find("Suite::%s::Untouchable" % (suite))
if not untouchable:
continue
print "[Untouchable] Processing suite %s" % (suite)
suite = suite.lower()
c.execute(query, [suite])
c.execute("ALTER TABLE suite ADD COLUMN announce text NOT NULL DEFAULT '*****@*****.**';")
query = "UPDATE suite SET announce = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
announce_list = Cnf.find("Suite::%s::Announce" % (suite))
print "[Announce] Processing suite %s" % (suite)
suite = suite.lower()
c.execute(query, [announce_list, suite])
c.execute("ALTER TABLE suite ADD COLUMN codename text;")
query = "UPDATE suite SET codename = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
codename = Cnf.find("Suite::%s::CodeName" % (suite))
print "[Codename] Processing suite %s" % (suite)
suite = suite.lower()
c.execute(query, [codename, suite])
c.execute("ALTER TABLE suite ADD COLUMN overridecodename text;")
query = "UPDATE suite SET overridecodename = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
codename = Cnf.find("Suite::%s::OverrideCodeName" % (suite))
print "[OverrideCodeName] Processing suite %s" % (suite)
suite = suite.lower()
c.execute(query, [codename, suite])
c.execute("ALTER TABLE suite ADD COLUMN validtime integer NOT NULL DEFAULT 604800;")
query = "UPDATE suite SET validtime = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
validtime = Cnf.find("Suite::%s::ValidTime" % (suite))
print "[ValidTime] Processing suite %s" % (suite)
if not validtime:
validtime = 0
suite = suite.lower()
c.execute(query, [validtime, suite])
c.execute("ALTER TABLE suite ADD COLUMN priority integer NOT NULL DEFAULT 0;")
query = "UPDATE suite SET priority = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
priority = Cnf.find("Suite::%s::Priority" % (suite))
print "[Priority] Processing suite %s" % (suite)
if not priority:
priority = 0
suite = suite.lower()
c.execute(query, [priority, suite])
c.execute("ALTER TABLE suite ADD COLUMN notautomatic BOOLEAN NOT NULL DEFAULT FALSE;")
query = "UPDATE suite SET notautomatic = TRUE WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
notautomatic = Cnf.find("Suite::%s::NotAutomatic" % (suite))
print "[NotAutomatic] Processing suite %s" % (suite)
if not notautomatic:
continue
suite = suite.lower()
c.execute(query, [suite])
c.execute("UPDATE config SET value = '7' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError("Unable to appy suite config updates, rollback issued. Error message : %s" % (str(msg)))
def do_update(self):
print "Moving some more of the suite config into the DB"
Cnf = get_conf()
try:
c = self.db.cursor()
c.execute("ALTER TABLE suite ADD COLUMN copychanges TEXT;")
query = "UPDATE suite SET copychanges = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
copychanges = Cnf.find("Suite::%s::CopyChanges" % (suite))
print "[CopyChanges] Processing suite %s" % (suite)
if not copychanges:
continue
suite = suite.lower()
c.execute(query, [copychanges, suite])
c.execute("ALTER TABLE suite ADD COLUMN copydotdak TEXT;")
query = "UPDATE suite SET copydotdak = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
copydotdak = Cnf.find("Suite::%s::CopyDotDak" % (suite))
print "[CopyDotDak] Processing suite %s" % (suite)
if not copydotdak:
continue
suite = suite.lower()
c.execute(query, [copydotdak, suite])
c.execute("ALTER TABLE suite ADD COLUMN commentsdir TEXT;")
query = "UPDATE suite SET commentsdir = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
commentsdir = Cnf.find("Suite::%s::CommentsDir" % (suite))
print "[CommentsDir] Processing suite %s" % (suite)
if not commentsdir:
continue
suite = suite.lower()
c.execute(query, [commentsdir, suite])
c.execute("ALTER TABLE suite ADD COLUMN overridesuite TEXT;")
query = "UPDATE suite SET overridesuite = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
overridesuite = Cnf.find("Suite::%s::OverrideSuite" % (suite))
print "[OverrideSuite] Processing suite %s" % (suite)
if not overridesuite:
continue
suite = suite.lower()
c.execute(query, [overridesuite, suite])
c.execute("ALTER TABLE suite ADD COLUMN changelogbase TEXT;")
query = "UPDATE suite SET changelogbase = %s WHERE suite_name = %s" #: Update query
for suite in Cnf.subtree("Suite").list():
changelogbase = Cnf.find("Suite::%s::ChangeLogBase" % (suite))
print "[ChangeLogBase] Processing suite %s" % (suite)
if not changelogbase:
continue
suite = suite.lower()
c.execute(query, [changelogbase, suite])
c.execute("UPDATE config SET value = '8' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError("Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
import threading
from daklib import utils
from daklib.config import Config
from daklib.dbconn import DBConn, get_component_by_package_suite
from daklib.gpg import SignedFile
from daklib.regexes import html_escaping, re_html_escaping, re_version, re_spacestrip, \
re_contrib, re_nonfree, re_localhost, re_newlinespace, \
re_package, re_doc_directory
from daklib.dak_exceptions import ChangesUnicodeError
import daklib.daksubprocess
################################################################################
Cnf = None
Cnf = utils.get_conf()
printed = threading.local()
printed.copyrights = {}
package_relations = {} #: Store relations of packages for later output
# default is to not output html.
use_html = False
################################################################################
def usage (exit_code=0):
print """Usage: dak examine-package [PACKAGE]...
Check NEW package(s).
-h, --help show this help and exit
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
]
suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if Options.has_key("Help"): usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
if not Options.has_key("PatchName"):
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime( format )
session = DBConn().session()
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive'])
suites = [ s.suite_name for s in query ]
for suitename in suites:
print "Processing: " + suitename
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
architectures = get_suite_architectures(suite, skipall=True, session=session)
components = [ c.component_name for c in session.query(Component.component_name) ]
suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
#print "DEBUG: Working on %s" % (component)
workpath=os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
#print "EXCLUDING %s" % (entry)
continue
(fname, fext) = os.path.splitext(entry)
processfile=os.path.join(workpath, fname)
#print "Working: %s" % (processfile)
storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
#print "Storefile: %s" % (storename)
genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
for component in components:
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s"% (architecture)
packages = "Packages"
maxsuite = maxpackages
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxcontents)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxsuite)
def britney_changelog(packages, suite, session):
old = {}
current = {}
Cnf = utils.get_conf()
try:
q = session.execute("SELECT changelog FROM suite WHERE id = :suiteid",
{'suiteid': suite.suite_id})
brit_file = q.fetchone()[0]
except:
brit_file = None
if brit_file:
brit_file = os.path.join(Cnf['Dir::Root'], brit_file)
else:
return
q = session.execute("""SELECT s.source, s.version, sa.id
FROM source s, src_associations sa
WHERE sa.suite = :suiteid
AND sa.source = s.id""", {'suiteid': suite.suite_id})
for p in q.fetchall():
current[p[0]] = p[1]
for p in packages.keys():
if p[2] == "source":
old[p[0]] = p[1]
new = {}
for p in current.keys():
if p in old:
if apt_pkg.version_compare(current[p], old[p]) > 0:
new[p] = [current[p], old[p]]
else:
new[p] = [current[p], 0]
query = "SELECT source, changelog FROM changelogs WHERE"
for p in new.keys():
query += " source = '%s' AND version > '%s' AND version <= '%s'" \
% (p, new[p][1], new[p][0])
query += " AND architecture LIKE '%source%' AND distribution in \
('unstable', 'experimental', 'testing-proposed-updates') OR"
query += " False ORDER BY source, version DESC"
q = session.execute(query)
pu = None
brit = utils.open_file(brit_file, 'w')
for u in q:
if pu and pu != u[0]:
brit.write("\n")
brit.write("%s\n" % u[1])
pu = u[0]
if q.rowcount:
brit.write("\n\n\n")
for p in list(set(old.keys()).difference(current.keys())):
brit.write("REMOVED: %s %s\n" % (p, old[p]))
brit.flush()
brit.close()
def main():
global Cnf
keyrings = None
Cnf = utils.get_conf()
Arguments = [('h', "help", "Add-User::Options::Help"),
('k', "key", "Add-User::Options::Key", "HasArg"),
('u', "user", "Add-User::Options::User", "HasArg"),
]
for i in ["help"]:
key = "Add-User::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Add-User::Options")
if Options["help"]:
usage()
session = DBConn().session()
if not keyrings:
keyrings = get_active_keyring_paths()
cmd = ["gpg", "--with-colons", "--no-secmem-warning",
"--no-auto-check-trustdb", "--with-fingerprint",
"--no-default-keyring"]
cmd.extend(utils.gpg_keyring_args(keyrings).split())
cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]])
output = subprocess.check_output(cmd).rstrip()
m = re_gpg_fingerprint_colon.search(output)
if not m:
print(output)
utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output,
" [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ", "")
uid = ""
if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]:
uid = Cnf["Add-User::Options::User"]
name = Cnf["Add-User::Options::User"]
else:
u = re_user_address.search(output)
if not u:
print(output)
utils.fubar("0x%s: (2) No userid found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] ")))
uid = u.group(1)
n = re_user_name.search(output)
name = n.group(1)
# Look for all email addresses on the key.
emails = []
for line in output.split('\n'):
e = re_user_mails.search(line)
if not e:
continue
emails.append(e.group(2))
print("0x%s -> %s <%s> -> %s -> %s" % (Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key))
prompt = "Add user %s with above data (y/N) ? " % (uid)
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
# Create an account for the user?
summary = ""
# Now add user to the database.
# Note that we provide a session, so we're responsible for committing
uidobj = get_or_set_uid(uid, session=session)
uid_id = uidobj.uid_id
session.commit()
# Lets add user to the email-whitelist file if its configured.
if "Dinstall::MailWhiteList" in Cnf and Cnf["Dinstall::MailWhiteList"] != "":
f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
for mail in emails:
f.write(mail + '\n')
f.close()
print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id,
name, primary_key))
# Should we send mail to the newly added user?
if Cnf.find_b("Add-User::SendEmail"):
mail = name + "<" + emails[0] + ">"
Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
Subst["__PRIMARY_KEY__"] = primary_key
Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
Subst["__SUMMARY__"] = summary
new_add_message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"] + "/add-user.added")
utils.send_mail(new_add_message)
else:
uid = None
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Graph::Options::Help"),
('x',"extra-rrd","Graph::Options::Extra-Rrd", "HasArg"),
('r',"rrd","Graph::Options::Rrd", "HasArg"),
('i',"images","Graph::Options::Images", "HasArg"),
('n',"names","Graph::Options::Names", "HasArg")]
for i in [ "help" ]:
if not Cnf.has_key("Graph::Options::%s" % (i)):
Cnf["Graph::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Graph::Options")
if Options["Help"]:
usage()
names = []
if Cnf.has_key("Graph::Options::Names"):
for i in Cnf["Graph::Options::Names"].split(","):
names.append(i)
elif Cnf.has_key("Graph::Names"):
names = Cnf.value_list("Graph::Names")
else:
names = default_names
extra_rrdtool_args = []
if Cnf.has_key("Graph::Options::Extra-Rrd"):
for i in Cnf["Graph::Options::Extra-Rrd"].split(","):
f = open(i)
extra_rrdtool_args.extend(f.read().strip().split("\n"))
f.close()
elif Cnf.has_key("Graph::Extra-Rrd"):
for i in Cnf.value_list("Graph::Extra-Rrd"):
f = open(i)
extra_rrdtool_args.extend(f.read().strip().split("\n"))
f.close()
if Cnf.has_key("Graph::Options::Rrd"):
rrd_dir = Cnf["Graph::Options::Rrd"]
elif Cnf.has_key("Dir::Rrd"):
rrd_dir = Cnf["Dir::Rrd"]
else:
print >> sys.stderr, "No directory to read RRD files from\n"
sys.exit(1)
if Cnf.has_key("Graph::Options::Images"):
image_dir = Cnf["Graph::Options::Images"]
else:
print >> sys.stderr, "No directory to write graph images to\n"
sys.exit(1)
for name in names:
stdargs = [rrd_dir, image_dir, name, extra_rrdtool_args]
graph(*(stdargs+['day', 'day', 'now-1d']))
graph(*(stdargs+['week', 'week', 'now-1w']))
graph(*(stdargs+['month', 'month', 'now-1m']))
graph(*(stdargs+['year', 'year', 'now-1y']))
graph(*(stdargs+['5years', '5 years', 'now-5y', True]))
graph(*(stdargs+['10years', '10 years', 'now-10y', True]))
def remove(session,
reason,
suites,
removals,
whoami=None,
partial=False,
components=None,
done_bugs=None,
date=None,
carbon_copy=None,
close_related_bugs=False):
"""Batch remove a number of packages
Verify that the files listed in the Files field of the .dsc are
those expected given the announced Format.
@type session: SQLA Session
@param session: The database session in use
@type reason: string
@param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
@type suites: list
@param suites: A list of the suite names in which the removal should occur
@type removals: list
@param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
for 4 items from the database (in order): package, version, architecture, (database) id.
For source packages, the "architecture" should be set to "source".
@type partial: bool
@param partial: Whether the removal is "partial" (e.g. architecture specific).
@type components: list
@param components: List of components involved in a partial removal. Can be an empty list to not restrict the
removal to any components.
@type whoami: string
@param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
@type date: string
@param date: The date of the removal. Defaults to commands.getoutput("date -R")
@type done_bugs: list
@param done_bugs: A list of bugs to be closed when doing this removal.
@type close_related_bugs: bool
@param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
for more than one suite.
@type carbon_copy: list
@param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
"dak rm".
@rtype: None
@return: Nothing
"""
# Generate the summary of what's to be removed
d = {}
summary = ""
sources = []
binaries = []
whitelists = []
versions = []
suite_ids_list = []
suites_list = utils.join_with_commas_and(suites)
cnf = utils.get_conf()
con_components = ''
#######################################################################################################
if not reason:
raise ValueError("Empty removal reason not permitted")
if not removals:
raise ValueError("Nothing to remove!?")
if not suites:
raise ValueError("Removals without a suite!?")
if whoami is None:
whoami = utils.whoami()
if date is None:
date = commands.getoutput("date -R")
if partial and components:
component_ids_list = []
for componentname in components:
component = get_component(componentname, session=session)
if component is None:
raise ValueError("component '%s' not recognised." %
componentname)
else:
component_ids_list.append(component.component_id)
if component_ids_list:
con_components = "AND component IN (%s)" % ", ".join(
[str(i) for i in component_ids_list])
for i in removals:
package = i[0]
version = i[1]
architecture = i[2]
if package not in d:
d[package] = {}
if version not in d[package]:
d[package][version] = []
if architecture not in d[package][version]:
d[package][version].append(architecture)
for package in sorted(d):
versions = sorted(d[package], cmp=apt_pkg.version_compare)
for version in versions:
d[package][version].sort(utils.arch_compare_sw)
summary += "%10s | %10s | %s\n" % (package, version, ", ".join(
d[package][version]))
for package in summary.split("\n"):
for row in package.split("\n"):
element = row.split("|")
if len(element) == 3:
if element[2].find("source") > 0:
sources.append(
"%s_%s" %
tuple(elem.strip(" ") for elem in element[:2]))
element[2] = sub("source\s?,?", "", element[2]).strip(" ")
if element[2]:
binaries.append("%s_%s [%s]" %
tuple(elem.strip(" ") for elem in element))
dsc_type_id = get_override_type('dsc', session).overridetype_id
deb_type_id = get_override_type('deb', session).overridetype_id
for suite in suites:
s = get_suite(suite, session=session)
if s is not None:
suite_ids_list.append(s.suite_id)
whitelists.append(s.mail_whitelist)
#######################################################################################################
log_filename = cnf["Rm::LogFile"]
log822_filename = cnf["Rm::LogFile822"]
with utils.open_file(log_filename, "a") as logfile, utils.open_file(
log822_filename, "a") as logfile822:
fcntl.lockf(logfile, fcntl.LOCK_EX)
fcntl.lockf(logfile822, fcntl.LOCK_EX)
logfile.write(
"=========================================================================\n"
)
logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
logfile.write("Removed the following packages from %s:\n\n%s" %
(suites_list, summary))
if done_bugs:
logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
logfile.write(
"\n------------------- Reason -------------------\n%s\n" % reason)
logfile.write("----------------------------------------------\n")
logfile822.write("Date: %s\n" % date)
logfile822.write("Ftpmaster: %s\n" % whoami)
logfile822.write("Suite: %s\n" % suites_list)
if sources:
logfile822.write("Sources:\n")
for source in sources:
logfile822.write(" %s\n" % source)
if binaries:
logfile822.write("Binaries:\n")
for binary in binaries:
logfile822.write(" %s\n" % binary)
logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
if done_bugs:
logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
for i in removals:
package = i[0]
architecture = i[2]
package_id = i[3]
for suite_id in suite_ids_list:
if architecture == "source":
session.execute(
"DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
{
'packageid': package_id,
'suiteid': suite_id
})
else:
session.execute(
"DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
{
'packageid': package_id,
'suiteid': suite_id
})
# Delete from the override file
if not partial:
if architecture == "source":
type_id = dsc_type_id
else:
type_id = deb_type_id
# TODO: Fix this properly to remove the remaining non-bind argument
session.execute(
"DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s"
% (con_components), {
'package': package,
'typeid': type_id,
'suiteid': suite_id
})
session.commit()
# ### REMOVAL COMPLETE - send mail time ### #
# If we don't have a Bug server configured, we're done
if "Dinstall::BugServer" not in cnf:
if done_bugs or close_related_bugs:
utils.warn(
"Cannot send mail to BugServer as Dinstall::BugServer is not configured"
)
logfile.write(
"=========================================================================\n"
)
logfile822.write("\n")
return
# read common subst variables for all bug closure mails
Subst_common = {}
Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
Subst_common["__CC__"] = "X-DAK: dak rm"
if carbon_copy:
Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
Subst_common["__SUITE_LIST__"] = suites_list
Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (
suites_list)
Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst_common["__WHOAMI__"] = whoami
# Send the bug closing messages
if done_bugs:
Subst_close_rm = Subst_common
bcc = []
if cnf.find("Dinstall::Bcc") != "":
bcc.append(cnf["Dinstall::Bcc"])
if cnf.find("Rm::Bcc") != "":
bcc.append(cnf["Rm::Bcc"])
if bcc:
Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst_close_rm["__BCC__"] = "X-Filler: 42"
summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (
summary, reason)
summarymail += "----------------------------------------------\n"
Subst_close_rm["__SUMMARY__"] = summarymail
for bug in done_bugs:
Subst_close_rm["__BUG_NUMBER__"] = bug
if close_related_bugs:
mail_message = utils.TemplateSubst(
Subst_close_rm,
cnf["Dir::Templates"] + "/rm.bug-close-with-related")
else:
mail_message = utils.TemplateSubst(
Subst_close_rm,
cnf["Dir::Templates"] + "/rm.bug-close")
utils.send_mail(mail_message, whitelists=whitelists)
# close associated bug reports
if close_related_bugs:
Subst_close_other = Subst_common
bcc = []
wnpp = utils.parse_wnpp_bug_file()
versions = list(set([re_bin_only_nmu.sub('', v)
for v in versions]))
if len(versions) == 1:
Subst_close_other["__VERSION__"] = versions[0]
else:
logfile.write(
"=========================================================================\n"
)
logfile822.write("\n")
raise ValueError(
"Closing bugs with multiple package versions is not supported. Do it yourself."
)
if bcc:
Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst_close_other["__BCC__"] = "X-Filler: 42"
# at this point, I just assume, that the first closed bug gives
# some useful information on why the package got removed
Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
if len(sources) == 1:
source_pkg = source.split("_", 1)[0]
else:
logfile.write(
"=========================================================================\n"
)
logfile822.write("\n")
raise ValueError(
"Closing bugs for multiple source packages is not supported. Please do it yourself."
)
Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
Subst_close_other["__SOURCE__"] = source_pkg
merged_bugs = set()
other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open',
'status', 'forwarded')
if other_bugs:
for bugno in other_bugs:
if bugno not in merged_bugs:
for bug in bts.get_status(bugno):
for merged in bug.mergedwith:
other_bugs.remove(merged)
merged_bugs.add(merged)
logfile.write("Also closing bug(s):")
logfile822.write("Also-Bugs:")
for bug in other_bugs:
Subst_close_other["__BUG_NUMBER_ALSO__"] += str(
bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
logfile.write(" " + str(bug))
logfile822.write(" " + str(bug))
logfile.write("\n")
logfile822.write("\n")
if source_pkg in wnpp:
logfile.write("Also closing WNPP bug(s):")
logfile822.write("Also-WNPP:")
for bug in wnpp[source_pkg]:
# the wnpp-rm file we parse also contains our removal
# bugs, filtering that out
if bug != Subst_close_other["__BUG_NUMBER__"]:
Subst_close_other["__BUG_NUMBER_ALSO__"] += str(
bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
logfile.write(" " + str(bug))
logfile822.write(" " + str(bug))
logfile.write("\n")
logfile822.write("\n")
mail_message = utils.TemplateSubst(
Subst_close_other,
cnf["Dir::Templates"] + "/rm.bug-close-related")
if Subst_close_other["__BUG_NUMBER_ALSO__"]:
utils.send_mail(mail_message)
logfile.write(
"=========================================================================\n"
)
logfile822.write("\n")
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "no-act", "Generate-Index-Diffs::Options::NoAct"),
('v', "verbose", "Generate-Index-Diffs::Options::Verbose"),
]
suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if "Help" in Options:
usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
# can only be set via config at the moment
max_parallel = int(Options.get("MaxParallel", "8"))
if "PatchName" not in Options:
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime(format)
session = DBConn().session()
pending_tasks = []
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
archives = utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
suites = [s.suite_name for s in query]
for suitename in suites:
print("Processing: " + suitename)
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print("Skipping: " + suite + " (untouchable)")
continue
skip_all = True
if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all:
skip_all = False
architectures = get_suite_architectures(suite, skipall=skip_all, session=session)
components = [c.component_name for c in session.query(Component.component_name)]
suite_suffix = utils.suite_suffix(suitename)
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
merged_pdiffs = suiteobj.merged_pdiffs
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
workpath = os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
continue
(fname, fext) = os.path.splitext(entry)
processfile = os.path.join(workpath, fname)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs)
pending_tasks.append(coroutine)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s" % architecture
packages = "Packages"
maxsuite = maxpackages
for component in components:
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs)
pending_tasks.append(coroutine)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs)
pending_tasks.append(coroutine)
asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h', "help", "Queue-Report::Options::Help"),
('n', "new", "Queue-Report::Options::New"),
('8', '822', "Queue-Report::Options::822"),
('s', "sort", "Queue-Report::Options::Sort", "HasArg"),
('a', "age", "Queue-Report::Options::Age", "HasArg"),
('r', "rrd", "Queue-Report::Options::Rrd", "HasArg"),
('d', "directories", "Queue-Report::Options::Directories",
"HasArg")]
for i in ["help"]:
key = "Queue-Report::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Queue-Report::Options")
if Options["Help"]:
usage()
if "Queue-Report::Options::New" in Cnf:
header()
queue_names = []
if "Queue-Report::Options::Directories" in Cnf:
for i in Cnf["Queue-Report::Options::Directories"].split(","):
queue_names.append(i)
elif "Queue-Report::Directories" in Cnf:
queue_names = Cnf.value_list("Queue-Report::Directories")
else:
queue_names = ["byhand", "new"]
if "Queue-Report::Options::Rrd" in Cnf:
rrd_dir = Cnf["Queue-Report::Options::Rrd"]
elif "Dir::Rrd" in Cnf:
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
f = None
if "Queue-Report::Options::822" in Cnf:
# Open the report file
f = sys.stdout
filename822 = Cnf.get("Queue-Report::ReportLocations::822Location")
if filename822:
f = open(filename822, "w")
session = DBConn().session()
for queue_name in queue_names:
queue = session.query(PolicyQueue).filter_by(
queue_name=queue_name).first()
if queue is not None:
process_queue(queue, f, rrd_dir)
else:
utils.warn("Cannot find queue %s" % queue_name)
if "Queue-Report::Options::822" in Cnf:
f.close()
if "Queue-Report::Options::New" in Cnf:
footer()
def britney_changelog(packages, suite, session):
old = {}
current = {}
Cnf = utils.get_conf()
try:
q = session.execute("SELECT changelog FROM suite WHERE id = :suiteid",
{'suiteid': suite.suite_id})
brit_file = q.fetchone()[0]
except:
brit_file = None
if brit_file:
brit_file = os.path.join(Cnf['Dir::Root'], brit_file)
else:
return
q = session.execute(
"""SELECT s.source, s.version, sa.id
FROM source s, src_associations sa
WHERE sa.suite = :suiteid
AND sa.source = s.id""",
{'suiteid': suite.suite_id})
for p in q.fetchall():
current[p[0]] = p[1]
for p in packages.keys():
if p[2] == "source":
old[p[0]] = p[1]
new = {}
for p in current.keys():
if p in old.keys():
if apt_pkg.version_compare(current[p], old[p]) > 0:
new[p] = [current[p], old[p]]
else:
new[p] = [current[p], 0]
query = "SELECT source, changelog FROM changelogs WHERE"
for p in new.keys():
query += " source = '%s' AND version > '%s' AND version <= '%s'" \
% (p, new[p][1], new[p][0])
query += " AND architecture LIKE '%source%' AND distribution in \
('unstable', 'experimental', 'testing-proposed-updates') OR"
query += " False ORDER BY source, version DESC"
q = session.execute(query)
pu = None
brit = utils.open_file(brit_file, 'w')
for u in q:
if pu and pu != u[0]:
brit.write("\n")
brit.write("%s\n" % u[1])
pu = u[0]
if q.rowcount:
brit.write("\n\n\n")
for p in list(set(old.keys()).difference(current.keys())):
brit.write("REMOVED: %s %s\n" % (p, old[p]))
brit.flush()
brit.close()
