def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
global row_number
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
query = '''SELECT source
FROM source_suite
WHERE source = :source
AND suite_name IN ('unstable', 'experimental')'''
if not session.execute(query, {'source': source}).rowcount:
trclass += " sourceNEW"
session.commit()
if row_number % 2 != 0:
print("<tr class=\"%s even\">" % (trclass))
else:
print("<tr class=\"%s odd\">" % (trclass))
if "sourceNEW" in trclass:
print("<td class=\"package\">%s</td>" % (source))
else:
print("<td class=\"package\"><a href=\"https://tracker.debian.org/pkg/%(source)s\">%(source)s</a></td>" % {'source': source})
print("<td class=\"version\">")
for vers in version.split():
print("<a href=\"new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers)))
print("</td>")
print("<td class=\"arch\">%s</td>" % (arch))
print("<td class=\"distribution\">")
for dist in distribution:
print("%s<br/>" % (dist))
print("</td>")
print("<td class=\"age\"><abbr title=\"%s\">%s</abbr></td>" % (
datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime('%a, %d %b %Y %T UTC'),
time_pp(last_mod),
))
(name, mail) = maint.split(":", 1)
print("<td class=\"upload-data\">")
print("<span class=\"maintainer\">Maintainer: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
(name, mail) = changedby.split(":", 1)
print("<span class=\"changed-by\">Changed-By: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
if sponsor:
print("<span class=\"sponsor\">Sponsor: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br/>" % (utils.html_escape(sponsor), utils.html_escape(sponsor)))
print("<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint))
print("</td>")
print("<td class=\"closes\">")
for close in closes:
print("<a href=\"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close)))
print("</td></tr>")
row_number += 1
def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
global row_number
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
query = '''SELECT source
FROM source_suite
WHERE source = :source
AND suite_name IN ('unstable', 'experimental')'''
if not session.execute(query, {'source': source}).rowcount:
trclass += " sourceNEW"
session.commit()
if row_number % 2 != 0:
print("<tr class=\"%s even\">" % (trclass))
else:
print("<tr class=\"%s odd\">" % (trclass))
if "sourceNEW" in trclass:
print("<td class=\"package\">%s</td>" % (source))
else:
print("<td class=\"package\"><a href=\"https://tracker.debian.org/pkg/%(source)s\">%(source)s</a></td>" % {'source': source})
print("<td class=\"version\">")
for vers in version.split():
print("<a href=\"new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers)))
print("</td>")
print("<td class=\"arch\">%s</td>" % (arch))
print("<td class=\"distribution\">")
for dist in distribution:
print("%s<br/>" % (dist))
print("</td>")
print("<td class=\"age\"><abbr title=\"%s\">%s</abbr></td>" % (
datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime('%a, %d %b %Y %T UTC'),
time_pp(last_mod),
))
(name, mail) = maint.split(":", 1)
print("<td class=\"upload-data\">")
print("<span class=\"maintainer\">Maintainer: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
(name, mail) = changedby.split(":", 1)
print("<span class=\"changed-by\">Changed-By: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
if sponsor:
print("<span class=\"sponsor\">Sponsor: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br/>" % (utils.html_escape(sponsor), utils.html_escape(sponsor)))
print("<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint))
print("</td>")
print("<td class=\"closes\">")
for close in closes:
print("<a href=\"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close)))
print("</td></tr>")
row_number += 1
def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
global row_number
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
if not len(session.query(DBSource).filter_by(source = source).all()):
trclass += " binNEW"
session.commit()
if row_number % 2 != 0:
print "<tr class=\"%s even\">" % (trclass)
else:
print "<tr class=\"%s odd\">" % (trclass)
if "binNEW" in trclass:
print "<td class=\"package\">%s</td>" % (source)
else:
print "<td class=\"package\"><a href=\"http://packages.qa.debian.org/%(source)s\">%(source)s</a></td>" % {'source': source}
print "<td class=\"version\">"
for vers in version.split():
print "<a href=\"new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers))
print "</td>"
print "<td class=\"arch\">%s</td>" % (arch)
print "<td class=\"distribution\">"
for dist in distribution:
print "%s<br/>" % (dist)
print "</td>"
print "<td class=\"age\">%s</td>" % (last_mod)
(name, mail) = maint.split(":", 1)
print "<td class=\"upload-data\">"
print "<span class=\"maintainer\">Maintainer: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
(name, mail) = changedby.split(":", 1)
print "<span class=\"changed-by\">Changed-By: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
if sponsor:
try:
(login, domain) = sponsor.split("@", 1)
print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br/>" % (utils.html_escape(login), utils.html_escape(login))
except Exception as e:
pass
print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
print "</td>"
print "<td class=\"closes\">"
for close in closes:
print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close))
print "</td></tr>"
row_number+=1
def acl_deny(acl_name, fingerprint, sources):
tbl = DBConn().tbl_acl_per_source
session = DBConn().session()
acl_id = session.query(ACL).filter_by(name=acl_name).one().id
fingerprint_id = session.query(Fingerprint).filter_by(fingerprint=fingerprint).one().fingerprint_id
# TODO: check that fpr is in ACL
for source in sources:
result = session.execute(tbl.delete().where(tbl.c.acl_id == acl_id).where(tbl.c.fingerprint_id == fingerprint_id).where(tbl.c.source == source))
if result.rowcount < 1:
print("W: Tried to deny uploads of '{}', but was not allowed before.".format(source))
session.commit()
def acl_set_fingerprints(acl_name, entries):
session = DBConn().session()
acl = session.query(ACL).filter_by(name=acl_name).one()
acl.fingerprints.clear()
for entry in entries:
entry = entry.strip()
if entry.startswith('#') or len(entry) == 0:
continue
fps = get_fingerprint(entry, session)
if len(fps) == 0:
print("Unknown key for '{0}'".format(entry))
else:
acl.fingerprints.update(fps)
session.commit()
class ImportThread(threading.Thread):
def __init__(self, parent, queue):
threading.Thread.__init__(self)
self.queue = queue
self.session = DBConn().session()
self.parent = parent
self.die = False
def plsDie(self):
self.die = True
def run(self):
while True:
try:
if self.die:
return
to_import = self.queue.dequeue()
if not to_import:
return
print( "Directory %s, file %7d, (%s)" % (to_import.dirpath[-10:], to_import.count, to_import.changesfile) )
changes = Changes()
changes.changes_file = to_import.changesfile
changesfile = os.path.join(to_import.dirpath, to_import.changesfile)
changes.changes = parse_changes(changesfile, signing_rules=-1)
changes.changes["fingerprint"] = check_signature(changesfile)
changes.add_known_changes(to_import.dirpath, session=self.session)
self.session.commit()
except InvalidDscError as line:
warn("syntax error in .dsc file '%s', line %s." % (f, line))
except ChangesUnicodeError:
warn("found invalid changes file, not properly utf-8 encoded")
except KeyboardInterrupt:
print("Caught C-c; on ImportThread. terminating.")
self.parent.plsDie()
sys.exit(1)
except:
self.parent.plsDie()
sys.exit(1)
def acl_allow(acl_name, fingerprint, sources):
tbl = DBConn().tbl_acl_per_source
session = DBConn().session()
acl_id = session.query(ACL).filter_by(name=acl_name).one().id
fingerprint_id = session.query(Fingerprint).filter_by(fingerprint=fingerprint).one().fingerprint_id
# TODO: check that fpr is in ACL
data = [
{
'acl_id': acl_id,
'fingerprint_id': fingerprint_id,
'source': source,
'reason': 'set by {} via CLI'.format(os.environ.get('USER', '(unknown)')),
}
for source in sources
]
session.execute(tbl.insert(), data)
session.commit()
def main():
global Cnf
keyrings = None
Cnf = utils.get_conf()
Arguments = [
('h', "help", "Add-User::Options::Help"),
('k', "key", "Add-User::Options::Key", "HasArg"),
('u', "user", "Add-User::Options::User", "HasArg"),
]
for i in ["help"]:
key = "Add-User::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Add-User::Options")
if Options["help"]:
usage()
session = DBConn().session()
if not keyrings:
keyrings = get_active_keyring_paths()
cmd = [
"gpg", "--with-colons", "--no-secmem-warning",
"--no-auto-check-trustdb", "--with-fingerprint", "--no-default-keyring"
]
cmd.extend(utils.gpg_keyring_args(keyrings).split())
cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]])
output = subprocess.check_output(cmd).rstrip()
m = re_gpg_fingerprint_colon.search(output)
if not m:
print(output)
utils.fubar(
"0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"],
utils.prefix_multi_line_string(output, " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ", "")
uid = ""
if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]:
uid = Cnf["Add-User::Options::User"]
name = Cnf["Add-User::Options::User"]
else:
u = re_user_address.search(output)
if not u:
print(output)
utils.fubar(
"0x%s: (2) No userid found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"],
utils.prefix_multi_line_string(output, " [GPG output:] ")))
uid = u.group(1)
n = re_user_name.search(output)
name = n.group(1)
# Look for all email addresses on the key.
emails = []
for line in output.split('\n'):
e = re_user_mails.search(line)
if not e:
continue
emails.append(e.group(2))
print("0x%s -> %s <%s> -> %s -> %s" %
(Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key))
prompt = "Add user %s with above data (y/N) ? " % (uid)
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
# Create an account for the user?
summary = ""
# Now add user to the database.
# Note that we provide a session, so we're responsible for committing
uidobj = get_or_set_uid(uid, session=session)
uid_id = uidobj.uid_id
session.commit()
# Lets add user to the email-whitelist file if its configured.
if "Dinstall::MailWhiteList" in Cnf and Cnf[
"Dinstall::MailWhiteList"] != "":
f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
for mail in emails:
f.write(mail + '\n')
f.close()
print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" %
(uid, uid_id, name, primary_key))
# Should we send mail to the newly added user?
if Cnf.find_b("Add-User::SendEmail"):
mail = name + "<" + emails[0] + ">"
Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
Subst["__PRIMARY_KEY__"] = primary_key
Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
Subst["__SUMMARY__"] = summary
new_add_message = utils.TemplateSubst(
Subst, Cnf["Dir::Templates"] + "/add-user.added")
utils.send_mail(new_add_message)
else:
uid = None
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h', "help", "Generate-Packages-Sources::Options::Help"),
('a', 'archive',
'Generate-Packages-Sources::Options::Archive', 'HasArg'),
('s', "suite", "Generate-Packages-Sources::Options::Suite",
'HasArg'),
('f', "force", "Generate-Packages-Sources::Options::Force"),
('o', 'option', '', 'ArbItem')]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
import daklib.utils
if Options.has_key("Suite"):
suites = []
suite_names = daklib.utils.split_args(Options['Suite'])
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
archive_names = daklib.utils.split_args(Options['Archive'])
query = query.join(Suite.archive).filter(
Archive.archive_name.in_(archive_names))
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log(msg)
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess received signal ', msg])
else:
logger.log(['E: ', msg])
# Lock tables so that nobody can change things underneath us
session.execute("LOCK TABLE src_associations IN SHARE MODE")
session.execute("LOCK TABLE bin_associations IN SHARE MODE")
for s in suites:
component_ids = [c.component_id for c in s.components]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar(
"Refusing to touch %s (untouchable and not forced)" %
s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c],
callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c],
callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages,
[s.suite_id, c, a.arch_id, 'deb'],
callback=parse_results)
pool.apply_async(generate_packages,
[s.suite_id, c, a.arch_id, 'udeb'],
callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
def main():
global Cnf
keyrings = None
Cnf = utils.get_conf()
Arguments = [('h', "help", "Add-User::Options::Help"),
('k', "key", "Add-User::Options::Key", "HasArg"),
('u', "user", "Add-User::Options::User", "HasArg"),
]
for i in ["help"]:
key = "Add-User::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Add-User::Options")
if Options["help"]:
usage()
session = DBConn().session()
if not keyrings:
keyrings = get_active_keyring_paths()
cmd = ["gpg", "--with-colons", "--no-secmem-warning",
"--no-auto-check-trustdb", "--with-fingerprint",
"--no-default-keyring"]
cmd.extend(utils.gpg_keyring_args(keyrings).split())
cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]])
output = subprocess.check_output(cmd).rstrip()
m = re_gpg_fingerprint_colon.search(output)
if not m:
print(output)
utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output,
" [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ", "")
uid = ""
if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]:
uid = Cnf["Add-User::Options::User"]
name = Cnf["Add-User::Options::User"]
else:
u = re_user_address.search(output)
if not u:
print(output)
utils.fubar("0x%s: (2) No userid found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] ")))
uid = u.group(1)
n = re_user_name.search(output)
name = n.group(1)
# Look for all email addresses on the key.
emails = []
for line in output.split('\n'):
e = re_user_mails.search(line)
if not e:
continue
emails.append(e.group(2))
print("0x%s -> %s <%s> -> %s -> %s" % (Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key))
prompt = "Add user %s with above data (y/N) ? " % (uid)
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
# Create an account for the user?
summary = ""
# Now add user to the database.
# Note that we provide a session, so we're responsible for committing
uidobj = get_or_set_uid(uid, session=session)
uid_id = uidobj.uid_id
session.commit()
# Lets add user to the email-whitelist file if its configured.
if "Dinstall::MailWhiteList" in Cnf and Cnf["Dinstall::MailWhiteList"] != "":
f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
for mail in emails:
f.write(mail + '\n')
f.close()
print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id,
name, primary_key))
# Should we send mail to the newly added user?
if Cnf.find_b("Add-User::SendEmail"):
mail = name + "<" + emails[0] + ">"
Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
Subst["__PRIMARY_KEY__"] = primary_key
Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
Subst["__SUMMARY__"] = summary
new_add_message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"] + "/add-user.added")
utils.send_mail(new_add_message)
else:
uid = None
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
('a','archive','Generate-Packages-Sources::Options::Archive','HasArg'),
('s',"suite","Generate-Packages-Sources::Options::Suite"),
('f',"force","Generate-Packages-Sources::Options::Force"),
('o','option','','ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
if Options.has_key("Suite"):
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess recieved signal ', msg])
else:
logger.log(['E: ', msg])
for s in suites:
component_ids = [ c.component_id for c in s.components ]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c], callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=parse_results)
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
