def check(self, upload):
changes = upload.changes
control = changes.changes
fn = changes.filename
for field in ('Distribution', 'Source', 'Binary', 'Architecture', 'Version', 'Maintainer', 'Files', 'Changes', 'Description'):
if field not in control:
raise Reject('{0}: misses mandatory field {1}'.format(fn, field))
check_fields_for_valid_utf8(fn, control)
source_match = re_field_source.match(control['Source'])
if not source_match:
raise Reject('{0}: Invalid Source field'.format(fn))
version_match = re_field_version.match(control['Version'])
if not version_match:
raise Reject('{0}: Invalid Version field'.format(fn))
version_without_epoch = version_match.group('without_epoch')
match = re_file_changes.match(fn)
if not match:
raise Reject('{0}: Does not match re_file_changes'.format(fn))
if match.group('package') != source_match.group('package'):
raise Reject('{0}: Filename does not match Source field'.format(fn))
if match.group('version') != version_without_epoch:
raise Reject('{0}: Filename does not match Version field'.format(fn))
for bn in changes.binary_names:
if not re_field_package.match(bn):
raise Reject('{0}: Invalid binary package name {1}'.format(fn, bn))
if 'source' in changes.architectures and changes.source is None:
raise Reject("Changes has architecture source, but no source found.")
if changes.source is not None and 'source' not in changes.architectures:
raise Reject("Upload includes source, but changes does not say so.")
try:
fix_maintainer(changes.changes['Maintainer'])
except ParseMaintError as e:
raise Reject('{0}: Failed to parse Maintainer field: {1}'.format(changes.filename, e))
try:
changed_by = changes.changes.get('Changed-By')
if changed_by is not None:
fix_maintainer(changed_by)
except ParseMaintError as e:
raise Reject('{0}: Failed to parse Changed-By field: {1}'.format(changes.filename, e))
if len(changes.files) == 0:
raise Reject("Changes includes no files.")
for bugnum in changes.closed_bugs:
if not re_isanum.match(bugnum):
raise Reject('{0}: "{1}" in Closes field is not a number'.format(changes.filename, bugnum))
return True
def assertValid(self, input, a, b, c, d):
a_, b_, c_, d_ = fix_maintainer(input)
self.assertEqual(a, a_)
self.assertEqual(b, b_)
self.assertEqual(c, c_)
self.assertEqual(d, d_)
def assertValid(self, input, a, b, c, d):
a_, b_, c_, d_ = fix_maintainer(input)
self.assertEqual(a, a_)
self.assertEqual(b, b_)
self.assertEqual(c, c_)
self.assertEqual(d, d_)
def _notify_uploader(self):
cnf = Config()
bcc = 'X-DAK: dak process-command'
if 'Dinstall::Bcc' in cnf:
bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
cc = set(fix_maintainer(address)[1] for address in self.cc)
subst = {
'__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
'__MAINTAINER_TO__': fix_maintainer(self.uploader)[1],
'__CC__': ", ".join(cc),
'__BCC__': bcc,
'__RESULTS__': "\n".join(self.result),
'__FILENAME__': self.filename,
}
message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-command.processed'))
send_mail(message)
def _notify_uploader(self):
cnf = Config()
bcc = "X-DAK: dak process-command"
if "Dinstall::Bcc" in cnf:
bcc = "{0}\nBcc: {1}".format(bcc, cnf["Dinstall::Bcc"])
cc = set(fix_maintainer(address)[1] for address in self.cc)
subst = {
"__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"],
"__MAINTAINER_TO__": fix_maintainer(self.uploader)[1],
"__CC__": ", ".join(cc),
"__BCC__": bcc,
"__RESULTS__": "\n".join(self.result),
"__FILENAME__": self.filename,
}
message = TemplateSubst(subst, os.path.join(cnf["Dir::Templates"], "process-command.processed"))
send_mail(message)
def _notify_uploader(self):
cnf = Config()
bcc = 'X-DAK: dak process-command'
if 'Dinstall::Bcc' in cnf:
bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
cc = set(fix_maintainer(address)[1] for address in self.cc)
subst = {
'__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
'__MAINTAINER_TO__': fix_maintainer(self.uploader)[1],
'__CC__': ", ".join(cc),
'__BCC__': bcc,
'__RESULTS__': "\n".join(self.result),
'__FILENAME__': self.filename,
}
message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-command.processed'))
send_mail(message)
def _subst_for_upload(upload):
""" Prepare substitutions used for announce mails.
@type upload: L{daklib.upload.Source} or L{daklib.upload.Binary}
@param upload: upload to handle
@rtype: dict
@returns: A dict of substition values for use by L{daklib.utils.TemplateSubst}
"""
cnf = Config()
maintainer = upload.maintainer or cnf['Dinstall::MyEmailAddress']
changed_by = upload.changed_by or maintainer
if upload.sourceful:
maintainer_to = mail_addresses_for_upload(maintainer, changed_by, upload.fingerprint)
else:
maintainer_to = mail_addresses_for_upload(maintainer, maintainer, upload.fingerprint)
bcc = 'X-DAK: dak {0}'.format(upload.program)
if 'Dinstall::Bcc' in cnf:
bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
subst = {
'__DISTRO__': cnf['Dinstall::MyDistribution'],
'__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
'__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
'__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
'__REJECTOR_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
'__MANUAL_REJECT_MESSAGE__': '',
'__BCC__': bcc,
'__MAINTAINER__': changed_by,
'__MAINTAINER_FROM__': fix_maintainer(changed_by)[1],
'__MAINTAINER_TO__': ', '.join(maintainer_to),
'__CHANGES_FILENAME__': upload.changes_filename,
'__FILE_CONTENTS__': upload.changes,
'__SOURCE__': upload.source,
'__VERSION__': upload.version,
'__ARCHITECTURE__': upload.architecture,
'__WARNINGS__': '\n'.join(upload.warnings),
}
override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
if override_maintainer:
subst['__MAINTAINER_FROM__'] = subst['__MAINTAINER_TO__'] = override_maintainer
return subst
def process_queue(queue, log, rrd_dir):
msg = ""
type = queue.queue_name
session = DBConn().session()
# Divide the .changes into per-source groups
per_source = {}
total_pending = 0
for upload in queue.uploads:
source = upload.changes.source
if source not in per_source:
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["processed"] = ""
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action():
per_source[source][
"processed"] = "PENDING %s" % handler.get_action()
total_pending += 1
per_source[source]["list"].append(upload)
per_source[source]["list"].sort(key=lambda x: x.changes.created,
reverse=True)
# Determine oldest time and have note status for each source group
for source in per_source.keys():
source_list = per_source[source]["list"]
first = source_list[0]
oldest = time.mktime(first.changes.created.timetuple())
have_note = 0
for d in per_source[source]["list"]:
mtime = time.mktime(d.changes.created.timetuple())
if "Queue-Report::Options::New" in Cnf:
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d.policy_queue, d.changes.source,
d.changes.version)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0 # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1 # some
else:
per_source[source]["note_state"] = 2 # all
per_source_items = per_source.items()
per_source_items.sort(key=functools.cmp_to_key(sg_compare))
update_graph_database(rrd_dir, type, len(per_source_items),
len(queue.uploads))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
try:
logins = get_logins_from_ldap()
except:
logins = dict()
for i in per_source_items:
maintainer = {}
maint = ""
distribution = ""
closes = ""
fingerprint = ""
changeby = {}
changedby = ""
sponsor = ""
filename = i[1]["list"][0].changes.changesname
last_modified = time.time() - i[1]["oldest"]
source = i[1]["list"][0].changes.source
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0].binaries
binary = ', '.join([b.package for b in binary_list])
arches = set()
versions = set()
for j in i[1]["list"]:
dbc = j.changes
changesbase = dbc.changesname
if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer(dbc.maintainer)
except ParseMaintError as msg:
print("Problems while parsing maintainer address\n")
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint = "%s:%s" % (maintainer["maintainername"],
maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer(dbc.changedby)
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby = "%s:%s" % (changeby["changedbyname"],
changeby["changedbyemail"])
distribution = dbc.distribution.split()
closes = dbc.closes
fingerprint = dbc.fingerprint
sponsor_name = get_uid_from_fingerprint(fingerprint).name
sponsor_login = get_uid_from_fingerprint(fingerprint).uid
if '@' in sponsor_login:
if fingerprint in logins:
sponsor_login = logins[fingerprint]
if (sponsor_name != maintainer["maintainername"]
and sponsor_name != changeby["changedbyname"]
and sponsor_login + '@debian.org' !=
maintainer["maintaineremail"]
and sponsor_name != changeby["changedbyemail"]):
sponsor = sponsor_login
for arch in dbc.architecture.split():
arches.add(arch)
versions.add(dbc.version)
arches_list = sorted(arches, key=utils.ArchKey)
arch_list = " ".join(arches_list)
version_list = " ".join(sorted(versions, reverse=True))
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([
source, binary, version_list, arch_list,
per_source[source]["processed"], note, last_modified, maint,
distribution, closes, fingerprint, sponsor, changedby, filename
])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if "Queue-Report::Options::Age" in Cnf:
age = Cnf["Queue-Report::Options::Age"]
if "Queue-Report::Options::New" in Cnf:
# If we produce html we always have oldest first.
direction.append([6, -1, "ao"])
else:
if "Queue-Report::Options::Sort" in Cnf:
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([6, -1, age])
elif i == "an":
# Age, newest first.
direction.append([6, 1, age])
elif i == "na":
# Name, Ascending.
direction.append([0, 1, 0])
elif i == "nd":
# Name, Descending.
direction.append([0, -1, 0])
elif i == "nl":
# Notes last.
direction.append([5, 1, 0])
elif i == "nf":
# Notes first.
direction.append([5, -1, 0])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if "Queue-Report::Options::822" in Cnf:
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, maint, distribution, closes, fingerprint, sponsor,
changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write((", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " +
str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
if sponsor:
log.write("Sponsored-By: %[email protected]\n" % sponsor)
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#" + bugs + ", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
total_count = len(queue.uploads)
source_count = len(per_source_items)
if "Queue-Report::Options::New" in Cnf:
direction.append([6, 1, "ao"])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, maint, distribution, closes, fingerprint,
sponsor, changedby, _) = entry
table_row(source, version_list, arch_list, last_modified,
maint, distribution, closes, fingerprint, sponsor,
changedby)
table_footer(type.upper())
elif "Queue-Report::Options::822" not in Cnf:
# The "normal" output without any formatting.
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, _, _, _, _, _, _, _) = entry
if processed:
format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (
max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, processed)
else:
format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (
max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, note,
time_pp(last_modified))
if msg:
print(type.upper())
print("-" * len(type))
print()
print(msg)
print((
"%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed."
% (source_count, type, plural(source_count), total_count, type,
plural(total_count), total_pending, type,
plural(total_pending))))
print()
def process_queue(queue, log, rrd_dir):
msg = ""
type = queue.queue_name
session = DBConn().session()
# Divide the .changes into per-source groups
per_source = {}
total_pending = 0
for upload in queue.uploads:
source = upload.changes.source
if source not in per_source:
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["processed"] = ""
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action():
per_source[source]["processed"] = "PENDING %s" % handler.get_action()
total_pending += 1
per_source[source]["list"].append(upload)
per_source[source]["list"].sort(key=lambda x: x.changes.created, reverse=True)
# Determine oldest time and have note status for each source group
for source in per_source.keys():
source_list = per_source[source]["list"]
first = source_list[0]
oldest = time.mktime(first.changes.created.timetuple())
have_note = 0
for d in per_source[source]["list"]:
mtime = time.mktime(d.changes.created.timetuple())
if "Queue-Report::Options::New" in Cnf:
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d.policy_queue, d.changes.source, d.changes.version)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0 # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1 # some
else:
per_source[source]["note_state"] = 2 # all
per_source_items = per_source.items()
per_source_items.sort(key=functools.cmp_to_key(sg_compare))
update_graph_database(rrd_dir, type, len(per_source_items), len(queue.uploads))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
try:
logins = get_logins_from_ldap()
except:
logins = dict()
for i in per_source_items:
maintainer = {}
maint = ""
distribution = ""
closes = ""
fingerprint = ""
changeby = {}
changedby = ""
sponsor = ""
filename = i[1]["list"][0].changes.changesname
last_modified = time.time() - i[1]["oldest"]
source = i[1]["list"][0].changes.source
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0].binaries
binary = ', '.join([b.package for b in binary_list])
arches = set()
versions = set()
for j in i[1]["list"]:
dbc = j.changes
changesbase = dbc.changesname
if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer(dbc.maintainer)
except ParseMaintError as msg:
print("Problems while parsing maintainer address\n")
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint = "%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer(dbc.changedby)
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby = "%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
distribution = dbc.distribution.split()
closes = dbc.closes
fingerprint = dbc.fingerprint
sponsor_name = get_uid_from_fingerprint(fingerprint).name
sponsor_login = get_uid_from_fingerprint(fingerprint).uid
if '@' in sponsor_login:
if fingerprint in logins:
sponsor_login = logins[fingerprint]
if (sponsor_name != maintainer["maintainername"]
and sponsor_name != changeby["changedbyname"]
and sponsor_login + '@debian.org' != maintainer["maintaineremail"]
and sponsor_name != changeby["changedbyemail"]):
sponsor = sponsor_login
for arch in dbc.architecture.split():
arches.add(arch)
versions.add(dbc.version)
arches_list = list(arches)
arches_list.sort(key=utils.ArchKey)
arch_list = " ".join(arches_list)
version_list = " ".join(sorted(versions, reverse=True))
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([source, binary, version_list, arch_list, per_source[source]["processed"], note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if "Queue-Report::Options::Age" in Cnf:
age = Cnf["Queue-Report::Options::Age"]
if "Queue-Report::Options::New" in Cnf:
# If we produce html we always have oldest first.
direction.append([6, -1, "ao"])
else:
if "Queue-Report::Options::Sort" in Cnf:
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([6, -1, age])
elif i == "an":
# Age, newest first.
direction.append([6, 1, age])
elif i == "na":
# Name, Ascending.
direction.append([0, 1, 0])
elif i == "nd":
# Name, Descending.
direction.append([0, -1, 0])
elif i == "nl":
# Notes last.
direction.append([5, 1, 0])
elif i == "nf":
# Notes first.
direction.append([5, -1, 0])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if "Queue-Report::Options::822" in Cnf:
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write((", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
if sponsor:
log.write("Sponsored-By: %[email protected]\n" % sponsor)
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#" + bugs + ", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
total_count = len(queue.uploads)
source_count = len(per_source_items)
if "Queue-Report::Options::New" in Cnf:
direction.append([6, 1, "ao"])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, _) = entry
table_row(source, version_list, arch_list, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby)
table_footer(type.upper())
elif "Queue-Report::Options::822" not in Cnf:
# The "normal" output without any formatting.
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, _, _, _, _, _, _, _) = entry
if processed:
format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, processed)
else:
format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
if msg:
print(type.upper())
print("-" * len(type))
print()
print(msg)
print(("%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed." %
(source_count, type, plural(source_count),
total_count, type, plural(total_count),
total_pending, type, plural(total_pending))))
print()
def assertNotValid(self, input):
self.assertRaises(ParseMaintError, lambda: fix_maintainer(input))
def assertNotValid(self, input):
self.assertRaises(ParseMaintError, lambda: fix_maintainer(input))
def process_changes_files(changes_files, type, log, rrd_dir):
msg = ""
cache = {}
unprocessed = []
# Read in all the .changes files
for filename in changes_files:
try:
u = Upload()
u.load_changes(filename)
cache[filename] = copy(u.pkg.changes)
cache[filename]["filename"] = filename
except Exception as e:
print "WARNING: Exception %s" % e
continue
# Divide the .changes into per-source groups
per_source = {}
for filename in cache.keys():
if not cache[filename].has_key("source"):
unprocessed.append(filename)
continue
source = cache[filename]["source"]
if not per_source.has_key(source):
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["list"].append(cache[filename])
# Determine oldest time and have note status for each source group
for source in per_source.keys():
source_list = per_source[source]["list"]
first = source_list[0]
oldest = os.stat(first["filename"])[stat.ST_MTIME]
have_note = 0
for d in per_source[source]["list"]:
mtime = os.stat(d["filename"])[stat.ST_MTIME]
if Cnf.has_key("Queue-Report::Options::New"):
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d["source"], d["version"])
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1; # some
else:
per_source[source]["note_state"] = 2; # all
per_source_items = per_source.items()
per_source_items.sort(sg_compare)
update_graph_database(rrd_dir, type, len(per_source_items), len(changes_files))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
for i in per_source_items:
maintainer = {}
maint=""
distribution=""
closes=""
fingerprint=""
changeby = {}
changedby=""
sponsor=""
filename=i[1]["list"][0]["filename"]
last_modified = time.time()-i[1]["oldest"]
source = i[1]["list"][0]["source"]
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0]["binary"].keys()
binary = ', '.join(binary_list)
arches = {}
versions = {}
for j in i[1]["list"]:
changesbase = os.path.basename(j["filename"])
try:
session = DBConn().session()
dbc = session.query(DBChange).filter_by(changesname=changesbase).one()
session.close()
except Exception as e:
print "Can't find changes file in NEW for %s (%s)" % (changesbase, e)
dbc = None
if Cnf.has_key("Queue-Report::Options::New") or Cnf.has_key("Queue-Report::Options::822"):
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer (j["maintainer"])
except ParseMaintError as msg:
print "Problems while parsing maintainer address\n"
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer (j["changed-by"])
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
distribution=j["distribution"].keys()
closes=j["closes"].keys()
if dbc:
fingerprint = dbc.fingerprint
sponsor_name = get_uid_from_fingerprint(fingerprint).name
sponsor_email = get_uid_from_fingerprint(fingerprint).uid + "@debian.org"
if sponsor_name != maintainer["maintainername"] and sponsor_name != changeby["changedbyname"] and \
sponsor_email != maintainer["maintaineremail"] and sponsor_name != changeby["changedbyemail"]:
sponsor = sponsor_email
for arch in j["architecture"].keys():
arches[arch] = ""
version = j["version"]
versions[version] = ""
arches_list = arches.keys()
arches_list.sort(utils.arch_compare_sw)
arch_list = " ".join(arches_list)
version_list = " ".join(versions.keys())
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if Cnf.has_key("Queue-Report::Options::Age"):
age = Cnf["Queue-Report::Options::Age"]
if Cnf.has_key("Queue-Report::Options::New"):
# If we produce html we always have oldest first.
direction.append([5,-1,"ao"])
else:
if Cnf.has_key("Queue-Report::Options::Sort"):
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([5,-1,age])
elif i == "an":
# Age, newest first.
direction.append([5,1,age])
elif i == "na":
# Name, Ascending.
direction.append([0,1,0])
elif i == "nd":
# Name, Descending.
direction.append([0,-1,0])
elif i == "nl":
# Notes last.
direction.append([4,1,0])
elif i == "nf":
# Notes first.
direction.append([4,-1,0])
entries.sort(lambda x, y: sortfunc(x, y))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if Cnf.has_key("Queue-Report::Options::822"):
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write( (", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <"+mail+">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <"+mail+">" + "\n")
if sponsor:
log.write("Sponsored-By: " + "@".join(sponsor.split("@")[:2]) + "\n")
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#"+bugs+", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
if Cnf.has_key("Queue-Report::Options::New"):
direction.append([5,1,"ao"])
entries.sort(lambda x, y: sortfunc(x, y))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
total_count = len(changes_files)
source_count = len(per_source_items)
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, undef) = entry
table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby)
table_footer(type.upper())
elif not Cnf.has_key("Queue-Report::Options::822"):
# The "normal" output without any formatting.
format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef, undef) = entry
msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
if msg:
total_count = len(changes_files)
source_count = len(per_source_items)
print type.upper()
print "-"*len(type)
print
print msg
print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count))
print
if len(unprocessed):
print "UNPROCESSED"
print "-----------"
for u in unprocessed:
print u
print
