def check_pkg(upload, upload_copy, session):
missing = []
changes = os.path.join(upload_copy.directory, upload.changes.changesname)
suite_name = upload.target_suite.suite_name
handler = PolicyQueueUploadHandler(upload, session)
missing = [(m['type'], m["package"])
for m in handler.missing_overrides(hints=missing)]
less_cmd = ("less", "-r", "-")
less_process = subprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE)
try:
less_fd = less_process.stdin
less_fd.write(
six.ensure_binary(
dak.examine_package.display_changes(suite_name, changes)))
source = upload.source
if source is not None:
source_file = os.path.join(
upload_copy.directory,
os.path.basename(source.poolfile.filename))
less_fd.write(
six.ensure_binary(
dak.examine_package.check_dsc(suite_name, source_file)))
for binary in upload.binaries:
binary_file = os.path.join(
upload_copy.directory,
os.path.basename(binary.poolfile.filename))
examined = dak.examine_package.check_deb(suite_name, binary_file)
# We always need to call check_deb to display package relations for every binary,
# but we print its output only if new overrides are being added.
if ("deb", binary.package) in missing:
less_fd.write(six.ensure_binary(examined))
less_fd.write(
six.ensure_binary(dak.examine_package.output_package_relations()))
less_process.stdin.close()
except IOError as e:
if e.errno == errno.EPIPE:
utils.warn("[examine_package] Caught EPIPE; skipping.")
else:
raise
except KeyboardInterrupt:
utils.warn("[examine_package] Caught C-c; skipping.")
finally:
less_process.communicate()
def do_pkg(upload, session):
# Try to get an included dsc
dsc = upload.source
cnf = Config()
group = cnf.get('Dinstall::UnprivGroup') or None
try:
with lock_package(upload.changes.source), \
UploadCopy(upload, group=group) as upload_copy:
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action() is not None:
print("PENDING %s\n" % handler.get_action())
return
do_new(upload, upload_copy, handler, session)
except AlreadyLockedError as e:
print("Seems to be locked by %s already, skipping..." % (e))
def check_pkg (upload, upload_copy, session):
missing = []
save_stdout = sys.stdout
changes = os.path.join(upload_copy.directory, upload.changes.changesname)
suite_name = upload.target_suite.suite_name
handler = PolicyQueueUploadHandler(upload, session)
missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)]
less_cmd = ("less", "-R", "-")
less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE)
try:
sys.stdout = less_process.stdin
print examine_package.display_changes(suite_name, changes)
source = upload.source
if source is not None:
source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename))
print examine_package.check_dsc(suite_name, source_file)
for binary in upload.binaries:
binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
examined = examine_package.check_deb(suite_name, binary_file)
# We always need to call check_deb to display package relations for every binary,
# but we print its output only if new overrides are being added.
if ("deb", binary.package) in missing:
print examined
print examine_package.output_package_relations()
less_process.stdin.close()
except IOError as e:
if e.errno == errno.EPIPE:
utils.warn("[examine_package] Caught EPIPE; skipping.")
else:
raise
except KeyboardInterrupt:
utils.warn("[examine_package] Caught C-c; skipping.")
finally:
less_process.wait()
sys.stdout = save_stdout
def do_pkg(upload, session):
# Try to get an included dsc
dsc = upload.source
cnf = Config()
group = cnf.get('Dinstall::UnprivGroup') or None
#bcc = "X-DAK: dak process-new"
#if cnf.has_key("Dinstall::Bcc"):
# u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
#else:
# u.Subst["__BCC__"] = bcc
try:
with lock_package(upload.changes.source):
with UploadCopy(upload, group=group) as upload_copy:
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action() is not None:
print "PENDING %s\n" % handler.get_action()
return
do_new(upload, upload_copy, handler, session)
except AlreadyLockedError as e:
print "Seems to be locked by %s already, skipping..." % (e)
def do_pkg(upload, session):
# Try to get an included dsc
dsc = upload.source
cnf = Config()
group = cnf.get('Dinstall::UnprivGroup') or None
#bcc = "X-DAK: dak process-new"
#if cnf.has_key("Dinstall::Bcc"):
# u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
#else:
# u.Subst["__BCC__"] = bcc
try:
with lock_package(upload.changes.source):
with UploadCopy(upload, group=group) as upload_copy:
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action() is not None:
print "PENDING %s\n" % handler.get_action()
return
do_new(upload, upload_copy, handler, session)
except AlreadyLockedError as e:
print "Seems to be locked by %s already, skipping..." % (e)
def process_queue(queue, log, rrd_dir):
msg = ""
type = queue.queue_name
session = DBConn().session()
# Divide the .changes into per-source groups
per_source = {}
total_pending = 0
for upload in queue.uploads:
source = upload.changes.source
if source not in per_source:
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["processed"] = ""
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action():
per_source[source][
"processed"] = "PENDING %s" % handler.get_action()
total_pending += 1
per_source[source]["list"].append(upload)
per_source[source]["list"].sort(key=lambda x: x.changes.created,
reverse=True)
# Determine oldest time and have note status for each source group
for source in per_source.keys():
source_list = per_source[source]["list"]
first = source_list[0]
oldest = time.mktime(first.changes.created.timetuple())
have_note = 0
for d in per_source[source]["list"]:
mtime = time.mktime(d.changes.created.timetuple())
if "Queue-Report::Options::New" in Cnf:
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d.policy_queue, d.changes.source,
d.changes.version)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0 # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1 # some
else:
per_source[source]["note_state"] = 2 # all
per_source_items = per_source.items()
per_source_items.sort(key=functools.cmp_to_key(sg_compare))
update_graph_database(rrd_dir, type, len(per_source_items),
len(queue.uploads))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
try:
logins = get_logins_from_ldap()
except:
logins = dict()
for i in per_source_items:
maintainer = {}
maint = ""
distribution = ""
closes = ""
fingerprint = ""
changeby = {}
changedby = ""
sponsor = ""
filename = i[1]["list"][0].changes.changesname
last_modified = time.time() - i[1]["oldest"]
source = i[1]["list"][0].changes.source
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0].binaries
binary = ', '.join([b.package for b in binary_list])
arches = set()
versions = set()
for j in i[1]["list"]:
dbc = j.changes
changesbase = dbc.changesname
if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer(dbc.maintainer)
except ParseMaintError as msg:
print("Problems while parsing maintainer address\n")
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint = "%s:%s" % (maintainer["maintainername"],
maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer(dbc.changedby)
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby = "%s:%s" % (changeby["changedbyname"],
changeby["changedbyemail"])
distribution = dbc.distribution.split()
closes = dbc.closes
fingerprint = dbc.fingerprint
sponsor_name = get_uid_from_fingerprint(fingerprint).name
sponsor_login = get_uid_from_fingerprint(fingerprint).uid
if '@' in sponsor_login:
if fingerprint in logins:
sponsor_login = logins[fingerprint]
if (sponsor_name != maintainer["maintainername"]
and sponsor_name != changeby["changedbyname"]
and sponsor_login + '@debian.org' !=
maintainer["maintaineremail"]
and sponsor_name != changeby["changedbyemail"]):
sponsor = sponsor_login
for arch in dbc.architecture.split():
arches.add(arch)
versions.add(dbc.version)
arches_list = sorted(arches, key=utils.ArchKey)
arch_list = " ".join(arches_list)
version_list = " ".join(sorted(versions, reverse=True))
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([
source, binary, version_list, arch_list,
per_source[source]["processed"], note, last_modified, maint,
distribution, closes, fingerprint, sponsor, changedby, filename
])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if "Queue-Report::Options::Age" in Cnf:
age = Cnf["Queue-Report::Options::Age"]
if "Queue-Report::Options::New" in Cnf:
# If we produce html we always have oldest first.
direction.append([6, -1, "ao"])
else:
if "Queue-Report::Options::Sort" in Cnf:
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([6, -1, age])
elif i == "an":
# Age, newest first.
direction.append([6, 1, age])
elif i == "na":
# Name, Ascending.
direction.append([0, 1, 0])
elif i == "nd":
# Name, Descending.
direction.append([0, -1, 0])
elif i == "nl":
# Notes last.
direction.append([5, 1, 0])
elif i == "nf":
# Notes first.
direction.append([5, -1, 0])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if "Queue-Report::Options::822" in Cnf:
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, maint, distribution, closes, fingerprint, sponsor,
changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write((", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " +
str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
if sponsor:
log.write("Sponsored-By: %[email protected]\n" % sponsor)
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#" + bugs + ", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
total_count = len(queue.uploads)
source_count = len(per_source_items)
if "Queue-Report::Options::New" in Cnf:
direction.append([6, 1, "ao"])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, maint, distribution, closes, fingerprint,
sponsor, changedby, _) = entry
table_row(source, version_list, arch_list, last_modified,
maint, distribution, closes, fingerprint, sponsor,
changedby)
table_footer(type.upper())
elif "Queue-Report::Options::822" not in Cnf:
# The "normal" output without any formatting.
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, _, _, _, _, _, _, _) = entry
if processed:
format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (
max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, processed)
else:
format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (
max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, note,
time_pp(last_modified))
if msg:
print(type.upper())
print("-" * len(type))
print()
print(msg)
print((
"%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed."
% (source_count, type, plural(source_count), total_count, type,
plural(total_count), total_pending, type,
plural(total_pending))))
print()
def process_queue(queue, log, rrd_dir):
msg = ""
type = queue.queue_name
session = DBConn().session()
# Divide the .changes into per-source groups
per_source = {}
total_pending = 0
for upload in queue.uploads:
source = upload.changes.source
if source not in per_source:
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["processed"] = ""
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action():
per_source[source]["processed"] = "PENDING %s" % handler.get_action()
total_pending += 1
per_source[source]["list"].append(upload)
per_source[source]["list"].sort(key=lambda x: x.changes.created, reverse=True)
# Determine oldest time and have note status for each source group
for source in per_source.keys():
source_list = per_source[source]["list"]
first = source_list[0]
oldest = time.mktime(first.changes.created.timetuple())
have_note = 0
for d in per_source[source]["list"]:
mtime = time.mktime(d.changes.created.timetuple())
if "Queue-Report::Options::New" in Cnf:
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d.policy_queue, d.changes.source, d.changes.version)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0 # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1 # some
else:
per_source[source]["note_state"] = 2 # all
per_source_items = per_source.items()
per_source_items.sort(key=functools.cmp_to_key(sg_compare))
update_graph_database(rrd_dir, type, len(per_source_items), len(queue.uploads))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
try:
logins = get_logins_from_ldap()
except:
logins = dict()
for i in per_source_items:
maintainer = {}
maint = ""
distribution = ""
closes = ""
fingerprint = ""
changeby = {}
changedby = ""
sponsor = ""
filename = i[1]["list"][0].changes.changesname
last_modified = time.time() - i[1]["oldest"]
source = i[1]["list"][0].changes.source
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0].binaries
binary = ', '.join([b.package for b in binary_list])
arches = set()
versions = set()
for j in i[1]["list"]:
dbc = j.changes
changesbase = dbc.changesname
if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer(dbc.maintainer)
except ParseMaintError as msg:
print("Problems while parsing maintainer address\n")
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint = "%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer(dbc.changedby)
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby = "%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
distribution = dbc.distribution.split()
closes = dbc.closes
fingerprint = dbc.fingerprint
sponsor_name = get_uid_from_fingerprint(fingerprint).name
sponsor_login = get_uid_from_fingerprint(fingerprint).uid
if '@' in sponsor_login:
if fingerprint in logins:
sponsor_login = logins[fingerprint]
if (sponsor_name != maintainer["maintainername"]
and sponsor_name != changeby["changedbyname"]
and sponsor_login + '@debian.org' != maintainer["maintaineremail"]
and sponsor_name != changeby["changedbyemail"]):
sponsor = sponsor_login
for arch in dbc.architecture.split():
arches.add(arch)
versions.add(dbc.version)
arches_list = list(arches)
arches_list.sort(key=utils.ArchKey)
arch_list = " ".join(arches_list)
version_list = " ".join(sorted(versions, reverse=True))
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([source, binary, version_list, arch_list, per_source[source]["processed"], note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if "Queue-Report::Options::Age" in Cnf:
age = Cnf["Queue-Report::Options::Age"]
if "Queue-Report::Options::New" in Cnf:
# If we produce html we always have oldest first.
direction.append([6, -1, "ao"])
else:
if "Queue-Report::Options::Sort" in Cnf:
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([6, -1, age])
elif i == "an":
# Age, newest first.
direction.append([6, 1, age])
elif i == "na":
# Name, Ascending.
direction.append([0, 1, 0])
elif i == "nd":
# Name, Descending.
direction.append([0, -1, 0])
elif i == "nl":
# Notes last.
direction.append([5, 1, 0])
elif i == "nf":
# Notes first.
direction.append([5, -1, 0])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if "Queue-Report::Options::822" in Cnf:
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write((", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
if sponsor:
log.write("Sponsored-By: %[email protected]\n" % sponsor)
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#" + bugs + ", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
total_count = len(queue.uploads)
source_count = len(per_source_items)
if "Queue-Report::Options::New" in Cnf:
direction.append([6, 1, "ao"])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, _) = entry
table_row(source, version_list, arch_list, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby)
table_footer(type.upper())
elif "Queue-Report::Options::822" not in Cnf:
# The "normal" output without any formatting.
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, _, _, _, _, _, _, _) = entry
if processed:
format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, processed)
else:
format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
if msg:
print(type.upper())
print("-" * len(type))
print()
print(msg)
print(("%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed." %
(source_count, type, plural(source_count),
total_count, type, plural(total_count),
total_pending, type, plural(total_pending))))
print()
