def do_pkg(changes_file): changes_file = utils.validate_changes_file_arg(changes_file, 0) if not changes_file: return print "\n" + changes_file u = Upload() u.pkg.changes_file = changes_file # We can afoord not to check the signature before loading the changes file # as we've validated it already (otherwise it couldn't be in new) # and we can more quickly skip over already processed files this way u.load_changes(changes_file) origchanges = os.path.abspath(u.pkg.changes_file) # Still be cautious in case paring the changes file went badly if u.pkg.changes.has_key('source') and u.pkg.changes.has_key('version'): htmlname = u.pkg.changes["source"] + "_" + u.pkg.changes["version"] + ".html" htmlfile = os.path.join(cnf["Show-New::HTMLPath"], htmlname) else: # Changes file was bad print "Changes file %s missing source or version field" % changes_file return # Have we already processed this? if os.path.exists(htmlfile) and \ os.stat(htmlfile).st_mtime > os.stat(origchanges).st_mtime: with open(htmlfile, "r") as fd: if fd.read() != timeout_str: sources.append(htmlname) return (PROC_STATUS_SUCCESS, '%s already up-to-date' % htmlfile) # Now we'll load the fingerprint session = DBConn().session() htmlfiles_to_process.append(htmlfile) (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file, session=session) new_queue = get_policy_queue('new', session ); u.pkg.directory = new_queue.path u.update_subst() files = u.pkg.files changes = u.pkg.changes sources.append(htmlname) for deb_filename, f in files.items(): if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"): u.binary_file_checks(deb_filename, session) u.check_binary_against_db(deb_filename, session) else: u.source_file_checks(deb_filename, session) u.check_source_against_db(deb_filename, session) u.pkg.changes["suite"] = u.pkg.changes["distribution"] new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, dsc=u.pkg.dsc, session=session) outfile = open(os.path.join(cnf["Show-New::HTMLPath"],htmlname),"w") filestoexamine = [] for pkg in new.keys(): for fn in new[pkg]["files"]: filestoexamine.append(fn) print >> outfile, html_header(changes["source"], filestoexamine) check_valid(new, session) distribution = changes["distribution"].keys()[0] print >> outfile, examine_package.display_changes(distribution, changes_file) for fn in filter(lambda fn: fn.endswith(".dsc"), filestoexamine): print >> outfile, examine_package.check_dsc(distribution, fn, session) for fn in filter(lambda fn: fn.endswith(".deb") or fn.endswith(".udeb"), filestoexamine): print >> outfile, examine_package.check_deb(distribution, fn, session) print >> outfile, html_footer() outfile.close() session.close() htmlfiles_to_process.remove(htmlfile) return (PROC_STATUS_SUCCESS, '%s already updated' % htmlfile)
sys.path.append('/srv/dak/dak') from daklib.dbconn import * from daklib import utils from daklib.queue import Upload i = 0 t = 0 pattern = '*.changes' changes_dir = '/srv/dak/done' def find_changes(pattern, root): for path, dirs, files in os.walk(os.path.abspath(root)): for filename in fnmatch.filter(files, pattern): yield os.path.join(path, filename) for changes_file in find_changes(pattern, changes_dir): t = t + 1 for changes_file in find_changes(pattern, changes_dir): u = Upload() u.pkg.changes_file = changes_file (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file) if u.load_changes(changes_file): try: u.store_changelog() except: print 'Unable to handle %s' % changes_file else: print u.rejects i = i + 1 sys.stdout.write('%d out of %d processed\r' % (i, t))
from daklib.queue import Upload i = 0 t = 0 pattern = '*.changes' changes_dir = '/srv/ftp.debian.org/queue/done' def find_changes(pattern, root): for path, dirs, files in os.walk(os.path.abspath(root)): for filename in fnmatch.filter(files, pattern): yield os.path.join(path, filename) for changes_file in find_changes(pattern, changes_dir): t = t + 1 for changes_file in find_changes(pattern, changes_dir): u = Upload() u.pkg.changes_file = changes_file (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file) if u.load_changes(changes_file): try: u.store_changelog() except: print 'Unable to handle %s' % changes_file else: print u.rejects i = i + 1 sys.stdout.write('%d out of %d processed\r' % (i, t))
def process_changes_files(changes_files, type, log, rrd_dir): msg = "" cache = {} unprocessed = [] # Read in all the .changes files for filename in changes_files: try: u = Upload() u.load_changes(filename) cache[filename] = copy(u.pkg.changes) cache[filename]["filename"] = filename except Exception as e: print "WARNING: Exception %s" % e continue # Divide the .changes into per-source groups per_source = {} for filename in cache.keys(): if not cache[filename].has_key("source"): unprocessed.append(filename) continue source = cache[filename]["source"] if not per_source.has_key(source): per_source[source] = {} per_source[source]["list"] = [] per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): source_list = per_source[source]["list"] first = source_list[0] oldest = os.stat(first["filename"])[stat.ST_MTIME] have_note = 0 for d in per_source[source]["list"]: mtime = os.stat(d["filename"])[stat.ST_MTIME] if Cnf.has_key("Queue-Report::Options::New"): if mtime > oldest: oldest = mtime else: if mtime < oldest: oldest = mtime have_note += has_new_comment(d["source"], d["version"]) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none elif have_note < len(source_list): per_source[source]["note_state"] = 1; # some else: per_source[source]["note_state"] = 2; # all per_source_items = per_source.items() per_source_items.sort(sg_compare) update_graph_database(rrd_dir, type, len(per_source_items), len(changes_files)) entries = [] max_source_len = 0 max_version_len = 0 max_arch_len = 0 for i in per_source_items: maintainer = {} maint="" distribution="" closes="" fingerprint="" changeby = {} changedby="" sponsor="" filename=i[1]["list"][0]["filename"] last_modified = time.time()-i[1]["oldest"] source = i[1]["list"][0]["source"] if len(source) > max_source_len: max_source_len = len(source) binary_list = i[1]["list"][0]["binary"].keys() binary = ', '.join(binary_list) arches = {} versions = {} for j in i[1]["list"]: changesbase = os.path.basename(j["filename"]) try: session = DBConn().session() dbc = session.query(DBChange).filter_by(changesname=changesbase).one() session.close() except Exception as e: print "Can't find changes file in NEW for %s (%s)" % (changesbase, e) dbc = None if Cnf.has_key("Queue-Report::Options::New") or Cnf.has_key("Queue-Report::Options::822"): try: (maintainer["maintainer822"], maintainer["maintainer2047"], maintainer["maintainername"], maintainer["maintaineremail"]) = \ fix_maintainer (j["maintainer"]) except ParseMaintError as msg: print "Problems while parsing maintainer address\n" maintainer["maintainername"] = "Unknown" maintainer["maintaineremail"] = "Unknown" maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"]) # ...likewise for the Changed-By: field if it exists. try: (changeby["changedby822"], changeby["changedby2047"], changeby["changedbyname"], changeby["changedbyemail"]) = \ fix_maintainer (j["changed-by"]) except ParseMaintError as msg: (changeby["changedby822"], changeby["changedby2047"], changeby["changedbyname"], changeby["changedbyemail"]) = \ ("", "", "", "") changedby="%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"]) distribution=j["distribution"].keys() closes=j["closes"].keys() if dbc: fingerprint = dbc.fingerprint sponsor_name = get_uid_from_fingerprint(fingerprint).name sponsor_email = get_uid_from_fingerprint(fingerprint).uid + "@debian.org" if sponsor_name != maintainer["maintainername"] and sponsor_name != changeby["changedbyname"] and \ sponsor_email != maintainer["maintaineremail"] and sponsor_name != changeby["changedbyemail"]: sponsor = sponsor_email for arch in j["architecture"].keys(): arches[arch] = "" version = j["version"] versions[version] = "" arches_list = arches.keys() arches_list.sort(utils.arch_compare_sw) arch_list = " ".join(arches_list) version_list = " ".join(versions.keys()) if len(version_list) > max_version_len: max_version_len = len(version_list) if len(arch_list) > max_arch_len: max_arch_len = len(arch_list) if i[1]["note_state"]: note = " | [N]" else: note = "" entries.append([source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename]) # direction entry consists of "Which field, which direction, time-consider" where # time-consider says how we should treat last_modified. Thats all. # Look for the options for sort and then do the sort. age = "h" if Cnf.has_key("Queue-Report::Options::Age"): age = Cnf["Queue-Report::Options::Age"] if Cnf.has_key("Queue-Report::Options::New"): # If we produce html we always have oldest first. direction.append([5,-1,"ao"]) else: if Cnf.has_key("Queue-Report::Options::Sort"): for i in Cnf["Queue-Report::Options::Sort"].split(","): if i == "ao": # Age, oldest first. direction.append([5,-1,age]) elif i == "an": # Age, newest first. direction.append([5,1,age]) elif i == "na": # Name, Ascending. direction.append([0,1,0]) elif i == "nd": # Name, Descending. direction.append([0,-1,0]) elif i == "nl": # Notes last. direction.append([4,1,0]) elif i == "nf": # Notes first. direction.append([4,-1,0]) entries.sort(lambda x, y: sortfunc(x, y)) # Yes, in theory you can add several sort options at the commandline with. But my mind is to small # at the moment to come up with a real good sorting function that considers all the sidesteps you # have with it. (If you combine options it will simply take the last one at the moment). # Will be enhanced in the future. if Cnf.has_key("Queue-Report::Options::822"): # print stuff out in 822 format for entry in entries: (source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry # We'll always have Source, Version, Arch, Mantainer, and Dist # For the rest, check to see if we have them, then print them out log.write("Source: " + source + "\n") log.write("Binary: " + binary + "\n") log.write("Version: " + version_list + "\n") log.write("Architectures: ") log.write( (", ".join(arch_list.split(" "))) + "\n") log.write("Age: " + time_pp(last_modified) + "\n") log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n") log.write("Queue: " + type + "\n") (name, mail) = maint.split(":", 1) log.write("Maintainer: " + name + " <"+mail+">" + "\n") if changedby: (name, mail) = changedby.split(":", 1) log.write("Changed-By: " + name + " <"+mail+">" + "\n") if sponsor: log.write("Sponsored-By: " + "@".join(sponsor.split("@")[:2]) + "\n") log.write("Distribution:") for dist in distribution: log.write(" " + dist) log.write("\n") log.write("Fingerprint: " + fingerprint + "\n") if closes: bug_string = "" for bugs in closes: bug_string += "#"+bugs+", " log.write("Closes: " + bug_string[:-2] + "\n") log.write("Changes-File: " + os.path.basename(changes_file) + "\n") log.write("\n") if Cnf.has_key("Queue-Report::Options::New"): direction.append([5,1,"ao"]) entries.sort(lambda x, y: sortfunc(x, y)) # Output for a html file. First table header. then table_footer. # Any line between them is then a <tr> printed from subroutine table_row. if len(entries) > 0: total_count = len(changes_files) source_count = len(per_source_items) table_header(type.upper(), source_count, total_count) for entry in entries: (source, binary, version_list, arch_list, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, undef) = entry table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes, fingerprint, sponsor, changedby) table_footer(type.upper()) elif not Cnf.has_key("Queue-Report::Options::822"): # The "normal" output without any formatting. format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len) msg = "" for entry in entries: (source, binary, version_list, arch_list, note, last_modified, undef, undef, undef, undef, undef, undef, undef) = entry msg += format % (source, version_list, arch_list, note, time_pp(last_modified)) if msg: total_count = len(changes_files) source_count = len(per_source_items) print type.upper() print "-"*len(type) print print msg print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)) print if len(unprocessed): print "UNPROCESSED" print "-----------" for u in unprocessed: print u print
def __init__(self, session=None): cnf = Config() try: newq = get_policy_queue('new', session) for changes_fn in glob.glob(newq.path + "/*.changes"): changes_bn = os.path.basename(changes_fn) chg = get_dbchange(changes_bn, session) u = Upload() success = u.load_changes(changes_fn) u.pkg.changes_file = changes_bn u.check_hashes() if not chg: chg = u.pkg.add_known_changes(newq.path, newq.policy_queue_id, session) session.add(chg) if not success: log.critical("failed to load %s" % changes_fn) sys.exit(1) else: log.critical("ACCLAIM: %s" % changes_fn) files=[] for chg_fn in u.pkg.files.keys(): try: f = open(os.path.join(newq.path, chg_fn)) cpf = ChangePendingFile() cpf.filename = chg_fn cpf.size = u.pkg.files[chg_fn]['size'] cpf.md5sum = u.pkg.files[chg_fn]['md5sum'] if u.pkg.files[chg_fn].has_key('sha1sum'): cpf.sha1sum = u.pkg.files[chg_fn]['sha1sum'] else: log.warning("Having to generate sha1sum for %s" % chg_fn) f.seek(0) cpf.sha1sum = apt_pkg.sha1sum(f) if u.pkg.files[chg_fn].has_key('sha256sum'): cpf.sha256sum = u.pkg.files[chg_fn]['sha256sum'] else: log.warning("Having to generate sha256sum for %s" % chg_fn) f.seek(0) cpf.sha256sum = apt_pkg.sha256sum(f) session.add(cpf) files.append(cpf) f.close() except IOError: # Can't find the file, try to look it up in the pool poolname = poolify(u.pkg.changes["source"], u.pkg.files[chg_fn]["component"]) l = get_location(cnf["Dir::Pool"], u.pkg.files[chg_fn]["component"], session=session) if not l: log.critical("ERROR: Can't find location for %s (component %s)" % (chg_fn, u.pkg.files[chg_fn]["component"])) found, poolfile = check_poolfile(os.path.join(poolname, chg_fn), u.pkg.files[chg_fn]['size'], u.pkg.files[chg_fn]["md5sum"], l.location_id, session=session) if found is None: log.critical("ERROR: Found multiple files for %s in pool" % chg_fn) sys.exit(1) elif found is False and poolfile is not None: log.critical("ERROR: md5sum / size mismatch for %s in pool" % chg_fn) sys.exit(1) else: if poolfile is None: log.critical("ERROR: Could not find %s in pool" % chg_fn) sys.exit(1) else: chg.poolfiles.append(poolfile) chg.files = files session.commit() except KeyboardInterrupt: print("Caught C-c; terminating.") utils.warn("Caught C-c; terminating.") self.plsDie()