def setup_cache(): cache_dir = mkdtemp(prefix="wnpp-by-tags") bugs_dir = "%s/wnpp/" % cache_dir os.makedirs(bugs_dir) os.makedirs("%s/popcon" % cache_dir) create_file("%s/orphaned.html" % bugs_dir, orphaned_raw_data) create_file("%s/rfa_bypackage.html" % bugs_dir, rfa_raw_data) create_file("%s/popcon/all-popcon-results.txt" % cache_dir, popcon_raw_data) return cache_dir
def main(): if len(sys.argv) < 2 or '-h' in sys.argv or '--help' in sys.argv: syntax() src_dir = sys.argv[1] dst_dir = sys.argv[2] try: if not os.path.isfile(sys.argv[3]): giveup("\"%s\" does not exist or is not a file" % sys.argv[3]) index_page_msg = open(sys.argv[3]).read() except IndexError: index_page_msg = "" if not os.path.isdir(src_dir): giveup("\"%s\" does not exist or is not a directory" % src_dir) facets = [] timestamp = "%d-%02d-%02d %02d:%02d" % time.gmtime()[:5] for src_facet_dir in glob("%s/*" % src_dir): if not os.path.isdir(src_facet_dir): continue facet = src_facet_dir.split("/")[-1] facet_values = [] facets.append(facet) for src_tag_file in glob("%s/*" % src_facet_dir): facet_value = os.path.basename(src_tag_file) dst_facet_dir = "%s/%s" % (dst_dir, facet) ensure_dir_exists(dst_facet_dir) bug_table_rows = [] for line in open(src_tag_file).readlines(): # the columns are: bug type, bug number, package, popcon, dust cols = line.rstrip().split(" ") _, bug_no, package_name, popcon = cols[:4] if len(cols) != 5: sys.stderr.write("skipping invalid line \"%s\"" % line) continue # add link for bug number and package cols[1] = '<a href="http://bugs.debian.org/%s">%s</a>' \ % (bug_no, bug_no) cols[2] = '<a href="http://packages.qa.debian.org/%s/%s.html">%s</a>' \ % (package_name[0], package_name, package_name) cols[3] = '<a href="http://qa.debian.org/popcon.php?package=%s">%s</a>' \ % (package_name, popcon) cols = "</td><td>".join(cols) bug_table_rows.append("<tr><td>%s</td></tr>" % cols) nbugs = len(bug_table_rows) if nbugs > 0: row = "<tr><td><a href=\"./%s.html\">%s</a></td><td>%d</td></tr>" \ % (facet_value, facet_value, nbugs) else: row = "<tr><td>%s</td><td>%d</td></tr>" % (facet_value, nbugs) facet_values.append(row) dst_tag_file = "%s/%s.html" % (dst_facet_dir, facet_value) # FIXME: use newstyle dicts instead tag = "%s::%s" % (facet, facet_value) html_doc = bugs_html_template % (tag, tag, "\n".join(bug_table_rows), facet, timestamp) create_file(dst_tag_file, html_doc) content = facet_values_html_template % \ (facet, facet, facet, "\n".join(sorted(facet_values)), timestamp) create_file("%s/index.html" % dst_facet_dir, content) formated_facets = ["<li><a href=\"./%s/index.html\">%s</a></li>" % (f, f) for f in facets] create_file("%s/index.html" % dst_dir, main_page_template % ("\n".join(sorted(formated_facets)), index_page_msg, timestamp))
def main(): # parse user-supplied arguments args = Arguments() # sanity checks if not os.path.exists(args.debtags_file): giveup("The debtags file %s doesn't exist" % args.debtags_file) if not os.path.exists(args.tags_file): giveup("The tag vocabulary file %s doesn't exist" % args.tags_file) if args.match_tags or args.list_tags or args.batch_queries: vocabulary = TagVocabulary(args.tags_file) if args.list_tags: print vocabulary exit(0) invalid_tags = vocabulary.invalid_tags(args.match_tags.union(args.excl_tags)) if invalid_tags: giveup("The following tags are not listed in %s:\n%s" % \ (args.tags_file, "\n".join(list(invalid_tags)))) # misc initialisations bugs_dir = "%s/wnpp" % args.cache_dir popcon_dir = "%s/popcon" % args.cache_dir ensure_dir_exists(bugs_dir) ensure_dir_exists(popcon_dir) update_bug_data(args.force_update, bugs_dir, args.full_name_bug_types, args.conf.bugs_update_period_in_days, args.verbose) update_popcon_data(args.force_update, popcon_dir, args.conf.popcon_update_period_in_days, args.verbose) debtags = Debtags(open(args.debtags_file)) popcon_file = "%s/%s" % (popcon_dir, POPCON_FNAME) assert os.path.isfile(popcon_file) popcon = Popcon(open(popcon_file, "r")) Package.init_sources(debtags.tags_of_pkg, popcon.inst_of_pkg) # build dict of package objects, indexed by package name, using the HTML # BTS pages pkgs_by_name = {} for bug_page in glob("%s/*.html" % bugs_dir): bug_type = BugType.abbreviation_of(os.path.basename(bug_page)[:-5]) if bug_type in args.bug_types: pkgs_by_name = extract_bugs(open(bug_page, "r"), pkgs_by_name, bug_type) if args.show_untagged: # select only packages without tags pkg_objs = [p for p in pkgs_by_name.itervalues() if not p.tags] print format_matches(pkg_objs, args) exit(0) if args.verbose: nbugs = sum([len(p.bugs) for p in pkgs_by_name.itervalues()]) print "loaded %d bugs in %s packages" % (nbugs, len(pkgs_by_name)) # filter packages using user-supplied tags tag_db = StringIO("\n".join([str(p) for p in pkgs_by_name.itervalues()])) if not args.batch_queries: pkg_objs = gen_matches(tag_db, pkgs_by_name, args) print format_matches(pkg_objs, args) exit(0) # else we're in batch mode for facet in args.batch_queries: if args.verbose: print "processing \"%s\" tags" % facet facet_dir = "%s/%s" % (args.batch_dir, facet) ensure_dir_exists(facet_dir) for tag in vocabulary.tags_of_facet(facet): args.match_tags = set([tag]) pkg_objs = gen_matches(tag_db, pkgs_by_name, args) try: tag_value = tag.split("::")[1] except ValueError: warn("skipping invalid tag \"%s\"" % tag) continue filename = "%s/%s" % (facet_dir, tag_value) matches = format_matches(pkg_objs, args) create_file(filename, matches) tag_db.seek(0)