def setup_cache():
cache_dir = mkdtemp(prefix="wnpp-by-tags")
bugs_dir = "%s/wnpp/" % cache_dir
os.makedirs(bugs_dir)
os.makedirs("%s/popcon" % cache_dir)
create_file("%s/orphaned.html" % bugs_dir, orphaned_raw_data)
create_file("%s/rfa_bypackage.html" % bugs_dir, rfa_raw_data)
create_file("%s/popcon/all-popcon-results.txt" % cache_dir, popcon_raw_data)
return cache_dir
def main():
if len(sys.argv) < 2 or '-h' in sys.argv or '--help' in sys.argv:
syntax()
src_dir = sys.argv[1]
dst_dir = sys.argv[2]
try:
if not os.path.isfile(sys.argv[3]):
giveup("\"%s\" does not exist or is not a file" % sys.argv[3])
index_page_msg = open(sys.argv[3]).read()
except IndexError:
index_page_msg = ""
if not os.path.isdir(src_dir):
giveup("\"%s\" does not exist or is not a directory" % src_dir)
facets = []
timestamp = "%d-%02d-%02d %02d:%02d" % time.gmtime()[:5]
for src_facet_dir in glob("%s/*" % src_dir):
if not os.path.isdir(src_facet_dir):
continue
facet = src_facet_dir.split("/")[-1]
facet_values = []
facets.append(facet)
for src_tag_file in glob("%s/*" % src_facet_dir):
facet_value = os.path.basename(src_tag_file)
dst_facet_dir = "%s/%s" % (dst_dir, facet)
ensure_dir_exists(dst_facet_dir)
bug_table_rows = []
for line in open(src_tag_file).readlines():
# the columns are: bug type, bug number, package, popcon, dust
cols = line.rstrip().split(" ")
_, bug_no, package_name, popcon = cols[:4]
if len(cols) != 5:
sys.stderr.write("skipping invalid line \"%s\"" % line)
continue
# add link for bug number and package
cols[1] = '<a href="http://bugs.debian.org/%s">%s</a>' \
% (bug_no, bug_no)
cols[2] = '<a href="http://packages.qa.debian.org/%s/%s.html">%s</a>' \
% (package_name[0], package_name, package_name)
cols[3] = '<a href="http://qa.debian.org/popcon.php?package=%s">%s</a>' \
% (package_name, popcon)
cols = "</td><td>".join(cols)
bug_table_rows.append("<tr><td>%s</td></tr>" % cols)
nbugs = len(bug_table_rows)
if nbugs > 0: row = "<tr><td><a href=\"./%s.html\">%s</a></td><td>%d</td></tr>" \
% (facet_value, facet_value, nbugs)
else:
row = "<tr><td>%s</td><td>%d</td></tr>" % (facet_value, nbugs)
facet_values.append(row)
dst_tag_file = "%s/%s.html" % (dst_facet_dir, facet_value)
# FIXME: use newstyle dicts instead
tag = "%s::%s" % (facet, facet_value)
html_doc = bugs_html_template % (tag, tag, "\n".join(bug_table_rows),
facet, timestamp)
create_file(dst_tag_file, html_doc)
content = facet_values_html_template % \
(facet, facet, facet, "\n".join(sorted(facet_values)), timestamp)
create_file("%s/index.html" % dst_facet_dir, content)
formated_facets = ["<li><a href=\"./%s/index.html\">%s</a></li>" % (f, f)
for f in facets]
create_file("%s/index.html" % dst_dir,
main_page_template % ("\n".join(sorted(formated_facets)),
index_page_msg, timestamp))
def main():
# parse user-supplied arguments
args = Arguments()
# sanity checks
if not os.path.exists(args.debtags_file):
giveup("The debtags file %s doesn't exist" % args.debtags_file)
if not os.path.exists(args.tags_file):
giveup("The tag vocabulary file %s doesn't exist" % args.tags_file)
if args.match_tags or args.list_tags or args.batch_queries:
vocabulary = TagVocabulary(args.tags_file)
if args.list_tags:
print vocabulary
exit(0)
invalid_tags = vocabulary.invalid_tags(args.match_tags.union(args.excl_tags))
if invalid_tags:
giveup("The following tags are not listed in %s:\n%s" % \
(args.tags_file, "\n".join(list(invalid_tags))))
# misc initialisations
bugs_dir = "%s/wnpp" % args.cache_dir
popcon_dir = "%s/popcon" % args.cache_dir
ensure_dir_exists(bugs_dir)
ensure_dir_exists(popcon_dir)
update_bug_data(args.force_update, bugs_dir, args.full_name_bug_types,
args.conf.bugs_update_period_in_days, args.verbose)
update_popcon_data(args.force_update, popcon_dir,
args.conf.popcon_update_period_in_days, args.verbose)
debtags = Debtags(open(args.debtags_file))
popcon_file = "%s/%s" % (popcon_dir, POPCON_FNAME)
assert os.path.isfile(popcon_file)
popcon = Popcon(open(popcon_file, "r"))
Package.init_sources(debtags.tags_of_pkg, popcon.inst_of_pkg)
# build dict of package objects, indexed by package name, using the HTML
# BTS pages
pkgs_by_name = {}
for bug_page in glob("%s/*.html" % bugs_dir):
bug_type = BugType.abbreviation_of(os.path.basename(bug_page)[:-5])
if bug_type in args.bug_types:
pkgs_by_name = extract_bugs(open(bug_page, "r"), pkgs_by_name, bug_type)
if args.show_untagged:
# select only packages without tags
pkg_objs = [p for p in pkgs_by_name.itervalues() if not p.tags]
print format_matches(pkg_objs, args)
exit(0)
if args.verbose:
nbugs = sum([len(p.bugs) for p in pkgs_by_name.itervalues()])
print "loaded %d bugs in %s packages" % (nbugs, len(pkgs_by_name))
# filter packages using user-supplied tags
tag_db = StringIO("\n".join([str(p) for p in pkgs_by_name.itervalues()]))
if not args.batch_queries:
pkg_objs = gen_matches(tag_db, pkgs_by_name, args)
print format_matches(pkg_objs, args)
exit(0)
# else we're in batch mode
for facet in args.batch_queries:
if args.verbose:
print "processing \"%s\" tags" % facet
facet_dir = "%s/%s" % (args.batch_dir, facet)
ensure_dir_exists(facet_dir)
for tag in vocabulary.tags_of_facet(facet):
args.match_tags = set([tag])
pkg_objs = gen_matches(tag_db, pkgs_by_name, args)
try:
tag_value = tag.split("::")[1]
except ValueError:
warn("skipping invalid tag \"%s\"" % tag)
continue
filename = "%s/%s" % (facet_dir, tag_value)
matches = format_matches(pkg_objs, args)
create_file(filename, matches)
tag_db.seek(0)
