def __init__(self):
self.conf = Config()
usage = \
"""usage: %prog --match-tags t1,t2,... [--exclude-tags t3,t4,...]
[-t RFA,O,...] [-f] [-v]
%prog --batch-queries-file --dest-dir <dirname>
%prog --list-valid-tags
%prog --untagged-pkgs-only"""
parser = OptionParser(usage)
parser.add_option("-b", "--batch-queries-file", dest="batch_qfile",
help="""in batch mode, read queries from this file
(one query per line)""", default=None)
parser.add_option("-d", "--dest-dir", dest="batch_dir",
help="""in batch mode, create a result file for
every query in this directory""", default=None)
parser.add_option("-m", "--match-tags", dest="match_tags",
help="""match packages having all these tags
(comma-separated list; not to be used with -l or -u)""")
parser.add_option("-x", "--exclude-tags", dest="excl_tags",
help="""filter out packages having any of these tags
(comma-separated list)""")
parser.add_option("-t", "--bug-types", dest="bug_types", default="any",
help="""query only against bugs of the types in this
comma-separated list (valid types: any, O, RFA, RFH,
ITP, being_adopted; default: any)""")
parser.add_option("-l", "--list-valid-tags", action="store_true",
dest="list_tags", help="""use this if you don't know
what to query for (not to be used with -m or -u)""")
parser.add_option("-u", "--untagged-pkgs-only", action="store_true",
dest="show_untagged",
help="""list only bugs for packages that haven't
been tagged yet (not to be used with -m or -l)""")
parser.add_option("-f", "--force-update", action="store_true",
dest="force_update", default=False,
help="""update bug and popcon data regardless of age
(by default, bug data is updated when it's
older than 7 days, and popcon data when it's
older than 30 days)""")
parser.add_option("--cache-dir", dest="cache_dir",
default=os.path.expanduser("~/.devscripts_cache"),
help="""cache directory for bug and popcon data
(default: ~/.devscripts_cache""")
parser.add_option("--debtags-file", dest="debtags_file",
default="/var/lib/debtags/package-tags",
help="""use an alternative debtags file
(default: /var/lib/debtags/package-tags)""")
parser.add_option("--vocabulary-file", dest="tags_file",
default="/var/lib/debtags/vocabulary",
help="""user-supplied tags for filtering are checked
against this tag file (default:
/var/lib/debtags/vocabulary)""")
parser.add_option("-v", "--verbose", action="store_true",
dest="verbose", default=False)
parser.add_option("-V", "--version", action="store_true",
dest="show_version", default=False,
help="show version information")
(options, args) = parser.parse_args()
if len(args) > 0:
parser.error("Unknown argument %s")
if options.show_version:
print "wnpp-by-tags %s" % __version__
exit(0)
options.match_tags or options.show_untagged or options.list_tags or \
options.batch_qfile or \
parser.error("Please specify at least either -m, -b, -u or -l")
options.match_tags and options.show_untagged and \
parser.error("Please specify either -m or -u")
if options.batch_qfile or options.batch_dir:
options.batch_qfile and options.batch_dir or \
parser.error("In batch mode, you have to specify both -b and -d")
if options.batch_qfile == "-":
queries_fd = sys.stdin
else:
assert os.path.exists(options.batch_qfile)
queries_fd = open(options.batch_qfile)
self.batch_queries = queries_fd.read().strip().split("\n")
ensure_dir_exists(options.batch_dir)
self.batch_dir = options.batch_dir
else:
self.batch_queries = None
self.batch_dir = None
self.match_tags = set()
self.excl_tags = set()
if options.match_tags:
# parse tags to match and tags to exclude
self.match_tags = set(options.match_tags.split(","))
if options.excl_tags:
self.excl_tags = set(options.excl_tags.split(","))
if options.bug_types == "any":
# query against default bug types
self.full_name_bug_types = self.conf.bug_types_to_query
self.bug_types = [BugType.abbreviation_of(bt) \
for bt in self.full_name_bug_types]
else:
# convert any bug type acronyms to uppercase
self.bug_types = [b.upper() if len(b) <= 3 else b \
for b in options.bug_types.split(",")]
# check that the user-supplied bug types are valid
self.full_name_bug_types = [BugType.full_name_of(bt) for bt in self.bug_types]
invalid_bug_types = set(self.full_name_bug_types).difference(self.conf.known_bug_types)
if invalid_bug_types:
giveup("invalid bug type(s): %s" % ", ".join(list(invalid_bug_types)))
self.bug_types = [BugType.abbreviation_of(bt) for bt in self.bug_types]
self.bug_types = set(self.bug_types)
self.force_update = options.force_update
self.verbose = options.verbose
self.list_tags = options.list_tags
self.show_untagged = options.show_untagged
self.debtags_file = os.path.abspath(options.debtags_file)
self.cache_dir = os.path.abspath(options.cache_dir)
self.tags_file = options.tags_file
def main():
if len(sys.argv) < 2 or '-h' in sys.argv or '--help' in sys.argv:
syntax()
src_dir = sys.argv[1]
dst_dir = sys.argv[2]
try:
if not os.path.isfile(sys.argv[3]):
giveup("\"%s\" does not exist or is not a file" % sys.argv[3])
index_page_msg = open(sys.argv[3]).read()
except IndexError:
index_page_msg = ""
if not os.path.isdir(src_dir):
giveup("\"%s\" does not exist or is not a directory" % src_dir)
facets = []
timestamp = "%d-%02d-%02d %02d:%02d" % time.gmtime()[:5]
for src_facet_dir in glob("%s/*" % src_dir):
if not os.path.isdir(src_facet_dir):
continue
facet = src_facet_dir.split("/")[-1]
facet_values = []
facets.append(facet)
for src_tag_file in glob("%s/*" % src_facet_dir):
facet_value = os.path.basename(src_tag_file)
dst_facet_dir = "%s/%s" % (dst_dir, facet)
ensure_dir_exists(dst_facet_dir)
bug_table_rows = []
for line in open(src_tag_file).readlines():
# the columns are: bug type, bug number, package, popcon, dust
cols = line.rstrip().split(" ")
_, bug_no, package_name, popcon = cols[:4]
if len(cols) != 5:
sys.stderr.write("skipping invalid line \"%s\"" % line)
continue
# add link for bug number and package
cols[1] = '<a href="http://bugs.debian.org/%s">%s</a>' \
% (bug_no, bug_no)
cols[2] = '<a href="http://packages.qa.debian.org/%s/%s.html">%s</a>' \
% (package_name[0], package_name, package_name)
cols[3] = '<a href="http://qa.debian.org/popcon.php?package=%s">%s</a>' \
% (package_name, popcon)
cols = "</td><td>".join(cols)
bug_table_rows.append("<tr><td>%s</td></tr>" % cols)
nbugs = len(bug_table_rows)
if nbugs > 0: row = "<tr><td><a href=\"./%s.html\">%s</a></td><td>%d</td></tr>" \
% (facet_value, facet_value, nbugs)
else:
row = "<tr><td>%s</td><td>%d</td></tr>" % (facet_value, nbugs)
facet_values.append(row)
dst_tag_file = "%s/%s.html" % (dst_facet_dir, facet_value)
# FIXME: use newstyle dicts instead
tag = "%s::%s" % (facet, facet_value)
html_doc = bugs_html_template % (tag, tag, "\n".join(bug_table_rows),
facet, timestamp)
create_file(dst_tag_file, html_doc)
content = facet_values_html_template % \
(facet, facet, facet, "\n".join(sorted(facet_values)), timestamp)
create_file("%s/index.html" % dst_facet_dir, content)
formated_facets = ["<li><a href=\"./%s/index.html\">%s</a></li>" % (f, f)
for f in facets]
create_file("%s/index.html" % dst_dir,
main_page_template % ("\n".join(sorted(formated_facets)),
index_page_msg, timestamp))
def main():
# parse user-supplied arguments
args = Arguments()
# sanity checks
if not os.path.exists(args.debtags_file):
giveup("The debtags file %s doesn't exist" % args.debtags_file)
if not os.path.exists(args.tags_file):
giveup("The tag vocabulary file %s doesn't exist" % args.tags_file)
if args.match_tags or args.list_tags or args.batch_queries:
vocabulary = TagVocabulary(args.tags_file)
if args.list_tags:
print vocabulary
exit(0)
invalid_tags = vocabulary.invalid_tags(args.match_tags.union(args.excl_tags))
if invalid_tags:
giveup("The following tags are not listed in %s:\n%s" % \
(args.tags_file, "\n".join(list(invalid_tags))))
# misc initialisations
bugs_dir = "%s/wnpp" % args.cache_dir
popcon_dir = "%s/popcon" % args.cache_dir
ensure_dir_exists(bugs_dir)
ensure_dir_exists(popcon_dir)
update_bug_data(args.force_update, bugs_dir, args.full_name_bug_types,
args.conf.bugs_update_period_in_days, args.verbose)
update_popcon_data(args.force_update, popcon_dir,
args.conf.popcon_update_period_in_days, args.verbose)
debtags = Debtags(open(args.debtags_file))
popcon_file = "%s/%s" % (popcon_dir, POPCON_FNAME)
assert os.path.isfile(popcon_file)
popcon = Popcon(open(popcon_file, "r"))
Package.init_sources(debtags.tags_of_pkg, popcon.inst_of_pkg)
# build dict of package objects, indexed by package name, using the HTML
# BTS pages
pkgs_by_name = {}
for bug_page in glob("%s/*.html" % bugs_dir):
bug_type = BugType.abbreviation_of(os.path.basename(bug_page)[:-5])
if bug_type in args.bug_types:
pkgs_by_name = extract_bugs(open(bug_page, "r"), pkgs_by_name, bug_type)
if args.show_untagged:
# select only packages without tags
pkg_objs = [p for p in pkgs_by_name.itervalues() if not p.tags]
print format_matches(pkg_objs, args)
exit(0)
if args.verbose:
nbugs = sum([len(p.bugs) for p in pkgs_by_name.itervalues()])
print "loaded %d bugs in %s packages" % (nbugs, len(pkgs_by_name))
# filter packages using user-supplied tags
tag_db = StringIO("\n".join([str(p) for p in pkgs_by_name.itervalues()]))
if not args.batch_queries:
pkg_objs = gen_matches(tag_db, pkgs_by_name, args)
print format_matches(pkg_objs, args)
exit(0)
# else we're in batch mode
for facet in args.batch_queries:
if args.verbose:
print "processing \"%s\" tags" % facet
facet_dir = "%s/%s" % (args.batch_dir, facet)
ensure_dir_exists(facet_dir)
for tag in vocabulary.tags_of_facet(facet):
args.match_tags = set([tag])
pkg_objs = gen_matches(tag_db, pkgs_by_name, args)
try:
tag_value = tag.split("::")[1]
except ValueError:
warn("skipping invalid tag \"%s\"" % tag)
continue
filename = "%s/%s" % (facet_dir, tag_value)
matches = format_matches(pkg_objs, args)
create_file(filename, matches)
tag_db.seek(0)