Example #1
0
    def __init__(self):
        self.conf = Config()
        usage = \
"""usage: %prog --match-tags t1,t2,... [--exclude-tags t3,t4,...]
                     [-t RFA,O,...] [-f] [-v]
       %prog --batch-queries-file --dest-dir <dirname>
       %prog --list-valid-tags
       %prog --untagged-pkgs-only"""
        parser = OptionParser(usage)
        parser.add_option("-b", "--batch-queries-file", dest="batch_qfile",
                          help="""in batch mode, read queries from this file
                          (one query per line)""", default=None)
        parser.add_option("-d", "--dest-dir", dest="batch_dir",
                          help="""in batch mode, create a result file for
                          every query in this directory""", default=None)
        parser.add_option("-m", "--match-tags", dest="match_tags",
                          help="""match packages having all these tags
                          (comma-separated list; not to be used with -l or -u)""")
        parser.add_option("-x", "--exclude-tags", dest="excl_tags",
                          help="""filter out packages having any of these tags
                          (comma-separated list)""")
        parser.add_option("-t", "--bug-types", dest="bug_types", default="any",
                          help="""query only against bugs of the types in this
                          comma-separated list (valid types: any, O, RFA, RFH,
                          ITP, being_adopted; default: any)""")
        parser.add_option("-l", "--list-valid-tags", action="store_true",
                          dest="list_tags", help="""use this if you don't know
                          what to query for (not to be used with -m or -u)""")
        parser.add_option("-u", "--untagged-pkgs-only", action="store_true",
                          dest="show_untagged",
                          help="""list only bugs for packages that haven't
                          been tagged yet (not to be used with -m or -l)""")
        parser.add_option("-f", "--force-update", action="store_true",
                          dest="force_update", default=False,
                          help="""update bug and popcon data regardless of age
                                  (by default,  bug data is updated when it's
                                  older than 7 days, and popcon data when it's
                                  older than 30 days)""")
        parser.add_option("--cache-dir", dest="cache_dir",
                          default=os.path.expanduser("~/.devscripts_cache"),
                          help="""cache directory for bug and popcon data
                          (default: ~/.devscripts_cache""")
        parser.add_option("--debtags-file", dest="debtags_file",
                          default="/var/lib/debtags/package-tags",
                          help="""use an alternative debtags file
                          (default: /var/lib/debtags/package-tags)""")
        parser.add_option("--vocabulary-file", dest="tags_file",
                          default="/var/lib/debtags/vocabulary",
                          help="""user-supplied tags for filtering are checked
                          against this tag file (default:
                          /var/lib/debtags/vocabulary)""")
        parser.add_option("-v", "--verbose", action="store_true",
                          dest="verbose", default=False)
        parser.add_option("-V", "--version", action="store_true",
                          dest="show_version", default=False,
                          help="show version information")
        (options, args) = parser.parse_args()
        if len(args) > 0:
            parser.error("Unknown argument %s")
        if options.show_version:
            print "wnpp-by-tags %s" % __version__
            exit(0)
        options.match_tags or options.show_untagged or options.list_tags or \
                options.batch_qfile or \
            parser.error("Please specify at least either -m, -b, -u or -l")
        options.match_tags and options.show_untagged and \
            parser.error("Please specify either -m or -u")
        if options.batch_qfile or options.batch_dir:
            options.batch_qfile and options.batch_dir or \
                parser.error("In batch mode, you have to specify both -b and -d")
            if options.batch_qfile == "-":
                queries_fd = sys.stdin
            else:
                assert os.path.exists(options.batch_qfile)
                queries_fd = open(options.batch_qfile)
            self.batch_queries = queries_fd.read().strip().split("\n")
            ensure_dir_exists(options.batch_dir)
            self.batch_dir = options.batch_dir
        else:
            self.batch_queries = None
            self.batch_dir = None

        self.match_tags = set()
        self.excl_tags = set()
        if options.match_tags:
            # parse tags to match and tags to exclude
            self.match_tags = set(options.match_tags.split(","))
            if options.excl_tags:
                self.excl_tags = set(options.excl_tags.split(","))

        if options.bug_types == "any":
            # query against default bug types
            self.full_name_bug_types = self.conf.bug_types_to_query
            self.bug_types = [BugType.abbreviation_of(bt) \
                              for bt in self.full_name_bug_types]
        else:
            # convert any bug type acronyms to uppercase
            self.bug_types = [b.upper() if len(b) <= 3 else b \
                              for b in options.bug_types.split(",")]
            # check that the user-supplied bug types are valid
            self.full_name_bug_types = [BugType.full_name_of(bt) for bt in self.bug_types]
            invalid_bug_types = set(self.full_name_bug_types).difference(self.conf.known_bug_types)
            if invalid_bug_types:
                giveup("invalid  bug type(s): %s" % ", ".join(list(invalid_bug_types)))
            self.bug_types = [BugType.abbreviation_of(bt) for bt in self.bug_types]
        self.bug_types = set(self.bug_types)

        self.force_update = options.force_update
        self.verbose = options.verbose
        self.list_tags = options.list_tags
        self.show_untagged = options.show_untagged
        self.debtags_file = os.path.abspath(options.debtags_file)
        self.cache_dir = os.path.abspath(options.cache_dir)
        self.tags_file = options.tags_file
Example #2
0
def main():
    if  len(sys.argv) < 2 or '-h' in sys.argv or '--help' in sys.argv:
        syntax()
    src_dir = sys.argv[1]
    dst_dir = sys.argv[2]
    try:
        if not os.path.isfile(sys.argv[3]):
            giveup("\"%s\" does not exist or is not a file" % sys.argv[3])
        index_page_msg = open(sys.argv[3]).read()
    except IndexError:
        index_page_msg = ""
    if not os.path.isdir(src_dir):
        giveup("\"%s\" does not exist or is not a directory" % src_dir)
    facets = []
    timestamp = "%d-%02d-%02d %02d:%02d" % time.gmtime()[:5]
    for src_facet_dir in glob("%s/*" % src_dir):
        if not os.path.isdir(src_facet_dir):
            continue
        facet = src_facet_dir.split("/")[-1]
        facet_values = []
        facets.append(facet)
        for src_tag_file in glob("%s/*" % src_facet_dir):
            facet_value = os.path.basename(src_tag_file)
            dst_facet_dir = "%s/%s" % (dst_dir, facet)
            ensure_dir_exists(dst_facet_dir)
            bug_table_rows = []
            for line in open(src_tag_file).readlines():
                # the columns are: bug type, bug number, package, popcon, dust
                cols = line.rstrip().split(" ")
                _, bug_no, package_name, popcon = cols[:4]
                if len(cols) != 5:
                    sys.stderr.write("skipping invalid line \"%s\"" % line)
                    continue
                # add link for bug number and package
                cols[1] = '<a href="http://bugs.debian.org/%s">%s</a>' \
                        % (bug_no, bug_no)
                cols[2] = '<a href="http://packages.qa.debian.org/%s/%s.html">%s</a>' \
                        % (package_name[0], package_name, package_name)
                cols[3] = '<a href="http://qa.debian.org/popcon.php?package=%s">%s</a>' \
                        % (package_name, popcon)
                cols = "</td><td>".join(cols)
                bug_table_rows.append("<tr><td>%s</td></tr>" % cols)
            nbugs = len(bug_table_rows)
            if nbugs > 0: row = "<tr><td><a href=\"./%s.html\">%s</a></td><td>%d</td></tr>" \
                        % (facet_value, facet_value, nbugs)
            else:
                row = "<tr><td>%s</td><td>%d</td></tr>" % (facet_value, nbugs)
            facet_values.append(row)
            dst_tag_file = "%s/%s.html" % (dst_facet_dir, facet_value)
            # FIXME: use newstyle dicts instead
            tag = "%s::%s" % (facet, facet_value)
            html_doc = bugs_html_template % (tag, tag, "\n".join(bug_table_rows),
                                             facet, timestamp)
            create_file(dst_tag_file, html_doc)
        content = facet_values_html_template % \
                (facet, facet, facet, "\n".join(sorted(facet_values)), timestamp)
        create_file("%s/index.html" % dst_facet_dir, content)
    formated_facets = ["<li><a href=\"./%s/index.html\">%s</a></li>" % (f, f)
                       for f in facets]
    create_file("%s/index.html" % dst_dir,
                main_page_template % ("\n".join(sorted(formated_facets)),
                                      index_page_msg,  timestamp))
Example #3
0
def main():
    # parse user-supplied arguments
    args = Arguments()
    # sanity checks
    if not os.path.exists(args.debtags_file):
        giveup("The debtags file %s doesn't exist" % args.debtags_file)
    if not os.path.exists(args.tags_file):
        giveup("The tag vocabulary file %s doesn't exist" % args.tags_file)

    if args.match_tags or args.list_tags or args.batch_queries:
        vocabulary = TagVocabulary(args.tags_file)
        if args.list_tags:
            print vocabulary
            exit(0)
        invalid_tags = vocabulary.invalid_tags(args.match_tags.union(args.excl_tags))
        if invalid_tags:
            giveup("The following tags are not listed in %s:\n%s" % \
                    (args.tags_file, "\n".join(list(invalid_tags))))

    # misc initialisations
    bugs_dir = "%s/wnpp" % args.cache_dir
    popcon_dir = "%s/popcon" % args.cache_dir
    ensure_dir_exists(bugs_dir)
    ensure_dir_exists(popcon_dir)
    update_bug_data(args.force_update, bugs_dir, args.full_name_bug_types,
            args.conf.bugs_update_period_in_days, args.verbose)
    update_popcon_data(args.force_update, popcon_dir,
            args.conf.popcon_update_period_in_days, args.verbose)
    debtags = Debtags(open(args.debtags_file))
    popcon_file = "%s/%s" % (popcon_dir, POPCON_FNAME)
    assert os.path.isfile(popcon_file)
    popcon = Popcon(open(popcon_file, "r"))
    Package.init_sources(debtags.tags_of_pkg, popcon.inst_of_pkg)

    # build dict of package objects, indexed by package name, using the HTML
    # BTS pages
    pkgs_by_name = {}
    for bug_page in glob("%s/*.html" % bugs_dir):
        bug_type = BugType.abbreviation_of(os.path.basename(bug_page)[:-5])
        if bug_type in args.bug_types:
            pkgs_by_name = extract_bugs(open(bug_page, "r"), pkgs_by_name, bug_type)

    if args.show_untagged:
        # select only packages without tags
        pkg_objs = [p for p in pkgs_by_name.itervalues() if not p.tags]
        print format_matches(pkg_objs, args)
        exit(0)

    if args.verbose:
        nbugs = sum([len(p.bugs) for p in pkgs_by_name.itervalues()])
        print "loaded %d bugs in %s packages" % (nbugs, len(pkgs_by_name))

    # filter packages using user-supplied tags
    tag_db = StringIO("\n".join([str(p) for p in pkgs_by_name.itervalues()]))
    if not args.batch_queries:
        pkg_objs = gen_matches(tag_db, pkgs_by_name, args)
        print format_matches(pkg_objs, args)
        exit(0)

    # else we're in batch mode
    for facet in args.batch_queries:
        if args.verbose:
            print "processing \"%s\" tags" % facet
        facet_dir = "%s/%s" % (args.batch_dir, facet)
        ensure_dir_exists(facet_dir)
        for tag in vocabulary.tags_of_facet(facet):
            args.match_tags = set([tag])
            pkg_objs = gen_matches(tag_db, pkgs_by_name, args)
            try:
                tag_value = tag.split("::")[1]
            except ValueError:
                warn("skipping invalid tag \"%s\"" % tag)
                continue
            filename = "%s/%s" % (facet_dir, tag_value)
            matches = format_matches(pkg_objs, args)
            create_file(filename, matches)
            tag_db.seek(0)