def main(argv): pool = Pool(20) for lib in LIBRARIES: pages, keywords = fetch_pages(lib) mapping = {} for names, page in pool.imap_unordered( fetch_page, [(lib, p, keywords) for p in pages]): print page mapping.update(names) ns = lib.namespace namespace, version = ns.split("-") with open(get_docref_path(namespace, version), "wb") as h: h.write(json.dumps(mapping, sort_keys=True, indent=4))
def main(argv): pool = Pool(20) for lib in LIBRARIES: pages, keywords = fetch_pages(lib) mapping = {} for names, page in pool.imap_unordered(fetch_page, [(lib, p, keywords) for p in pages]): print page mapping.update(names) ns = lib.namespace namespace, version = ns.split("-") with open(get_docref_path(namespace, version), "wb") as h: h.write(json.dumps(mapping, sort_keys=True, indent=4))
def main(argv): pool = Pool(20) parser = argparse.ArgumentParser(description='Fetch docrefs') parser.add_argument('namespace', nargs="*", help='namespace including version e.g. Gtk-3.0') try: args = parser.parse_args(argv[1:]) except SystemExit: raise SystemExit(1) if not args.namespace: libraries = LIBRARIES else: libraries = [] for l in LIBRARIES: if l.namespace in args.namespace: libraries.append(l) if len(args.namespace) != len(libraries): print("Invalid namespaces in %s" % args.namespace) raise SystemExit(1) for lib in libraries: pages, keywords = fetch_pages(lib) mapping = {} for names, page in pool.imap_unordered( fetch_page, [(lib, p, keywords) for p in pages]): print(page) mapping.update(names) ns = lib.namespace namespace, version = ns.split("-") with open(get_docref_path(namespace, version), "wb") as h: h.write( json.dumps(mapping, sort_keys=True, indent=4).encode("utf-8"))
def test_get_docref_path(self): self.assertTrue(os.path.isfile(get_docref_path("Gtk", "3.0")))