def main(argv):
pool = Pool(20)
for lib in LIBRARIES:
pages, keywords = fetch_pages(lib)
mapping = {}
for names, page in pool.imap_unordered(
fetch_page, [(lib, p, keywords) for p in pages]):
print page
mapping.update(names)
ns = lib.namespace
namespace, version = ns.split("-")
with open(get_docref_path(namespace, version), "wb") as h:
h.write(json.dumps(mapping, sort_keys=True, indent=4))
def main(argv):
pool = Pool(20)
for lib in LIBRARIES:
pages, keywords = fetch_pages(lib)
mapping = {}
for names, page in pool.imap_unordered(fetch_page, [(lib, p, keywords)
for p in pages]):
print page
mapping.update(names)
ns = lib.namespace
namespace, version = ns.split("-")
with open(get_docref_path(namespace, version), "wb") as h:
h.write(json.dumps(mapping, sort_keys=True, indent=4))
def main(argv):
pool = Pool(20)
parser = argparse.ArgumentParser(description='Fetch docrefs')
parser.add_argument('namespace', nargs="*",
help='namespace including version e.g. Gtk-3.0')
try:
args = parser.parse_args(argv[1:])
except SystemExit:
raise SystemExit(1)
if not args.namespace:
libraries = LIBRARIES
else:
libraries = []
for l in LIBRARIES:
if l.namespace in args.namespace:
libraries.append(l)
if len(args.namespace) != len(libraries):
print("Invalid namespaces in %s" % args.namespace)
raise SystemExit(1)
for lib in libraries:
pages, keywords = fetch_pages(lib)
mapping = {}
for names, page in pool.imap_unordered(
fetch_page, [(lib, p, keywords) for p in pages]):
print(page)
mapping.update(names)
ns = lib.namespace
namespace, version = ns.split("-")
with open(get_docref_path(namespace, version), "wb") as h:
h.write(
json.dumps(mapping, sort_keys=True, indent=4).encode("utf-8"))
def test_get_docref_path(self):
self.assertTrue(os.path.isfile(get_docref_path("Gtk", "3.0")))
def test_get_docref_path(self):
self.assertTrue(os.path.isfile(get_docref_path("Gtk", "3.0")))
