def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
id2wiki = {}
for x in metabook.wikis:
id2wiki[x.ident] = (x, [])
for x in metabook.articles():
assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (x.wikiident, x)
id2wiki[x.wikiident][1].append(x)
is_multiwiki = len(id2wiki) > 1
if is_multiwiki:
progress = fetch.shared_progress(status=status)
else:
progress = None
fetchers = []
for id, (wikiconf, articles) in id2wiki.items():
if id is None:
id = ""
assert not is_multiwiki, "id must be set in multiwiki"
if not is_multiwiki:
id = ""
assert "/" not in id, "bad id: %r" % (id,)
my_fsdir = os.path.join(fsdir, id)
if is_multiwiki:
my_mb = collection()
my_mb.items = articles
else:
my_mb = metabook
wikitrust(wikiconf.baseurl, my_mb)
fetchers.append(start_fetcher(fsdir=my_fsdir, progress=progress, base_url=wikiconf.baseurl,
metabook=my_mb, options=options, podclient=podclient, status=status))
if is_multiwiki:
if not os.path.exists(fsdir):
os.makedirs(fsdir)
open(os.path.join(fsdir, "metabook.json"), "wb").write(metabook.dumps())
myjson.dump(dict(format="multi-nuwiki"), open(os.path.join(fsdir, "nfo.json"), "wb"))
pool = gevent.pool.Pool()
for x in fetchers:
pool.spawn(x.run)
pool.join(raise_error=True)
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
def dump_json(self, **kw):
for k, v in kw.items():
p = os.path.join(self.path, k+".json")
json.dump(v, open(p, "wb"), indent=4, sort_keys=True)
def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
id2wiki = {}
for x in metabook.wikis:
id2wiki[x.ident] = (x, [])
for x in metabook.articles():
assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (
x.wikiident, x)
id2wiki[x.wikiident][1].append(x)
is_multiwiki = len(id2wiki) > 1
if is_multiwiki:
progress = fetch.shared_progress(status=status)
else:
progress = None
fetchers = []
for id, (wikiconf, articles) in id2wiki.items():
if id is None:
id = ""
assert not is_multiwiki, "id must be set in multiwiki"
if not is_multiwiki:
id = ""
assert "/" not in id, "bad id: %r" % (id, )
my_fsdir = os.path.join(fsdir, id)
if is_multiwiki:
my_mb = collection()
my_mb.items = articles
else:
my_mb = metabook
wikitrust(wikiconf.baseurl, my_mb)
fetchers.append(
start_fetcher(fsdir=my_fsdir,
progress=progress,
base_url=wikiconf.baseurl,
metabook=my_mb,
options=options,
podclient=podclient,
status=status))
if is_multiwiki:
if not os.path.exists(fsdir):
os.makedirs(fsdir)
open(os.path.join(fsdir, "metabook.json"),
"wb").write(metabook.dumps())
myjson.dump(dict(format="multi-nuwiki"),
open(os.path.join(fsdir, "nfo.json"), "wb"))
pool = gevent.pool.Pool()
for x in fetchers:
pool.spawn(x.run)
pool.join(raise_error=True)
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
def dump_json(self, **kw):
for k, v in kw.items():
p = os.path.join(self.path, k + ".json")
json.dump(v, open(p, "wb"), indent=4, sort_keys=True)
def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
id2wiki = {}
for x in metabook.wikis:
id2wiki[x.ident] = (x, [])
for x in metabook.articles():
assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (x.wikiident, x)
id2wiki[x.wikiident][1].append(x)
is_multiwiki = len(id2wiki)>1
if is_multiwiki:
progress = fetch.shared_progress(status=status)
else:
progress = None
fetchers =[]
for id, (wikiconf, articles) in id2wiki.items():
if id is None:
id = ""
assert not is_multiwiki, "id must be set in multiwiki"
if not is_multiwiki:
id = ""
assert "/" not in id, "bad id: %r" % (id,)
my_fsdir = os.path.join(fsdir, id)
if is_multiwiki:
my_mb = collection()
my_mb.items = articles
else:
my_mb = metabook
wikitrust(wikiconf.baseurl, my_mb)
fetchers.append(start_fetcher(fsdir=my_fsdir, progress=progress, base_url=wikiconf.baseurl, metabook=my_mb, options=options, podclient=podclient, status=status))
if is_multiwiki:
if not os.path.exists(fsdir):
os.makedirs(fsdir)
open(os.path.join(fsdir, "metabook.json"), "wb").write(metabook.dumps())
myjson.dump(dict(format="multi-nuwiki"), open(os.path.join(fsdir, "nfo.json"), "wb"))
retval = []
def done(listres):
retval.extend(listres)
reactor.stop()
def run():
return defer.DeferredList([x.run() for x in fetchers])
reactor.callLater(0.0, lambda: run().addBoth(done))
reactor.run()
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
if not retval:
raise KeyboardInterrupt("interrupted")
for success, val in retval:
if not success:
raise RuntimeError(str(val))