Ejemplo n.º 1
0
def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
    id2wiki = {}
    for x in metabook.wikis:
        id2wiki[x.ident] = (x, [])

    for x in metabook.articles():
        assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (x.wikiident, x)
        id2wiki[x.wikiident][1].append(x)

    is_multiwiki = len(id2wiki) > 1

    if is_multiwiki:
        progress = fetch.shared_progress(status=status)
    else:
        progress = None

    fetchers = []
    for id, (wikiconf, articles) in id2wiki.items():
        if id is None:
            id = ""
            assert not is_multiwiki, "id must be set in multiwiki"

        if not is_multiwiki:
            id = ""

        assert "/" not in id, "bad id: %r" % (id,)
        my_fsdir = os.path.join(fsdir, id)

        if is_multiwiki:
            my_mb = collection()
            my_mb.items = articles
        else:
            my_mb = metabook

        wikitrust(wikiconf.baseurl, my_mb)

        fetchers.append(start_fetcher(fsdir=my_fsdir, progress=progress, base_url=wikiconf.baseurl,
                                      metabook=my_mb, options=options, podclient=podclient, status=status))

    if is_multiwiki:
        if not os.path.exists(fsdir):
            os.makedirs(fsdir)
        open(os.path.join(fsdir, "metabook.json"), "wb").write(metabook.dumps())
        myjson.dump(dict(format="multi-nuwiki"), open(os.path.join(fsdir, "nfo.json"), "wb"))

    pool = gevent.pool.Pool()
    for x in fetchers:
        pool.spawn(x.run)
    pool.join(raise_error=True)

    import signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)
    signal.signal(signal.SIGTERM, signal.SIG_DFL)
Ejemplo n.º 2
0
Archivo: fetch.py Proyecto: ingob/mwlib
 def dump_json(self, **kw):
     for k, v in kw.items():
         p = os.path.join(self.path, k+".json")
         json.dump(v, open(p, "wb"), indent=4, sort_keys=True)
Ejemplo n.º 3
0
def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
    id2wiki = {}
    for x in metabook.wikis:
        id2wiki[x.ident] = (x, [])

    for x in metabook.articles():
        assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (
            x.wikiident, x)
        id2wiki[x.wikiident][1].append(x)

    is_multiwiki = len(id2wiki) > 1

    if is_multiwiki:
        progress = fetch.shared_progress(status=status)
    else:
        progress = None

    fetchers = []
    for id, (wikiconf, articles) in id2wiki.items():
        if id is None:
            id = ""
            assert not is_multiwiki, "id must be set in multiwiki"

        if not is_multiwiki:
            id = ""

        assert "/" not in id, "bad id: %r" % (id, )
        my_fsdir = os.path.join(fsdir, id)

        if is_multiwiki:
            my_mb = collection()
            my_mb.items = articles
        else:
            my_mb = metabook

        wikitrust(wikiconf.baseurl, my_mb)

        fetchers.append(
            start_fetcher(fsdir=my_fsdir,
                          progress=progress,
                          base_url=wikiconf.baseurl,
                          metabook=my_mb,
                          options=options,
                          podclient=podclient,
                          status=status))

    if is_multiwiki:
        if not os.path.exists(fsdir):
            os.makedirs(fsdir)
        open(os.path.join(fsdir, "metabook.json"),
             "wb").write(metabook.dumps())
        myjson.dump(dict(format="multi-nuwiki"),
                    open(os.path.join(fsdir, "nfo.json"), "wb"))

    pool = gevent.pool.Pool()
    for x in fetchers:
        pool.spawn(x.run)
    pool.join(raise_error=True)

    import signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)
    signal.signal(signal.SIGTERM, signal.SIG_DFL)
Ejemplo n.º 4
0
 def dump_json(self, **kw):
     for k, v in kw.items():
         p = os.path.join(self.path, k + ".json")
         json.dump(v, open(p, "wb"), indent=4, sort_keys=True)
Ejemplo n.º 5
0
def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
    id2wiki = {}
    for x in metabook.wikis:
        id2wiki[x.ident] = (x, [])

    for x in metabook.articles():
        assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (x.wikiident,  x)
        id2wiki[x.wikiident][1].append(x)

    is_multiwiki = len(id2wiki)>1
    
    if is_multiwiki:
        progress = fetch.shared_progress(status=status)
    else:
        progress = None
        
    fetchers =[]
    for id, (wikiconf, articles) in id2wiki.items():
        if id is None:
            id = ""
            assert not is_multiwiki, "id must be set in multiwiki"

        if not is_multiwiki:
            id = ""
        
        assert "/" not in id, "bad id: %r" % (id,)
        my_fsdir = os.path.join(fsdir, id)
        
        if is_multiwiki:
            my_mb = collection()
            my_mb.items = articles
        else:
            my_mb = metabook

        wikitrust(wikiconf.baseurl, my_mb)

        fetchers.append(start_fetcher(fsdir=my_fsdir, progress=progress, base_url=wikiconf.baseurl, metabook=my_mb, options=options, podclient=podclient, status=status))

    if is_multiwiki:
        if not os.path.exists(fsdir):
            os.makedirs(fsdir)
        open(os.path.join(fsdir, "metabook.json"),  "wb").write(metabook.dumps())
        myjson.dump(dict(format="multi-nuwiki"), open(os.path.join(fsdir, "nfo.json"), "wb"))
        
        
    retval = []
    def done(listres):
        retval.extend(listres)
        reactor.stop()

    def run():
        return defer.DeferredList([x.run() for x in fetchers])
            
    reactor.callLater(0.0, lambda: run().addBoth(done))
    reactor.run()
    import signal
    signal.signal(signal.SIGINT,  signal.SIG_DFL)
    signal.signal(signal.SIGTERM,  signal.SIG_DFL)
    
    if not retval:
        raise KeyboardInterrupt("interrupted")

    for success, val in retval:
        if not success:
            raise RuntimeError(str(val))