Exemple #1
0
 def mkcolldir(self, name):
     cid = serve.make_collection_id({'metabook': json.dumps({'title': name,  "type":"collection"})})
     d = os.path.join(self.tmpdir, cid[0], cid[:2], cid)
     os.makedirs(d)
     f = open(os.path.join(d, 'output.rl'), 'wb')
     f.write('bla')
     f.close()
     return d
Exemple #2
0
def main():
    import optparse
    import sys

    from mwlib.client import Client
    import mwlib.myjson as json

    parser = optparse.OptionParser(usage="%prog [OPTIONS] COMMAND [ARGS]")
    default_url = 'http://localhost:8899/'
    parser.add_option('-u', '--url',
                      help='URL of HTTP interface to mw-serve (default: %r)' % default_url,
                      default=default_url,
                      )
    options, args = parser.parse_args()

    if not args:
        parser.error('argument required')

    command = args[0]
    data = {}
    for arg in args[1:]:
        if '=' in arg:
            key, value = [x.strip() for x in arg.split('=', 1)]
        else:
            key = arg.strip()
            value = True
        data[key] = value

    if 'metabook' in data:
        data['metabook'] = open(data['metabook'], 'rb').read()

    client = Client(options.url)
    if not client.request(command, data, is_json=(command != 'download')):
        if client.error is not None:
            sys.exit('request failed: %s' % client.error)
        else:
            sys.exit('request failed: got response code %d\n%r' %
                     (client.response_code, client.response))

    if command == "download":
        fn = 'output'
        open(fn, 'w').write(client.response)
        print 'wrote %d bytes to %r' % (len(client.response), fn)
    else:
        print json.dumps(client.response, indent=4)
Exemple #3
0
def mkcolldir(tmpdir, name):
    cid = serve.make_collection_id(
        {'metabook': json.dumps({
            'title': name,
            "type": "collection"
        })})
    d = tmpdir.join(cid[0], cid[:2], cid).ensure(dir=1)
    d.join("output.rl").write("bla")
    return d
Exemple #4
0
            revisions = p.get("revisions")
            
            if revisions is None:
                continue
            
            for r in revisions:
                revid = r.get("revid")
                txt = r["*"]
                if revid not in self.seen:
                    rev = dict(title=title, ns=ns)
                    if revid is not None:
                        self.seen[revid] = rev
                        rev["revid"] = revid
                    self.seen[title] = rev
                    
                    header = "\n --page-- %s\n" % json.dumps(rev, sort_keys=True)
                    self.revfile.write(header)
                    self.revfile.write(txt.encode("utf-8"))
                # else:    
                #     print "fsoutput: skipping duplicate:", dict(revid=revid, title=title)

    def write_edits(self, edits):
        self.dump_json(edits=edits)

    def write_redirects(self, redirects):
        self.dump_json(redirects=redirects)
        
                        
def splitblocks(lst, limit):
    """Split list lst in blocks of max. limit entries. Return list of blocks."""
Exemple #5
0
        {
            'type': 'chapter',
            'title': 'Chapter 2',
            'items': [
                {
                    'type': 'article',
                    'title': 'Article 3',
                    'displaytitle': 'Display Title',
                    'content_type': 'text/x-wiki',
                },
            ],
        },
    ],
}

test_metabook = json.loads(json.dumps(test_metabook))


def test_parse_collection_page():
    #first parsestring
    mb = metabook.parse_collection_page(test_wikitext1)
    print mb

    assert mb['type'] == 'collection'
    assert mb['version'] == 1
    assert mb['title'] == 'Title'
    assert mb['subtitle'] == 'Subtitle'
    assert mb['summary'] == 'Summary line 1 Summary line 2 '
    items = mb['items']
    assert len(items) == 2
    assert items[0]['type'] == 'chapter'
Exemple #6
0
            'chapter',
            'title':
            'Chapter 2',
            'items': [
                {
                    'type': 'article',
                    'title': 'Article 3',
                    'displaytitle': 'Display Title',
                    'content_type': 'text/x-wiki',
                },
            ],
        },
    ],
}

test_metabook = json.loads(json.dumps(test_metabook))


def test_parse_collection_page():
    #first parsestring
    mb = metabook.parse_collection_page(test_wikitext1)
    print mb

    assert mb['type'] == 'collection'
    assert mb['version'] == 1
    assert mb['title'] == 'Title'
    assert mb['subtitle'] == 'Subtitle'
    assert mb['summary'] == 'Summary line 1 Summary line 2 '
    items = mb['items']
    assert len(items) == 2
    assert items[0]['type'] == 'chapter'
Exemple #7
0
 def set_db_key(self, name, key, value):
     storage = getattr(self, name, None)
     assert storage is not None, 'storage not existant %s' % name
     storage[key] = json.dumps(value)
Exemple #8
0
    def write_siteinfo(self, siteinfo):
        self.dump_json(siteinfo=siteinfo)

    def write_excluded(self, excluded):
        self.dump_json(excluded=excluded)

    def write_licenses(self, licenses):
        self.dump_json(licenses=licenses)

    def write_expanded_page(self, title, ns, txt, revid=None):
        rev = dict(title=title, ns=ns, expanded=1)
        if revid is not None:
            rev["revid"] = revid

        header = "\n --page-- %s\n" % json.dumps(rev, sort_keys=True)
        self.revfile.write(header)
        self.revfile.write(txt.encode("utf-8"))
        self.seen[title] = rev

    def write_pages(self, data):
        pages = data.get("pages", {}).values()
        for p in pages:

            title = p.get("title")
            ns = p.get("ns")
            revisions = p.get("revisions")

            if revisions is None:
                continue
Exemple #9
0
def mkcolldir(tmpdir, name):
    cid = serve.make_collection_id({'metabook': json.dumps({'title': name,  "type": "collection"})})
    d = tmpdir.join(cid[0], cid[:2], cid).ensure(dir=1)
    d.join("output.rl").write("bla")
    return d
Exemple #10
0
 def wrapper(*args, **kwargs):
     result = fn(*args, **kwargs)
     if isinstance(result, Response):
         return result
     return Response(json.dumps(result), content_type='application/json')
Exemple #11
0
 def dumps(self):
     from mwlib import myjson
     return myjson.dumps(self, sort_keys=True, indent=4)
Exemple #12
0
 def dumps(self):
     from mwlib import myjson
     return myjson.dumps(self, sort_keys=True, indent=4)