def mkcolldir(self, name):
cid = serve.make_collection_id({'metabook': json.dumps({'title': name, "type":"collection"})})
d = os.path.join(self.tmpdir, cid[0], cid[:2], cid)
os.makedirs(d)
f = open(os.path.join(d, 'output.rl'), 'wb')
f.write('bla')
f.close()
return d
def main():
import optparse
import sys
from mwlib.client import Client
import mwlib.myjson as json
parser = optparse.OptionParser(usage="%prog [OPTIONS] COMMAND [ARGS]")
default_url = 'http://localhost:8899/'
parser.add_option('-u', '--url',
help='URL of HTTP interface to mw-serve (default: %r)' % default_url,
default=default_url,
)
options, args = parser.parse_args()
if not args:
parser.error('argument required')
command = args[0]
data = {}
for arg in args[1:]:
if '=' in arg:
key, value = [x.strip() for x in arg.split('=', 1)]
else:
key = arg.strip()
value = True
data[key] = value
if 'metabook' in data:
data['metabook'] = open(data['metabook'], 'rb').read()
client = Client(options.url)
if not client.request(command, data, is_json=(command != 'download')):
if client.error is not None:
sys.exit('request failed: %s' % client.error)
else:
sys.exit('request failed: got response code %d\n%r' %
(client.response_code, client.response))
if command == "download":
fn = 'output'
open(fn, 'w').write(client.response)
print 'wrote %d bytes to %r' % (len(client.response), fn)
else:
print json.dumps(client.response, indent=4)
def mkcolldir(tmpdir, name):
cid = serve.make_collection_id(
{'metabook': json.dumps({
'title': name,
"type": "collection"
})})
d = tmpdir.join(cid[0], cid[:2], cid).ensure(dir=1)
d.join("output.rl").write("bla")
return d
revisions = p.get("revisions")
if revisions is None:
continue
for r in revisions:
revid = r.get("revid")
txt = r["*"]
if revid not in self.seen:
rev = dict(title=title, ns=ns)
if revid is not None:
self.seen[revid] = rev
rev["revid"] = revid
self.seen[title] = rev
header = "\n --page-- %s\n" % json.dumps(rev, sort_keys=True)
self.revfile.write(header)
self.revfile.write(txt.encode("utf-8"))
# else:
# print "fsoutput: skipping duplicate:", dict(revid=revid, title=title)
def write_edits(self, edits):
self.dump_json(edits=edits)
def write_redirects(self, redirects):
self.dump_json(redirects=redirects)
def splitblocks(lst, limit):
"""Split list lst in blocks of max. limit entries. Return list of blocks."""
{
'type': 'chapter',
'title': 'Chapter 2',
'items': [
{
'type': 'article',
'title': 'Article 3',
'displaytitle': 'Display Title',
'content_type': 'text/x-wiki',
},
],
},
],
}
test_metabook = json.loads(json.dumps(test_metabook))
def test_parse_collection_page():
#first parsestring
mb = metabook.parse_collection_page(test_wikitext1)
print mb
assert mb['type'] == 'collection'
assert mb['version'] == 1
assert mb['title'] == 'Title'
assert mb['subtitle'] == 'Subtitle'
assert mb['summary'] == 'Summary line 1 Summary line 2 '
items = mb['items']
assert len(items) == 2
assert items[0]['type'] == 'chapter'
'chapter',
'title':
'Chapter 2',
'items': [
{
'type': 'article',
'title': 'Article 3',
'displaytitle': 'Display Title',
'content_type': 'text/x-wiki',
},
],
},
],
}
test_metabook = json.loads(json.dumps(test_metabook))
def test_parse_collection_page():
#first parsestring
mb = metabook.parse_collection_page(test_wikitext1)
print mb
assert mb['type'] == 'collection'
assert mb['version'] == 1
assert mb['title'] == 'Title'
assert mb['subtitle'] == 'Subtitle'
assert mb['summary'] == 'Summary line 1 Summary line 2 '
items = mb['items']
assert len(items) == 2
assert items[0]['type'] == 'chapter'
def set_db_key(self, name, key, value):
storage = getattr(self, name, None)
assert storage is not None, 'storage not existant %s' % name
storage[key] = json.dumps(value)
def write_siteinfo(self, siteinfo):
self.dump_json(siteinfo=siteinfo)
def write_excluded(self, excluded):
self.dump_json(excluded=excluded)
def write_licenses(self, licenses):
self.dump_json(licenses=licenses)
def write_expanded_page(self, title, ns, txt, revid=None):
rev = dict(title=title, ns=ns, expanded=1)
if revid is not None:
rev["revid"] = revid
header = "\n --page-- %s\n" % json.dumps(rev, sort_keys=True)
self.revfile.write(header)
self.revfile.write(txt.encode("utf-8"))
self.seen[title] = rev
def write_pages(self, data):
pages = data.get("pages", {}).values()
for p in pages:
title = p.get("title")
ns = p.get("ns")
revisions = p.get("revisions")
if revisions is None:
continue
def mkcolldir(tmpdir, name):
cid = serve.make_collection_id({'metabook': json.dumps({'title': name, "type": "collection"})})
d = tmpdir.join(cid[0], cid[:2], cid).ensure(dir=1)
d.join("output.rl").write("bla")
return d
def wrapper(*args, **kwargs):
result = fn(*args, **kwargs)
if isinstance(result, Response):
return result
return Response(json.dumps(result), content_type='application/json')
def dumps(self):
from mwlib import myjson
return myjson.dumps(self, sort_keys=True, indent=4)
def dumps(self):
from mwlib import myjson
return myjson.dumps(self, sort_keys=True, indent=4)