def test_parse_collection_page():
#first parsestring
mb = metabook.parse_collection_page(test_wikitext1)
print mb
assert mb['type'] == 'collection'
assert mb['version'] == 1
assert mb['title'] == 'Title'
assert mb['subtitle'] == 'Subtitle'
assert mb['summary'] == 'Summary line 1 Summary line 2 '
items = mb['items']
assert len(items) == 2
assert items[0]['type'] == 'chapter'
assert items[0]['title'] == 'Chapter 1'
arts = items[0]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 1'
assert arts[1]['type'] == 'article'
assert arts[1]['title'] == 'Article 2'
assert items[1]['type'] == 'chapter'
assert items[1]['title'] == 'Chapter 2'
arts = items[1]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 3'
assert arts[0]['displaytitle'] == 'Display Title 1'
assert arts[1]['title'] == 'Article 4'
assert arts[1]['revision'] == '4'
assert arts[1]['displaytitle'] == 'Display Title 2'
#second parsestring
mb = metabook.parse_collection_page(test_wikitext2)
assert mb['type'] == 'collection'
assert mb['version'] == metabook.collection.version
assert mb['title'] == 'Title'
assert mb['subtitle'] == 'Subtitle'
assert mb['summary'] == 'Summary line 1 Summary line 2 '
items = mb['items']
assert len(items) == 2
assert items[0]['type'] == 'chapter'
assert items[0]['title'] == 'Chapter 1'
arts = items[0]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 1'
assert arts[1]['type'] == 'article'
assert arts[1]['title'] == 'Article 2'
assert items[1]['type'] == 'chapter'
assert items[1]['title'] == 'Chapter 2'
arts = items[1]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 3'
assert arts[0]['displaytitle'] == 'Display Title 1'
assert arts[1]['title'] == 'Article 4'
assert arts[1]['revision'] == '4'
assert arts[1]['displaytitle'] == 'Display Title 2'
def test_parse_collection_page():
#first parsestring
mb = metabook.parse_collection_page(test_wikitext1)
print mb
assert mb['type'] == 'collection'
assert mb['version'] == 1
assert mb['title'] == 'Title'
assert mb['subtitle'] == 'Subtitle'
assert mb['summary'] == 'Summary line 1 Summary line 2 '
items = mb['items']
assert len(items) == 2
assert items[0]['type'] == 'chapter'
assert items[0]['title'] == 'Chapter 1'
arts = items[0]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 1'
assert arts[1]['type'] == 'article'
assert arts[1]['title'] == 'Article 2'
assert items[1]['type'] == 'chapter'
assert items[1]['title'] == 'Chapter 2'
arts = items[1]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 3'
assert arts[0]['displaytitle'] == 'Display Title 1'
assert arts[1]['title'] == 'Article 4'
assert arts[1]['revision'] == '4'
assert arts[1]['displaytitle'] == 'Display Title 2'
#second parsestring
mb = metabook.parse_collection_page(test_wikitext2)
assert mb['type'] == 'collection'
assert mb['version'] == metabook.collection.version
assert mb['title'] == 'Title'
assert mb['subtitle'] == 'Subtitle'
assert mb['summary'] == 'Summary line 1 Summary line 2 '
items = mb['items']
assert len(items) == 2
assert items[0]['type'] == 'chapter'
assert items[0]['title'] == 'Chapter 1'
arts = items[0]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 1'
assert arts[1]['type'] == 'article'
assert arts[1]['title'] == 'Article 2'
assert items[1]['type'] == 'chapter'
assert items[1]['title'] == 'Chapter 2'
arts = items[1]['items']
assert len(arts) == 2
assert arts[0]['type'] == 'article'
assert arts[0]['title'] == 'Article 3'
assert arts[0]['displaytitle'] == 'Display Title 1'
assert arts[1]['title'] == 'Article 4'
assert arts[1]['revision'] == '4'
assert arts[1]['displaytitle'] == 'Display Title 2'
def fetch_collectionpage(self, api):
cp = self.options.collectionpage
if cp is None:
return api
try:
cp = unicode(urllib.unquote(str(cp)), "utf-8")
except Exception:
pass
self.nfo["collectionpage"] = cp
val = api.fetch_pages([cp])
rawtext = val["pages"].values()[0]["revisions"][0]["*"]
mb = self.metabook = parse_collection_page(rawtext)
wikitrust(api.baseurl, mb)
# XXX: localised template parameter names???
meta = extract_metadata(
rawtext, ("cover-image", "cover-color", "text-color", "editor", "description", "sort_as")
)
mb.editor = meta["editor"]
mb.cover_image = meta["cover-image"]
mb.cover_color = meta["cover-color"]
mb.text_color = meta["text-color"]
mb.description = meta["description"]
mb.sort_as = meta["sort_as"]
p = os.path.join(self.fsout.path, "collectionpage.txt")
if isinstance(rawtext, unicode):
rawtext = rawtext.encode("utf-8")
open(p, "wb").write(rawtext)
return api
def fetch_collectionpage(self, api):
cp = self.options.collectionpage
if cp is None:
return api
try:
cp = unicode(urllib.unquote(str(cp)), "utf-8")
except Exception:
pass
self.nfo["collectionpage"] = cp
val = api.fetch_pages([cp])
rawtext = val["pages"].values()[0]["revisions"][0]["*"]
mb = self.metabook = parse_collection_page(rawtext)
wikitrust(api.baseurl, mb)
# XXX: localised template parameter names???
meta = extract_metadata(rawtext,
("cover-image", "cover-color", "text-color",
"editor", "description", "sort_as"))
mb.editor = meta["editor"]
mb.cover_image = meta["cover-image"]
mb.cover_color = meta["cover-color"]
mb.text_color = meta["text-color"]
mb.description = meta["description"]
mb.sort_as = meta["sort_as"]
p = os.path.join(self.fsout.path, "collectionpage.txt")
if isinstance(rawtext, unicode):
rawtext = rawtext.encode("utf-8")
open(p, "wb").write(rawtext)
return api
def got_pages(val):
rawtext = val["pages"].values()[0]["revisions"][0]["*"]
mb = self.metabook = parse_collection_page(rawtext)
wikitrust(api.baseurl, mb) # XXX blocking twisted reactor. we really should use gevent
# XXX: localised template parameter names???
meta = extract_metadata(rawtext, ("cover-image", "cover-color", "text-color", "editor", "description", "sort_as"))
mb.editor = meta["editor"]
mb.cover_image = meta["cover-image"]
mb.cover_color = meta["cover-color"]
mb.text_color = meta["text-color"]
mb.description = meta["description"]
mb.sort_as = meta["sort_as"]
p = os.path.join(self.fsout.path, "collectionpage.txt")
if isinstance(rawtext, unicode):
rawtext=rawtext.encode("utf-8")
open(p,"wb").write(rawtext)
return api
def makewiki(self):
username, password, domain = None, None, None
if self.options.login:
if self.options.login.count(':') == 1:
username, password = self.options.login.split(':', 1)
else:
username, password, domain = self.options.login.split(':', 2)
env = wiki.makewiki(
self.options.config,
metabook=self.metabook,
username=username,
password=password,
domain=domain,
script_extension=self.options.script_extension,
)
if self.options.noimages:
env.images = None
if self.options.template_blacklist or self.options.template_exclusion_category:
if hasattr(env.wiki, 'setTemplateExclusion'):
env.wiki.setTemplateExclusion(
blacklist=self.options.template_blacklist,
category=self.options.template_exclusion_category,
)
else:
log.warn(
'WikiDB does not support setting a template blacklist')
if self.options.collectionpage:
wikitext = env.wiki.getRawArticle(self.options.collectionpage)
if wikitext is None:
raise RuntimeError('No such collection page: %r' %
(self.options.collectionpage, ))
self.metabook = metabook.parse_collection_page(wikitext)
env.metabook = self.metabook
if self.options.title:
env.metabook['title'] = self.options.title
if self.options.subtitle:
env.metabook['subtitle'] = self.options.subtitle
return env
def makewiki(self):
username, password, domain = None, None, None
if self.options.login:
if self.options.login.count(':') == 1:
username, password = self.options.login.split(':', 1)
else:
username, password, domain = self.options.login.split(':', 2)
env = wiki.makewiki(self.options.config,
metabook=self.metabook,
username=username,
password=password,
domain=domain,
script_extension=self.options.script_extension,
)
if self.options.noimages:
env.images = None
if self.options.template_blacklist or self.options.template_exclusion_category:
if hasattr(env.wiki, 'setTemplateExclusion'):
env.wiki.setTemplateExclusion(
blacklist=self.options.template_blacklist,
category=self.options.template_exclusion_category,
)
else:
log.warn('WikiDB does not support setting a template blacklist')
if self.options.collectionpage:
wikitext = env.wiki.getRawArticle(self.options.collectionpage)
if wikitext is None:
raise RuntimeError('No such collection page: %r' % (
self.options.collectionpage,
))
self.metabook = metabook.parse_collection_page(wikitext)
env.metabook = self.metabook
if self.options.title:
env.metabook['title'] = self.options.title
if self.options.subtitle:
env.metabook['subtitle'] = self.options.subtitle
return env
def makewiki(self):
username, password, domain = None, None, None
if self.options.login:
if self.options.login.count(':') == 1:
username, password = unicode(self.options.login, 'utf-8').split(':', 1)
else:
username, password, domain = unicode(self.options.login, 'utf-8').split(':', 2)
if self.options.script_extension:
script_extension = unicode(self.options.script_extension, 'utf-8')
else:
script_extension = None
env = wiki.makewiki(self.options.config,
metabook=self.metabook,
username=username,
password=password,
domain=domain,
script_extension=script_extension,
)
if self.options.noimages:
env.images = None
if self.options.template_blacklist:
template_blacklist = unicode(self.options.template_blacklist, 'utf-8')
else:
template_blacklist = None
if self.options.template_exclusion_category:
template_exclusion_category = unicode(self.options.template_exclusion_category, 'utf-8')
else:
template_exclusion_category = None
if self.options.print_template_pattern:
print_template_pattern = unicode(self.options.print_template_pattern, 'utf-8')
else:
print_template_pattern = None
if self.options.print_template_prefix:
if print_template_pattern is not None:
log.warn('Both --print-template-pattern and --print-template-prefix (deprecated) specified. Using --print-template-pattern only.')
else:
print_template_pattern = '%s$1' % unicode(self.options.print_template_prefix, 'utf-8')
if template_blacklist\
or template_exclusion_category\
or print_template_pattern:
if hasattr(env.wiki, 'setTemplateExclusion'):
env.wiki.setTemplateExclusion(
blacklist=template_blacklist,
category=template_exclusion_category,
pattern=print_template_pattern,
)
else:
log.warn('WikiDB does not support setting a template blacklist')
if self.options.collectionpage:
wikitext = env.wiki.getRawArticle(unicode(self.options.collectionpage, 'utf-8'))
if wikitext is None:
raise RuntimeError('No such collection page: %r' % (
self.options.collectionpage,
))
self.metabook = metabook.parse_collection_page(wikitext)
env.metabook = self.metabook
if self.options.title:
env.metabook['title'] = unicode(self.options.title, 'utf-8')
if self.options.subtitle:
env.metabook['subtitle'] = unicode(self.options.subtitle, 'utf-8')
if self.options.editor:
env.metabook['editor'] = unicode(self.options.editor, 'utf-8')
return env
