def test_showtree(self):
"""test Text.__showtree__()"""
output = []
node1 = Text("foobar")
node2 = Text("f贸贸bar")
node3 = Text("饜尣饜尶饜崉")
node1.__showtree__(output.append, None, None)
node2.__showtree__(output.append, None, None)
node3.__showtree__(output.append, None, None)
res = ["foobar", r"f\xf3\xf3bar", "\\U00010332\\U0001033f\\U00010344"]
self.assertEqual(res, output)
def test_children(self):
"""test Wikilink.__children__()"""
node1 = Wikilink(wraptext("foobar"))
node2 = Wikilink(wraptext("foo"), wrap([Text("bar"), Text("baz")]))
gen1 = node1.__children__()
gen2 = node2.__children__()
self.assertEqual(node1.title, next(gen1))
self.assertEqual(node2.title, next(gen2))
self.assertEqual(node2.text, next(gen2))
self.assertRaises(StopIteration, next, gen1)
self.assertRaises(StopIteration, next, gen2)
def test_children(self):
"""test Argument.__children__()"""
node1 = Argument(wraptext("foobar"))
node2 = Argument(wraptext("foo"), wrap([Text("bar"), Text("baz")]))
gen1 = node1.__children__()
gen2 = node2.__children__()
self.assertIs(node1.name, next(gen1))
self.assertIs(node2.name, next(gen2))
self.assertIs(node2.default, next(gen2))
self.assertRaises(StopIteration, next, gen1)
self.assertRaises(StopIteration, next, gen2)
def test_children(self):
"""test ExternalLink.__children__()"""
node1 = ExternalLink(wraptext("http://example.com/"), brackets=False)
node2 = ExternalLink(wraptext("http://example.com/"),
wrap([Text("Example"), Text("Page")]))
gen1 = node1.__children__()
gen2 = node2.__children__()
self.assertEqual(node1.url, next(gen1))
self.assertEqual(node2.url, next(gen2))
self.assertEqual(node2.title, next(gen2))
self.assertRaises(StopIteration, next, gen1)
self.assertRaises(StopIteration, next, gen2)
def test_nodes(self):
"""test getter/setter for the nodes attribute"""
code = parse("Have a {{template}}")
self.assertEqual(["Have a ", "{{template}}"], code.nodes)
L1 = SmartList([Text("foobar"), Template(wraptext("abc"))])
L2 = [Text("barfoo"), Template(wraptext("cba"))]
L3 = "abc{{def}}"
code.nodes = L1
self.assertIs(L1, code.nodes)
code.nodes = L2
self.assertIs(L2, code.nodes)
code.nodes = L3
self.assertEqual(["abc", "{{def}}"], code.nodes)
self.assertRaises(ValueError, setattr, code, "nodes", object)
def test_parse_anything_valid(self):
"""tests for valid input to utils.parse_anything()"""
tests = [
(wraptext("foobar"), wraptext("foobar")),
(Template(wraptext("spam")), wrap([Template(wraptext("spam"))])),
("fóóbar", wraptext("fóóbar")),
(b"foob\xc3\xa1r", wraptext("foobár")), (123, wraptext("123")),
(True, wraptext("True")), (None, wrap([])),
([Text("foo"), Text("bar"),
Text("baz")], wraptext("foo", "bar", "baz")),
([wraptext("foo"), Text("bar"), "baz", 123,
456], wraptext("foo", "bar", "baz", "123", "456")),
([[[([[((("foo", ), ), )], "bar"], )]]], wraptext("foo", "bar"))
]
for test, valid in tests:
self.assertWikicodeEqual(valid, parse_anything(test))
def test_insert(self):
"""test Wikicode.insert()"""
code = parse("Have a {{template}} and a [[page|link]]")
code.insert(1, "{{{argument}}}")
self.assertEqual(
"Have a {{{argument}}}{{template}} and a [[page|link]]", code)
self.assertIsInstance(code.get(1), Argument)
code.insert(2, None)
self.assertEqual(
"Have a {{{argument}}}{{template}} and a [[page|link]]", code)
code.insert(-3, Text("foo"))
self.assertEqual(
"Have a {{{argument}}}foo{{template}} and a [[page|link]]", code)
code2 = parse("{{foo}}{{bar}}{{baz}}")
code2.insert(1, "abc{{def}}ghi[[jk]]")
self.assertEqual("{{foo}}abc{{def}}ghi[[jk]]{{bar}}{{baz}}", code2)
self.assertEqual([
"{{foo}}", "abc", "{{def}}", "ghi", "[[jk]]", "{{bar}}", "{{baz}}"
], code2.nodes)
code3 = parse("{{foo}}bar")
code3.insert(1000, "[[baz]]")
code3.insert(-1000, "derp")
self.assertEqual("derp{{foo}}bar[[baz]]", code3)
def test_value(self):
"""test getter/setter for the value attribute"""
node = Text("foobar")
self.assertEqual("foobar", node.value)
self.assertIsInstance(node.value, str)
node.value = "h茅h茅h茅"
self.assertEqual("h茅h茅h茅", node.value)
self.assertIsInstance(node.value, str)
def test_children(self):
"""test Template.__children__()"""
node2p1 = Parameter(wraptext("1"), wraptext("bar"), showkey=False)
node2p2 = Parameter(wraptext("abc"), wrap([Text("def"), Text("ghi")]),
showkey=True)
node1 = Template(wraptext("foobar"))
node2 = Template(wraptext("foo"), [node2p1, node2p2])
gen1 = node1.__children__()
gen2 = node2.__children__()
self.assertEqual(node1.name, next(gen1))
self.assertEqual(node2.name, next(gen2))
self.assertEqual(node2.params[0].value, next(gen2))
self.assertEqual(node2.params[1].name, next(gen2))
self.assertEqual(node2.params[1].value, next(gen2))
self.assertRaises(StopIteration, next, gen1)
self.assertRaises(StopIteration, next, gen2)
def test_integration(self):
"""a test for building a combination of templates together"""
# {{{{{{{{foo}}bar|baz=biz}}buzz}}usr|{{bin}}}}
test = [
tokens.TemplateOpen(),
tokens.TemplateOpen(),
tokens.TemplateOpen(),
tokens.TemplateOpen(),
tokens.Text(text="foo"),
tokens.TemplateClose(),
tokens.Text(text="bar"),
tokens.TemplateParamSeparator(),
tokens.Text(text="baz"),
tokens.TemplateParamEquals(),
tokens.Text(text="biz"),
tokens.TemplateClose(),
tokens.Text(text="buzz"),
tokens.TemplateClose(),
tokens.Text(text="usr"),
tokens.TemplateParamSeparator(),
tokens.TemplateOpen(),
tokens.Text(text="bin"),
tokens.TemplateClose(),
tokens.TemplateClose()
]
valid = wrap([
Template(wrap([
Template(
wrap([
Template(wrap([Template(wraptext("foo")),
Text("bar")]),
params=[
Parameter(wraptext("baz"),
wraptext("biz"))
]),
Text("buzz")
])),
Text("usr")
]),
params=[
Parameter(wraptext("1"),
wrap([Template(wraptext("bin"))]),
showkey=False)
])
])
self.assertWikicodeEqual(valid, self.builder.build(test))
def test_closing_tag(self):
"""test getter/setter for the closing_tag attribute"""
tag = wraptext("ref")
node = Tag(tag, wraptext("foobar"))
self.assertIs(tag, node.closing_tag)
node.closing_tag = "ref {{ignore me}}"
parsed = wrap([Text("ref "), Template(wraptext("ignore me"))])
self.assertWikicodeEqual(parsed, node.closing_tag)
self.assertEqual("<ref>foobar</ref {{ignore me}}>", node)
def test_contents(self):
"""test getter/setter for the contents attribute"""
contents = wraptext("text")
node = Tag(wraptext("ref"), contents)
self.assertIs(contents, node.contents)
node.contents = "text and a {{template}}"
parsed = wrap([Text("text and a "), Template(wraptext("template"))])
self.assertWikicodeEqual(parsed, node.contents)
self.assertEqual("<ref>text and a {{template}}</ref>", node)
def test_append(self):
"""test Wikicode.append()"""
code = parse("Have a {{template}}")
code.append("{{{argument}}}")
self.assertEqual("Have a {{template}}{{{argument}}}", code)
self.assertIsInstance(code.get(2), Argument)
code.append(None)
self.assertEqual("Have a {{template}}{{{argument}}}", code)
code.append(Text(" foo"))
self.assertEqual("Have a {{template}}{{{argument}}} foo", code)
self.assertRaises(ValueError, code.append, slice(0, 1))
def test_skip_style_tags(self):
"""test Parser.parse(skip_style_tags=True)"""
def test():
with_style = parser.Parser().parse(text, skip_style_tags=False)
without_style = parser.Parser().parse(text, skip_style_tags=True)
self.assertWikicodeEqual(a, with_style)
self.assertWikicodeEqual(b, without_style)
text = "This is an example with ''italics''!"
a = wrap([Text("This is an example with "),
Tag(wraptext("i"), wraptext("italics"), wiki_markup="''"),
Text("!")])
b = wraptext("This is an example with ''italics''!")
restore = parser.use_c
if parser.use_c:
test()
parser.use_c = False
test()
parser.use_c = restore
def test_parsing(self):
"""integration test for parsing overall"""
text = "this is text; {{this|is=a|template={{with|[[links]]|in}}it}}"
expected = wrap([
Text("this is text; "),
Template(wraptext("this"), [
Parameter(wraptext("is"), wraptext("a")),
Parameter(wraptext("template"), wrap([
Template(wraptext("with"), [
Parameter(wraptext("1"),
wrap([Wikilink(wraptext("links"))]),
showkey=False),
Parameter(wraptext("2"),
wraptext("in"), showkey=False)
]),
Text("it")
]))
])
])
actual = parser.Parser().parse(text)
self.assertWikicodeEqual(expected, actual)
def handle_template(tpl, code, namespace=None):
if tpl.has('sprache'):
if tpl.get('sprache').value.strip().lower() in ('englisch', 'english'):
set_param_value(tpl, 'sprache', 'en')
if tpl.get('sprache').value.strip().lower() in ('deutsch', 'german'):
set_param_value(tpl, 'sprache', 'de')
if tpl.has('wann'):
if tpl.get('wann').value.strip() in ('Sommersemester', 'ss'):
set_param_value(tpl, 'wann', 'SS')
elif tpl.get('wann').value.strip() in ('Wintersemester', 'ws'):
set_param_value(tpl, 'wann', 'WS')
elif tpl.get('wann').value.strip() in ('Winter- und Sommersemester',
'Sommer- und Wintersemester'):
set_param_value(tpl, 'wann', 'beide')
if tpl.has('tiss'):
if tpl.get('tiss').value.strip() == '1234567890':
tpl.remove('tiss')
archived = False
successor = None
if tpl.has('veraltet'):
archived = True
tpl.remove('veraltet')
if tpl.has('nachfolger'):
archived = True
successor = tpl.get('nachfolger').value.strip()
tpl.remove('nachfolger')
for t in code.ifilter_templates(
matches=lambda t: t.name.matches('Veraltet')):
archived = True
code.remove(t)
archivedFlag = code.filter_templates(
matches=lambda t: t.name.matches('Archiv'))
if archived and not archivedFlag:
tpl = Template(Wikicode([Text('Archiv')]))
if successor:
tpl.add('nachfolger', successor)
code.insert(0, tpl)
code.insert(1, '\n\n')
if tpl.has('zuordnungen'):
rels = tpl.get('zuordnungen').value.filter_templates()
for rel in rels:
if rel.has('2'):
rel.get('2').value = str(rel.get('2').value).replace('–', '-')
rels.sort(key=lambda x: x.get('1'))
tpl.get('zuordnungen').value = '\n' + '\n'.join(
[' ' * 4 + str(r) for r in rels]) + '\n'
return 'fixe LVA-Daten'
def test_children(self):
"""test Heading.__children__()"""
node = Heading(wrap([Text("foo"), Text("bar")]), 3)
gen = node.__children__()
self.assertEqual(node.title, next(gen))
self.assertRaises(StopIteration, next, gen)
def test_strip(self):
"""test Text.__strip__()"""
node = Text("foobar")
for a in (True, False):
for b in (True, False):
self.assertIs(node, node.__strip__(a, b))
def test_children(self):
"""test Text.__children__()"""
node = Text("foobar")
gen = node.__children__()
self.assertRaises(StopIteration, next, gen)
def test_unicode(self):
"""test Text.__unicode__()"""
node = Text("foobar")
self.assertEqual("foobar", str(node))
node2 = Text("f贸贸bar")
self.assertEqual("f贸贸bar", str(node2))
def test_strip(self):
"""test Text.__strip__()"""
node = Text("foobar")
self.assertIs(node, node.__strip__())
from __future__ import unicode_literals
try:
from unittest2 import TestCase
except ImportError:
from unittest import TestCase
from mwparserfromhell.compat import range
from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,
Tag, Template, Text, Wikilink)
from mwparserfromhell.nodes.extras import Attribute, Parameter
from mwparserfromhell.smart_list import SmartList
from mwparserfromhell.wikicode import Wikicode
wrap = lambda L: Wikicode(SmartList(L))
wraptext = lambda *args: wrap([Text(t) for t in args])
class TreeEqualityTestCase(TestCase):
"""A base test case with support for comparing the equality of node trees.
This adds a number of type equality functions, for Wikicode, Text,
Templates, and Wikilinks.
"""
def assertNodeEqual(self, expected, actual):
"""Assert that two Nodes have the same type and have the same data."""
registry = {
Argument: self.assertArgumentNodeEqual,
Comment: self.assertCommentNodeEqual,
Heading: self.assertHeadingNodeEqual,
HTMLEntity: self.assertHTMLEntityNodeEqual,
def wikify(x):
# Convert a string from the Geograph database into Wikicode.
x = re.sub(r"&#(\d+);", lambda m: chr(int(m.group(1))), x)
if ("''" in x or '}}' in x or '|' in x or '[[' in x):
return Tag('nowiki', Text(x))
return Text(x)
def test_tag(self):
"""tests for building Tag nodes"""
tests = [
# <ref></ref>
([
tokens.TagOpenOpen(),
tokens.Text(text="ref"),
tokens.TagCloseOpen(padding=""),
tokens.TagOpenClose(),
tokens.Text(text="ref"),
tokens.TagCloseClose()
],
wrap(
[Tag(wraptext("ref"), wrap([]),
closing_tag=wraptext("ref"))])),
# <ref name></ref>
([
tokens.TagOpenOpen(),
tokens.Text(text="ref"),
tokens.TagAttrStart(pad_first=" ",
pad_before_eq="",
pad_after_eq=""),
tokens.Text(text="name"),
tokens.TagCloseOpen(padding=""),
tokens.TagOpenClose(),
tokens.Text(text="ref"),
tokens.TagCloseClose()
],
wrap([
Tag(wraptext("ref"),
wrap([]),
attrs=[Attribute(wraptext("name"))])
])),
# <ref name="abc" />
([
tokens.TagOpenOpen(),
tokens.Text(text="ref"),
tokens.TagAttrStart(pad_first=" ",
pad_before_eq="",
pad_after_eq=""),
tokens.Text(text="name"),
tokens.TagAttrEquals(),
tokens.TagAttrQuote(char='"'),
tokens.Text(text="abc"),
tokens.TagCloseSelfclose(padding=" ")
],
wrap([
Tag(wraptext("ref"),
attrs=[Attribute(wraptext("name"), wraptext("abc"))],
self_closing=True,
padding=" ")
])),
# <br/>
([
tokens.TagOpenOpen(),
tokens.Text(text="br"),
tokens.TagCloseSelfclose(padding="")
], wrap([Tag(wraptext("br"), self_closing=True)])),
# <li>
([
tokens.TagOpenOpen(),
tokens.Text(text="li"),
tokens.TagCloseSelfclose(padding="", implicit=True)
], wrap([Tag(wraptext("li"), self_closing=True, implicit=True)])),
# </br>
([
tokens.TagOpenOpen(invalid=True),
tokens.Text(text="br"),
tokens.TagCloseSelfclose(padding="", implicit=True)
],
wrap([
Tag(wraptext("br"),
self_closing=True,
invalid=True,
implicit=True)
])),
# </br/>
([
tokens.TagOpenOpen(invalid=True),
tokens.Text(text="br"),
tokens.TagCloseSelfclose(padding="")
], wrap([Tag(wraptext("br"), self_closing=True, invalid=True)])),
# <ref name={{abc}} foo="bar {{baz}}" abc={{de}}f ghi=j{{k}}{{l}}
# mno = '{{p}} [[q]] {{r}}'>[[Source]]</ref>
([
tokens.TagOpenOpen(),
tokens.Text(text="ref"),
tokens.TagAttrStart(pad_first=" ",
pad_before_eq="",
pad_after_eq=""),
tokens.Text(text="name"),
tokens.TagAttrEquals(),
tokens.TemplateOpen(),
tokens.Text(text="abc"),
tokens.TemplateClose(),
tokens.TagAttrStart(pad_first=" ",
pad_before_eq="",
pad_after_eq=""),
tokens.Text(text="foo"),
tokens.TagAttrEquals(),
tokens.TagAttrQuote(char='"'),
tokens.Text(text="bar "),
tokens.TemplateOpen(),
tokens.Text(text="baz"),
tokens.TemplateClose(),
tokens.TagAttrStart(pad_first=" ",
pad_before_eq="",
pad_after_eq=""),
tokens.Text(text="abc"),
tokens.TagAttrEquals(),
tokens.TemplateOpen(),
tokens.Text(text="de"),
tokens.TemplateClose(),
tokens.Text(text="f"),
tokens.TagAttrStart(pad_first=" ",
pad_before_eq="",
pad_after_eq=""),
tokens.Text(text="ghi"),
tokens.TagAttrEquals(),
tokens.Text(text="j"),
tokens.TemplateOpen(),
tokens.Text(text="k"),
tokens.TemplateClose(),
tokens.TemplateOpen(),
tokens.Text(text="l"),
tokens.TemplateClose(),
tokens.TagAttrStart(pad_first=" \n ",
pad_before_eq=" ",
pad_after_eq=" "),
tokens.Text(text="mno"),
tokens.TagAttrEquals(),
tokens.TagAttrQuote(char="'"),
tokens.TemplateOpen(),
tokens.Text(text="p"),
tokens.TemplateClose(),
tokens.Text(text=" "),
tokens.WikilinkOpen(),
tokens.Text(text="q"),
tokens.WikilinkClose(),
tokens.Text(text=" "),
tokens.TemplateOpen(),
tokens.Text(text="r"),
tokens.TemplateClose(),
tokens.TagCloseOpen(padding=""),
tokens.WikilinkOpen(),
tokens.Text(text="Source"),
tokens.WikilinkClose(),
tokens.TagOpenClose(),
tokens.Text(text="ref"),
tokens.TagCloseClose()
],
wrap([
Tag(wraptext("ref"), wrap([Wikilink(wraptext("Source"))]), [
Attribute(wraptext("name"),
wrap([Template(wraptext("abc"))]), None),
Attribute(wraptext("foo"),
wrap([Text("bar "),
Template(wraptext("baz"))]),
pad_first=" "),
Attribute(wraptext("abc"),
wrap([Template(wraptext("de")),
Text("f")]), None),
Attribute(
wraptext("ghi"),
wrap([
Text("j"),
Template(wraptext("k")),
Template(wraptext("l"))
]), None),
Attribute(
wraptext("mno"),
wrap([
Template(wraptext("p")),
Text(" "),
Wikilink(wraptext("q")),
Text(" "),
Template(wraptext("r"))
]), "'", " \n ", " ", " ")
])
])),
# "''italic text''"
([
tokens.TagOpenOpen(wiki_markup="''"),
tokens.Text(text="i"),
tokens.TagCloseOpen(),
tokens.Text(text="italic text"),
tokens.TagOpenClose(),
tokens.Text(text="i"),
tokens.TagCloseClose()
],
wrap([
Tag(wraptext("i"), wraptext("italic text"), wiki_markup="''")
])),
# * bullet
([
tokens.TagOpenOpen(wiki_markup="*"),
tokens.Text(text="li"),
tokens.TagCloseSelfclose(),
tokens.Text(text=" bullet")
],
wrap([
Tag(wraptext("li"), wiki_markup="*", self_closing=True),
Text(" bullet")
])),
]
for test, valid in tests:
self.assertWikicodeEqual(valid, self.builder.build(test))
def handle(site, index):
src_ns = next(site.results(prop='info', titles=index))['ns']
for page in site.results(generator='allpages',
gapprefix=index.split(':')[1] + '/Beispiel ',
gaplimit='max',
prop='revisions',
rvprop='content',
gapnamespace=src_ns):
orig = page['revisions'][0]['*']
if mwbot.parse_redirect(orig):
continue
code = mwbot.parse(orig)
templates = code.filter_templates(
matches=lambda x: x.name.matches('Beispiel'))
if len(templates) > 0:
template = templates[0]
else:
template = Template(Wikicode([Text('Beispiel')]))
code.insert(0, template)
code.insert(1, '\n')
# legacy format handling
template.name = 'Beispiel'
if template.has('1') and not template.get('1').value.startswith('\n'):
template.get('1').value.insert(0, '\n')
if template.has('status'):
if str(template.get('status').value).strip() not in ('extern',
'Datei'):
print('unknown status: {}'.format(k))
if template.has('1'):
if str(template.get('1').value).strip() == 'teils':
template.add('teils', '')
template.remove('1')
elif str(template.get('1').value).strip() == 'falsch':
template.add('falsch', '')
template.remove('1')
if not template.has('1'):
angabe_div = code.filter_tags(
matches=lambda x: x.tag.matches('blockquote') or len([
x for x in x.attributes
if '{{Angabe}}' in x or '#EFEFEF' in x
]))
if angabe_div:
template.add('1',
'\n' + str(angabe_div[0].contents).strip() + '\n',
showkey=True)
code.remove(angabe_div[0])
else:
angabe_sec = code.get_sections(matches='Angabe|Aufgabe')
if angabe_sec:
code.remove(angabe_sec[0].nodes[0])
template.add('1',
'\n' + str(angabe_sec[0]).strip() + '\n',
showkey=True)
code.replace(angabe_sec[0], '\n')
mwbot.save(site,
page['title'],
orig,
str(code),
'beispiel_fixer.py',
strip_consec_nl=True)
