def test_integration2(self): """an even more audacious test for building a horrible wikicode mess""" # {{a|b|{{c|[[d]]{{{e}}}}}}}[[f|{{{g}}}<!--h-->]]{{i|j= }} test = [tokens.TemplateOpen(), tokens.Text(text="a"), tokens.TemplateParamSeparator(), tokens.Text(text="b"), tokens.TemplateParamSeparator(), tokens.TemplateOpen(), tokens.Text(text="c"), tokens.TemplateParamSeparator(), tokens.WikilinkOpen(), tokens.Text(text="d"), tokens.WikilinkClose(), tokens.ArgumentOpen(), tokens.Text(text="e"), tokens.ArgumentClose(), tokens.TemplateClose(), tokens.TemplateClose(), tokens.WikilinkOpen(), tokens.Text(text="f"), tokens.WikilinkSeparator(), tokens.ArgumentOpen(), tokens.Text(text="g"), tokens.ArgumentClose(), tokens.CommentStart(), tokens.Text(text="h"), tokens.CommentEnd(), tokens.WikilinkClose(), tokens.TemplateOpen(), tokens.Text(text="i"), tokens.TemplateParamSeparator(), tokens.Text(text="j"), tokens.TemplateParamEquals(), tokens.HTMLEntityStart(), tokens.Text(text="nbsp"), tokens.HTMLEntityEnd(), tokens.TemplateClose()] valid = wrap( [Template(wraptext("a"), params=[Parameter(wraptext("1"), wraptext( "b"), showkey=False), Parameter(wraptext("2"), wrap([Template( wraptext("c"), params=[Parameter(wraptext("1"), wrap([Wikilink( wraptext("d")), Argument(wraptext("e"))]), showkey=False)])]), showkey=False)]), Wikilink(wraptext("f"), wrap([Argument(wraptext( "g")), Comment("h")])), Template(wraptext("i"), params=[ Parameter(wraptext("j"), wrap([HTMLEntity("nbsp", named=True)]))])]) self.assertWikicodeEqual(valid, self.builder.build(test))
def test_value(self): """test getter/setter for the value attribute""" value = wraptext("bar") node = Parameter(wraptext("foo"), value) self.assertIs(value, node.value) node.value = "héhehé" self.assertWikicodeEqual(wraptext("héhehé"), node.value)
def test_showkey(self): """test getter/setter for the showkey attribute""" node1 = Parameter(wraptext("1"), wraptext("foo"), showkey=False) node2 = Parameter(wraptext("foo"), wraptext("bar")) self.assertFalse(node1.showkey) self.assertTrue(node2.showkey) node1.showkey = True node2.showkey = "" self.assertTrue(node1.showkey) self.assertFalse(node2.showkey)
def test_showkey(self): """test getter/setter for the showkey attribute""" node1 = Parameter(wraptext("1"), wraptext("foo"), showkey=False) node2 = Parameter(wraptext("foo"), wraptext("bar")) self.assertFalse(node1.showkey) self.assertTrue(node2.showkey) node1.showkey = True self.assertTrue(node1.showkey) node1.showkey = "" self.assertFalse(node1.showkey) self.assertRaises(ValueError, setattr, node2, "showkey", False)
def test_name(self): """test getter/setter for the name attribute""" name1 = wraptext("1") name2 = wraptext("foobar") node1 = Parameter(name1, wraptext("foobar"), showkey=False) node2 = Parameter(name2, wraptext("baz")) self.assertIs(name1, node1.name) self.assertIs(name2, node2.name) node1.name = "héhehé" node2.name = "héhehé" self.assertWikicodeEqual(wraptext("héhehé"), node1.name) self.assertWikicodeEqual(wraptext("héhehé"), node2.name)
def test_template(self): """tests for building Template nodes""" tests = [ ([tokens.TemplateOpen(), tokens.Text(text="foobar"), tokens.TemplateClose()], wrap([Template(wraptext("foobar"))])), ([tokens.TemplateOpen(), tokens.Text(text="spam"), tokens.Text(text="eggs"), tokens.TemplateClose()], wrap([Template(wraptext("spam", "eggs"))])), ([tokens.TemplateOpen(), tokens.Text(text="foo"), tokens.TemplateParamSeparator(), tokens.Text(text="bar"), tokens.TemplateClose()], wrap([Template(wraptext("foo"), params=[ Parameter(wraptext("1"), wraptext("bar"), showkey=False)])])), ([tokens.TemplateOpen(), tokens.Text(text="foo"), tokens.TemplateParamSeparator(), tokens.Text(text="bar"), tokens.TemplateParamEquals(), tokens.Text(text="baz"), tokens.TemplateClose()], wrap([Template(wraptext("foo"), params=[ Parameter(wraptext("bar"), wraptext("baz"))])])), ([tokens.TemplateOpen(), tokens.TemplateParamSeparator(), tokens.TemplateParamSeparator(), tokens.TemplateParamEquals(), tokens.TemplateParamSeparator(), tokens.TemplateClose()], wrap([Template(wrap([]), params=[ Parameter(wraptext("1"), wrap([]), showkey=False), Parameter(wrap([]), wrap([]), showkey=True), Parameter(wraptext("2"), wrap([]), showkey=False)])])), ([tokens.TemplateOpen(), tokens.Text(text="foo"), tokens.TemplateParamSeparator(), tokens.Text(text="bar"), tokens.TemplateParamEquals(), tokens.Text(text="baz"), tokens.TemplateParamSeparator(), tokens.Text(text="biz"), tokens.TemplateParamSeparator(), tokens.Text(text="buzz"), tokens.TemplateParamSeparator(), tokens.Text(text="3"), tokens.TemplateParamEquals(), tokens.Text(text="buff"), tokens.TemplateParamSeparator(), tokens.Text(text="baff"), tokens.TemplateClose()], wrap([Template(wraptext("foo"), params=[ Parameter(wraptext("bar"), wraptext("baz")), Parameter(wraptext("1"), wraptext("biz"), showkey=False), Parameter(wraptext("2"), wraptext("buzz"), showkey=False), Parameter(wraptext("3"), wraptext("buff")), Parameter(wraptext("3"), wraptext("baff"), showkey=False)])])), ] for test, valid in tests: self.assertWikicodeEqual(valid, self.builder.build(test))
def combine_template_chains(wc: Wikicode, new_template_name: str, template_indices: List[int], text_indices: List[int]) -> None: """ Helper function for combining templates that are linked via free text into a structured template hierarchy. """ index_combos = [] index_combo = [] combine = False for i in template_indices: if (i + 1 in text_indices) or (i - 2 in index_combo and combine): index_combo.append(i) combine = i + 1 in text_indices if not combine: if len(index_combo) > 1: index_combos.append(index_combo) index_combo = [] if len(index_combo) > 1: index_combos.append(index_combo) combo_nodes = [[wc.nodes[i] for i in chain] for chain in index_combos] for combo in combo_nodes: params = [ Parameter(str(i + 1), t, showkey=False) for i, t in enumerate(combo) ] new_template = Template(new_template_name, params=params) wc.insert_before(combo[0], new_template, recursive=False) for node in combo: wc.remove(node, recursive=False)
def backup_template(self, template: Template, page: Union[str, Page], key: Union[str, List[str]]): """ Backs up a template in the `Backup` namespace. The template can later be restored with `get_restored_template`. :param template: Template object :param page: Page or title where the template is located on :param key: Identifying set of params that we can use to locate the template when we restore it :return: null """ if isinstance(page, str): page = self.client.pages[page] if isinstance(key, str): key = [key] key_template = Template('BackupKey') for key_param in key: key_template.add(key_param, template.get(key_param, Parameter('', '')).value) # this method will be used in TemplateModifier so it is essential that # we do not modify the original copy_template = copy.deepcopy(template) copy_template.add('backup_key', str(key_template)) self.client.pages['Backup:' + page.name].append('\n' + str(copy_template), contentmodel='text')
def create_wikitext(self, vals): wikitext = '' empty_pages = [] first_page = '' for ind, val in vals.items(): t_name, = val[1] if t_name != self.template_name: raise ValueError( f"Unexpected template name {t_name} instead of {self.template_name}" ) t_params = val[1][t_name] t_params2 = { k: t_params[k] for k in t_params if k not in self.ignore_params } if t_params: if not first_page: first_page = val[0] wikitext += f"* _INDEX_={ind}\n" wikitext += str( Template(t_name, params=[ Parameter(k, v) for k, v in t_params2.items() ])) wikitext += '\n' else: empty_pages.append( PageContent(title=val[0], timestamp=datetime.utcnow(), data={})) if wikitext: wikitext += f"* _INDEX_=END\n" return empty_pages, wikitext, first_page
def test_children(self): """test Template.__children__()""" node2p1 = Parameter(wraptext("1"), wraptext("bar"), showkey=False) node2p2 = Parameter(wraptext("abc"), wrap([Text("def"), Text("ghi")]), showkey=True) node1 = Template(wraptext("foobar")) node2 = Template(wraptext("foo"), [node2p1, node2p2]) gen1 = node1.__children__() gen2 = node2.__children__() self.assertEqual(node1.name, next(gen1)) self.assertEqual(node2.name, next(gen2)) self.assertEqual(node2.params[0].value, next(gen2)) self.assertEqual(node2.params[1].name, next(gen2)) self.assertEqual(node2.params[1].value, next(gen2)) self.assertRaises(StopIteration, next, gen1) self.assertRaises(StopIteration, next, gen2)
def test_integration(self): """a test for building a combination of templates together""" # {{{{{{{{foo}}bar|baz=biz}}buzz}}usr|{{bin}}}} test = [ tokens.TemplateOpen(), tokens.TemplateOpen(), tokens.TemplateOpen(), tokens.TemplateOpen(), tokens.Text(text="foo"), tokens.TemplateClose(), tokens.Text(text="bar"), tokens.TemplateParamSeparator(), tokens.Text(text="baz"), tokens.TemplateParamEquals(), tokens.Text(text="biz"), tokens.TemplateClose(), tokens.Text(text="buzz"), tokens.TemplateClose(), tokens.Text(text="usr"), tokens.TemplateParamSeparator(), tokens.TemplateOpen(), tokens.Text(text="bin"), tokens.TemplateClose(), tokens.TemplateClose() ] valid = wrap([ Template(wrap([ Template( wrap([ Template(wrap([Template(wraptext("foo")), Text("bar")]), params=[ Parameter(wraptext("baz"), wraptext("biz")) ]), Text("buzz") ])), Text("usr") ]), params=[ Parameter(wraptext("1"), wrap([Template(wraptext("bin"))]), showkey=False) ]) ]) self.assertWikicodeEqual(valid, self.builder.build(test))
def set(self, key, value, showkey=True): # Check for uniqueness. # Wikicode is not hashable, so we instead use the stringified version # as the key and include the Wikicode <parameter name> within the object. skey = str(key) if not self.is_supported(skey): raise ValueError("Not supported: parameter %s" % (skey, )) if skey not in self._parameters_map: o = Parameter(key, value, showkey) self._parameters.append(o) # Needed for consistency - allow future lookups of this parameter. self._parameters_map[skey] = o else: # Overwrite existing parameter. o = self._parameters_map[skey] # Don't allow changing parameter names - creates a consistency problem. o.value = value o.showkey = showkey
def merge_etyl_templates(wc: Wikicode) -> Wikicode: """ Given a chunk of wikicode, finds instances where the deprecated `etyl` template is immediately followed by either a word in free text, a linked word, or a generic `mention`/`link`/`langname-mention` template. It replaces this pattern with a new `derived-parsed` template -- meaning the same thing as the `derived` template but namespaced to differentiate. For cases where the `mention` language is different from the `etyl` language, we use the former. The template is removed if we can't parse it effectively. """ etyl_indices = [ i for i, node in enumerate(wc.nodes) if isinstance(node, Template) and node.name == "etyl" and i < len(wc.nodes) - 1 ] nodes_to_remove = [] for i in etyl_indices: make_new_template = False etyl: Template = wc.nodes[i] related_language = etyl.params[0] if len(etyl.params) == 1: language = "en" else: language = etyl.params[1] node = wc.nodes[i + 1] if isinstance(node, Text): val = re.split(",| |", node.value.strip())[0] if val: make_new_template = True elif isinstance(node, Wikilink): val = node.text or node.title val = re.split(",| |", val.strip())[0] if val: make_new_template = True elif isinstance(node, Template): if node.name in ("m", "mention", "m+", "langname-mention", "l", "link"): related_language = node.params[0] if len(node.params) > 1: val = node.params[1].value make_new_template = True nodes_to_remove.append(node) if make_new_template: params = [ Parameter(str(i + 1), str(param), showkey=False) for i, param in enumerate([language, related_language, val]) ] new_template = Template("derived-parsed", params=params) wc.replace(etyl, new_template, recursive=False) else: nodes_to_remove.append(etyl) for node in nodes_to_remove: wc.remove(node, recursive=False) return wc
def test_parsing(self): """integration test for parsing overall""" text = "this is text; {{this|is=a|template={{with|[[links]]|in}}it}}" expected = wrap([ Text("this is text; "), Template(wraptext("this"), [ Parameter(wraptext("is"), wraptext("a")), Parameter(wraptext("template"), wrap([ Template(wraptext("with"), [ Parameter(wraptext("1"), wrap([Wikilink(wraptext("links"))]), showkey=False), Parameter(wraptext("2"), wraptext("in"), showkey=False) ]), Text("it") ])) ]) ]) actual = parser.Parser().parse(text) self.assertWikicodeEqual(expected, actual)
def get_restored_template( self, template: Template, page: Union[str, Page], key: Union[str, List[str]]) -> Optional[Template]: """ Looks for the backed-up version of the specified template on the backup page & returns it The template should have been backed up using the backup_template method earlier. :param template: Template object that we want to restore from backup page :param page: Page or title where the template is located on :param key: Identifying set of params to use to restore the template from :return: Template object, if found, else None """ if isinstance(page, str): page = self.client.pages[page] if isinstance(key, str): key = [key] backup_text = self.client.pages['Backup:' + page.name].text() for backup_template in mwparserfromhell.parse( backup_text).filter_templates(): if not backup_template.name.matches(template.name): continue # kinda need to do a hack to get this as a template backup_key_str = str(backup_template.get('backup_key').value) backup_key_wikitext = mwparserfromhell.parse(backup_key_str) backup_key = None for tl in backup_key_wikitext.filter_templates(): if tl.name.matches('BackupKey'): backup_key = tl break # now backup_key is a template value of BackupKey is_match = True i = 0 for param in backup_key.params: name = param.name.strip() if name in key: if param.value.strip() == template.get( name, Parameter('', '')).value.strip(): i += 1 else: is_match = False if i == len(key) and is_match: return backup_template return None
def test_strip(self): """test Template.__strip__()""" node1 = Template(wraptext("foobar")) node2 = Template( wraptext("foo"), [pgenh("1", "bar"), pgens("foo", ""), pgens("abc", "def")]) node3 = Template(wraptext("foo"), [ pgenh("1", "foo"), Parameter(wraptext("2"), wrap([Template(wraptext("hello"))]), showkey=False), pgenh("3", "bar") ]) self.assertEqual(None, node1.__strip__(keep_template_params=False)) self.assertEqual(None, node2.__strip__(keep_template_params=False)) self.assertEqual("", node1.__strip__(keep_template_params=True)) self.assertEqual("bar def", node2.__strip__(keep_template_params=True)) self.assertEqual("foo bar", node3.__strip__(keep_template_params=True))
def otherfields_from_row(row): return Parameter("other fields", creditline_from_row(row))
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from __future__ import unicode_literals try: import unittest2 as unittest except ImportError: import unittest from mwparserfromhell.compat import str from mwparserfromhell.nodes import HTMLEntity, Template, Text from mwparserfromhell.nodes.extras import Parameter from ._test_tree_equality import TreeEqualityTestCase, wrap, wraptext pgens = lambda k, v: Parameter(wraptext(k), wraptext(v), showkey=True) pgenh = lambda k, v: Parameter(wraptext(k), wraptext(v), showkey=False) class TestTemplate(TreeEqualityTestCase): """Test cases for the Template node.""" def test_unicode(self): """test Template.__unicode__()""" node = Template(wraptext("foobar")) self.assertEqual("{{foobar}}", str(node)) node2 = Template( wraptext("foo"), [pgenh("1", "bar"), pgens("abc", "def")]) self.assertEqual("{{foo|bar|abc=def}}", str(node2)) def test_children(self):
def test_unicode(self): """test Parameter.__unicode__()""" node = Parameter(wraptext("1"), wraptext("foo"), showkey=False) self.assertEqual("foo", str(node)) node2 = Parameter(wraptext("foo"), wraptext("bar")) self.assertEqual("foo=bar", str(node2))