def test_token_iter_newline_lbox(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("'a")) new2 = TextNode(Terminal("\r")) new3 = TextNode(Terminal("b")) new4 = TextNode(MagicTerminal("<SQL>")) new5 = TextNode(Terminal("c'")) bos.insert_after(new1) new1.insert_after(new2) new2.insert_after(new3) new3.insert_after(new4) new4.insert_after(new5) it = self.lexer.get_token_iter(new1) assert it.next() == (["'a", "\r", "b", lbph, "c'"], "string", 1, [ TextNode(Terminal("'a")), TextNode(Terminal("\r")), TextNode(Terminal("b")), TextNode(MagicTerminal("<SQL>")), TextNode(Terminal("c'")) ]) with pytest.raises(StopIteration): it.next()
def test_relex_stop(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1+2")) old1 = TextNode(Terminal("*")) old2 = TextNode(Terminal("3")) old2.lookup = "INT" bos.insert_after(new) new.insert_after(old1) old1.insert_after(old2) self.relex(new) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("2") node = node.next_term assert node.symbol == Terminal("*") node = node.next_term assert node.symbol == Terminal("3") node = node.next_term assert isinstance(node, EOS)
def test_stringwrapper(self): ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("abc")) text2 = TextNode(Terminal("+")) text3 = TextNode(Terminal("1")) text4 = TextNode(Terminal("*")) text5 = TextNode(Terminal("3456")) bos.insert_after(text1) text1.insert_after(text2) text2.insert_after(text3) text3.insert_after(text4) text4.insert_after(text5) wrapper = StringWrapper(text1) assert wrapper[0] == "a" assert wrapper[2] == "c" assert wrapper[3] == "+" assert wrapper[4] == "1" assert wrapper[5] == "*" assert wrapper[6] == "3" assert wrapper[9] == "6" s = "abc+1*3456" for i in range(len(s)): for j in range(len(s)): assert wrapper[i:j] == s[i:j] print(i, j, wrapper[i:j])
def test_relex3(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("1+2")) new2 = TextNode(Terminal("345")) new3 = TextNode(Terminal("6+")) new4 = TextNode(Terminal("789")) # this should never be touched new5 = TextNode(Terminal("+")) # this should never be touched bos.insert_after(new1) new1.insert_after(new2) new2.insert_after(new3) new3.insert_after(new4) new4.insert_after(new5) self.relex(new1) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term; assert node.symbol == Terminal("1") node = node.next_term; assert node.symbol == Terminal("+") node = node.next_term; assert node.symbol == Terminal("23456") node = node.next_term; assert node.symbol == Terminal("+") # check that 789 hasn't been relexed assert node.next_term is new4 assert node.next_term.symbol is new4.symbol
def test_relex_altered_string(self): lexer = IncrementalLexer(""" "\"[^\"]*\"":str "[0-9]+":INT "\+":PLUS """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("123+\"\"")) bos.insert_after(text1) lexer.relex(text1) assert bos.next_term.symbol == Terminal("123") assert bos.next_term.lookup == "INT" assert bos.next_term.lookahead == 1 assert bos.next_term.next_term.symbol == Terminal("+") assert bos.next_term.next_term.lookup == "PLUS" assert bos.next_term.next_term.lookahead == 0 assert bos.next_term.next_term.next_term.symbol == Terminal("\"\"") assert bos.next_term.next_term.next_term.lookup == "str" assert bos.next_term.next_term.next_term.lookahead == 0 string = bos.next_term.next_term.next_term string.symbol.name = "\"abc\"" lexer.relex(string)
def test_triplequotes1(self): lexer = IncrementalLexer(""" "\"\"\"[^\"]*\"\"\"":triplestring "\"[^\"]*\"":string "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("\"\"\"")) text2 = TextNode(Terminal("abc")) text3 = TextNode(Terminal("\"\"\"")) bos.insert_after(text1) text1.insert_after(text2) text2.insert_after(text3) lexer.relex(text1) assert bos.next_term.symbol == Terminal("\"\"\"abc\"\"\"") assert bos.next_term.lookup == "triplestring" bos.next_term.symbol.name = "\"\"\"ab\"\"\"c\"\"\"" pytest.raises(LexingError, lexer.relex, bos.next_term) bos.next_term.symbol.name = "\"\"\"ab\"\"\"c\"\"" lexer.relex(bos.next_term)
def test_multitoken_real_lbox_relex(self): lexer = IncrementalLexer(""" "\"[^\"]*\"":str """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("\"abc")) lbox = TextNode(MagicTerminal("<SQL>")) text2 = TextNode(Terminal("def\"")) bos.insert_after(text1) text1.insert_after(lbox) lbox.insert_after(text2) lexer.relex(text1) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), MagicTerminal("<SQL>"), Terminal("def\"")]) bos.next_term.children[0].symbol.name = "\"ab\rc" lexer.relex(bos.next_term) assert bos.next_term == mk_multitextnode([ Terminal("\"ab"), Terminal("\r"), Terminal("c"), MagicTerminal("<SQL>"), Terminal("def\"") ])
def test_multitoken_real_lbox_relex_cut_off_string(self): lexer = IncrementalLexer(""" "\"[^\"]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("\"abc")) lbox = TextNode(MagicTerminal("<SQL>")) text2 = TextNode(Terminal("def\"")) bos.insert_after(text1) text1.insert_after(lbox) lbox.insert_after(text2) lexer.relex(text1) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), MagicTerminal("<SQL>"), Terminal("def\"")]) assert bos.next_term.lookahead == 0 bos.next_term.children[2].symbol.name = "d\"ef\"" pytest.raises(LexingError, lexer.relex, bos.next_term) bos.next_term.children[2].symbol.name = "d\"ef" lexer.relex(bos.next_term) assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), MagicTerminal("<SQL>"), Terminal("d\"")]) assert bos.next_term.next_term.symbol.name == "ef"
def test_multitoken_real_lbox_multiple(self): lexer = IncrementalLexer(""" "\"[^\"]*\"":str """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] n1 = TextNode(Terminal("\"abc")) n2 = TextNode(MagicTerminal("<SQL>")) n3 = TextNode(Terminal("def")) n4 = TextNode(MagicTerminal("<Calc>")) n5 = TextNode(Terminal("ghi\"")) bos.insert_after(n1) n1.insert_after(n2) n2.insert_after(n3) n3.insert_after(n4) n4.insert_after(n5) lexer.relex(n1) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode([ Terminal("\"abc"), MagicTerminal("<SQL>"), Terminal("def"), MagicTerminal("<Calc>"), Terminal("ghi\"") ])
def test_multitoken_real_lbox_cut_off_string(self): lexer = IncrementalLexer(""" "\"[^\"]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("\"abc")) lbox = TextNode(MagicTerminal("<SQL>")) text2 = TextNode(Terminal("d\"ef\"g")) bos.insert_after(text1) text1.insert_after(lbox) lbox.insert_after(text2) pytest.raises(LexingError, lexer.relex, text1) assert type(bos.next_term) is MultiTextNode assert bos.next_term.children[0] is text1 assert bos.next_term.children[1] is lbox assert bos.next_term.children[2] is text2 assert text2.symbol.name == "d\"" assert bos.next_term.next_term.symbol.name == "ef" leftover = bos.next_term.next_term.next_term assert leftover.symbol.name == "\"g" leftover.symbol.name = "g" leftover.changed = True lexer.relex(leftover) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), MagicTerminal("<SQL>"), Terminal("d\"")]) assert bos.next_term.next_term.symbol.name == "efg"
def test_multi(self): ast = AST() ast.init() new = TextNode(Terminal('--[[test\rtest]]')) ast.parent.children[0].insert_after(new) it = self.lexer.get_token_iter(new) assert next(it) == (['--[[test', '\r', 'test]]'], "mcomment", 0, [TextNode(Terminal('--[[test\rtest]]'))], 0)
def test_stringwrapper(self): ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("abc")) text2 = TextNode(Terminal("+")) text3 = TextNode(Terminal("1")) text4 = TextNode(Terminal("*")) text5 = TextNode(Terminal("3456")) bos.insert_after(text1) text1.insert_after(text2) text2.insert_after(text3) text3.insert_after(text4) text4.insert_after(text5) wrapper = StringWrapper(text1) assert wrapper[0] == "a" assert wrapper[2] == "c" assert wrapper[3] == "+" assert wrapper[4] == "1" assert wrapper[5] == "*" assert wrapper[6] == "3" assert wrapper[9] == "6" s = "abc+1*3456" for i in range(len(s)): for j in range(len(s)): assert wrapper[i:j] == s[i:j] print(i,j,wrapper[i:j])
def test_lookahead(self): ast = AST() ast.init() new = TextNode(Terminal('--[[test\rtest')) ast.parent.children[0].insert_after(new) it = self.lexer.get_token_iter(new) assert next(it) == ('--[[test', "scomment", 6, [TextNode(Terminal('--[[test\rtest'))], -5)
def test_simple(self): ast = AST() ast.init() new = TextNode(Terminal('--[[testtest]]')) ast.parent.children[0].insert_after(new) it = self.lexer.get_token_iter(new) assert next(it) == ('--[[testtest]]', "mcomment", 0, [TextNode(Terminal('--[[testtest]]'))], 0)
def test_simple(self): ast = AST() ast.init() new = TextNode(Terminal("asd")) ast.parent.children[0].insert_after(new) it = self.lexer.get_token_iter(new) assert next(it) == ("asd", "NAME", 1, [TextNode(Terminal("asd"))], 0)
def test_relex(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1 + 2 * 3")) bos.insert_after(new) self.relex(new) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") assert node.lookahead == 1 node = node.next_term assert node.symbol == Terminal(" ") assert node.lookahead == 1 node = node.next_term assert node.symbol == Terminal("+") assert node.lookahead == 0 node = node.next_term assert node.symbol == Terminal(" ") node = node.next_term assert node.symbol == Terminal("2") node = node.next_term assert node.symbol == Terminal(" ") node = node.next_term assert node.symbol == Terminal("*") node = node.next_term assert node.symbol == Terminal(" ") node = node.next_term assert node.symbol == Terminal("3") node = node.next_term assert isinstance(node, EOS)
def test_backwards_lexing(self): lexer = IncrementalLexer(""" "::=":doublecolon "=":equal ":":singlecolon """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal(":")) bos.insert_after(text) lexer.relex(text) assert bos.next_term.symbol.name == ":" assert bos.next_term.lookup == "singlecolon" assert text.lookahead == 1 text2 = TextNode(Terminal(":")) text.insert_after(text2) lexer.relex(text2) assert text2.lookahead == 1 assert bos.next_term.symbol.name == ":" assert bos.next_term.next_term.symbol.name == ":" text3 = TextNode(Terminal("=")) text2.insert_after(text3) lexer.relex(text3) assert bos.next_term.symbol.name == "::=" assert isinstance(bos.next_term.next_term, EOS)
def test_multitoken_relex_merge(self): lexer = IncrementalLexer(""" "\"[a-z\r\x80]*\"":str """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal("\"abc\rde\rf\"")) bos.insert_after(text) lexer.relex(text) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode([ Terminal("\"abc"), Terminal("\r"), Terminal("de"), Terminal("\r"), Terminal("f\"") ]) bos.next_term.children.pop(3) # remove a newline bos.next_term.update_children() child0 = bos.next_term.children[0] child1 = bos.next_term.children[1] child2 = bos.next_term.children[2] lexer.relex(bos.next_term) assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), Terminal("\r"), Terminal("def\"")]) assert bos.next_term.children[0] is child0 assert bos.next_term.children[1] is child1 assert bos.next_term.children[2] is child2
def test_relex3(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("1+2")) new2 = TextNode(Terminal("345")) new3 = TextNode(Terminal("6+")) new4 = TextNode(Terminal("789")) # this should never be touched new4.lookup = "INT" new5 = TextNode(Terminal("+")) # this should never be touched new5.lookup = "plus" bos.insert_after(new1) new1.insert_after(new2) new2.insert_after(new3) new3.insert_after(new4) new4.insert_after(new5) self.relex(new1) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("23456") node = node.next_term assert node.symbol == Terminal("+") # check that 789 hasn't been relexed assert node.next_term is new4 assert node.next_term.symbol is new4.symbol
def test_multi_to_multi_normal_multi(self): lexer = IncrementalLexer(""" "\"[a-z\r\x80]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] n1 = mk_multitextnode([ Terminal("\"ab"), Terminal("\r"), Terminal("cd\"xy\"ef"), Terminal("\r"), Terminal("gaaaah\"") ]) bos.insert_after(n1) lexer.relex(n1) assert bos.next_term == mk_multitextnode( [Terminal("\"ab"), Terminal("\r"), Terminal("cd\"")]) assert bos.next_term.next_term.symbol == Terminal("xy") assert bos.next_term.next_term.next_term == mk_multitextnode( [Terminal("\"ef"), Terminal("\r"), Terminal("gaaaah\"")]) assert bos.next_term.next_term.next_term.next_term is eos
def test_normal_and_multi_to_multi2(self): lexer = IncrementalLexer(""" "\"[a-z\r\x80]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] n1 = TextNode(Terminal("\"abc")) n2 = TextNode(Terminal("\r")) n3 = mk_multitextnode( [Terminal("def"), Terminal("\r"), Terminal("gh\"")]) bos.insert_after(n1) n1.insert_after(n2) n2.insert_after(n3) lexer.relex(n1) assert bos.next_term == mk_multitextnode([ Terminal("\"abc"), Terminal("\r"), Terminal("def"), Terminal("\r"), Terminal("gh\"") ]) assert bos.next_term.next_term is eos
def test_multix2_to_multi(self): lexer = IncrementalLexer(""" "\"[a-z\r\x80]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] n1 = mk_multitextnode( [Terminal("\"ab"), Terminal("\r"), Terminal("cd")]) n2 = mk_multitextnode( [Terminal("ef"), Terminal("\r"), Terminal("gh\"")]) bos.insert_after(n1) n1.insert_after(n2) lexer.relex(n1) assert bos.next_term == mk_multitextnode([ Terminal("\"ab"), Terminal("\r"), Terminal("cdef"), Terminal("\r"), Terminal("gh\"") ]) assert bos.next_term.next_term is eos
def test_normal_to_normal_and_multi(self): lexer = IncrementalLexer(""" "\"[^\"]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal("\"abc\rdef\"")) bos.insert_after(text) lexer.relex(text) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), Terminal("\r"), Terminal("def\"")]) bos.next_term.children[0].symbol.name = "ab\"c" lexer.relex(bos.next_term) assert bos.next_term == TextNode(Terminal("ab")) assert bos.next_term.next_term == mk_multitextnode( [Terminal("\"c"), Terminal("\r"), Terminal("def\"")])
def test_normal_and_multi_to_normal_and_multi(self): lexer = IncrementalLexer(""" "\"[a-z\r\x80]*\"":str "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] n1 = TextNode(Terminal("ab")) n2 = mk_multitextnode( [Terminal("cd\"e"), Terminal("\r"), Terminal("fg\"")]) bos.insert_after(n1) n1.insert_after(n2) lexer.relex(n1) assert bos.next_term.symbol.name == "abcd" assert bos.next_term is n1 assert bos.next_term.next_term.lookup == "str" assert bos.next_term.next_term == mk_multitextnode( [Terminal("\"e"), Terminal("\r"), Terminal("fg\"")]) assert bos.next_term.next_term is n2 assert bos.next_term.next_term.next_term is eos
def test_lookahead(self): lexer = IncrementalLexer(""" "aaa":aaa "a":a "b":b """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal("baab")) bos.insert_after(text) lexer.relex(text) assert ast.parent.children[1].symbol.name == "b" assert ast.parent.children[2].symbol.name == "a" assert ast.parent.children[3].symbol.name == "a" assert ast.parent.children[4].symbol.name == "b" ast.parent.children[ 1].symbol = None # Check that lookback doesn't overreach ast.parent.children[3].symbol.name = "aa" lexer.relex(ast.parent.children[3]) assert ast.parent.children[2].symbol.name == "aaa" assert ast.parent.children[3].symbol.name == "aa" assert ast.parent.children[3].deleted is True assert ast.parent.children[4].symbol.name == "b"
def test_backwards_lexing(self): lexer = IncrementalLexer(""" "::=":doublecolon "=":equal ":":singlecolon """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal(":")) bos.insert_after(text) lexer.relex(text) assert bos.next_term.symbol.name == ":" assert bos.next_term.lookup == "singlecolon" assert text.lookahead == 1 text.symbol.name = "::" lexer.relex(text) assert text.lookahead == 2 text2 = text.next_term assert text2.lookback == 1 assert bos.next_term.symbol.name == ":" assert bos.next_term.next_term.symbol.name == ":" text2.symbol.name = ":=" lexer.relex(text2) assert bos.next_term.symbol.name == "::=" assert isinstance(bos.next_term.next_term, EOS)
def test_simple3(self): ast = AST() ast.init() new = TextNode(Terminal('"""')) ast.parent.children[0].insert_after(new) it = self.lexer.get_token_iter(new) assert next(it) == ('""', "dstring", 2, [TextNode(Terminal('"""'))], -1)
def test_relex2(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1")) bos.insert_after(new) self.relex(new) node = bos.next_term assert node.symbol == Terminal("1") new.symbol.name = "1+" self.relex(new) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node.symbol.name = "+2" self.relex(node) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("2")
def test_simple(self): ast = AST() ast.init() new = TextNode(Terminal('"""abc"""')) ast.parent.children[0].insert_after(new) it = self.lexer.get_token_iter(new) assert it.next() == ('"""abc"""', "MLS", 0, [TextNode(Terminal('"""abc"""'))], 0)
def test_multitoken_reuse1(self): lexer = IncrementalLexer(""" "\"[a-z\r\x80]*\"":str """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal("\"abc\rdef\"")) bos.insert_after(text) lexer.relex(text) assert bos.next_term.lookup == "str" assert bos.next_term == mk_multitextnode( [Terminal("\"abc"), Terminal("\r"), Terminal("def\"")]) assert bos.next_term.children[0] is text bos.next_term.children[ 2].symbol.name = "de\rf\"" # insert another newline child0 = bos.next_term.children[0] child1 = bos.next_term.children[1] child2 = bos.next_term.children[2] mt = bos.next_term lexer.relex(bos.next_term) assert bos.next_term == mk_multitextnode([ Terminal("\"abc"), Terminal("\r"), Terminal("de"), Terminal("\r"), Terminal("f\"") ]) # test if nodes within a MultiTextNode are reused assert bos.next_term.children[0] is child0 assert bos.next_term.children[1] is child1 assert bos.next_term.children[2] is child2 child3 = bos.next_term.children[3] child4 = bos.next_term.children[4] assert child0.prev_term is None assert child0.next_term is child1 assert child1.prev_term is child0 assert child1.next_term is child2 assert child2.prev_term is child1 assert child2.next_term is child3 assert child3.prev_term is child2 assert child3.next_term is child4 assert child4.prev_term is child3 assert child4.next_term is None assert bos.next_term is mt # reused the MultiTextNode
def test_lexingerror(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1b")) bos.insert_after(new) it = self.lexer.get_token_iter(new) assert next(it) == ("1", "INT", 1, [TextNode(Terminal("1b"))], -1) with pytest.raises(LexingError): next(it)
def test_lookback(self): lexer = IncrementalLexer(""" "\"\"\"[^\"]*\"\"\"":triplestring "\"[^\"]*\"":string "[a-z]+":var """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text1 = TextNode(Terminal("abcd")) text2 = TextNode(Terminal("ef")) text3 = TextNode(Terminal("ghij")) text4 = TextNode(Terminal("k")) text5 = TextNode(Terminal("lm")) text6 = TextNode(Terminal("nopqr")) text7 = TextNode(Terminal("stu")) bos.insert_after(text1) text1.insert_after(text2) text2.insert_after(text3) text3.insert_after(text4) text4.insert_after(text5) text5.insert_after(text6) text6.insert_after(text7) lexer.relexed.add(text1) lexer.relexed.add(text2) lexer.relexed.add(text3) lexer.relexed.add(text4) lexer.relexed.add(text5) lexer.relexed.add(text6) text1.lookahead = 7 text2.lookahead = 7 text3.lookahead = 1 text4.lookahead = 0 text5.lookahead = 0 text6.lookahead = 3 text7.lookahead = 0 lexer.update_lookback(text1, text1) assert text1.lookback == 0 assert text2.lookback == 1 assert text3.lookback == 2 assert text4.lookback == 3 assert text5.lookback == 3 assert text6.lookback == 0 assert text7.lookback == 1
def test_token_iter(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1+2*3")) bos.insert_after(new) next_token = self.lexer.lexer.get_token_iter(new).next assert next_token() == ("1", "INT", 1, [TextNode(Terminal("1+2*3"))]) assert next_token() == ("+", "plus", 0, [TextNode(Terminal("1+2*3"))]) assert next_token() == ("2", "INT", 1, [TextNode(Terminal("1+2*3"))]) assert next_token() == ("*", "mul", 0, [TextNode(Terminal("1+2*3"))]) assert next_token() == ("3", "INT", 1, [TextNode(Terminal("1+2*3"))])
def test_relex_newline(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("1+2\r3+4")) bos.insert_after(new1) self.relex(new1) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term; assert node.symbol == Terminal("1") node = node.next_term; assert node.symbol == Terminal("+") node = node.next_term; assert node.symbol == Terminal("2") node = node.next_term; assert node.symbol == Terminal("\r") node = node.next_term; assert node.symbol == Terminal("3") node = node.next_term; assert node.symbol == Terminal("+") node = node.next_term; assert node.symbol == Terminal("4")
def test_relex2(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1")) bos.insert_after(new) self.relex(new) node = bos.next_term; assert node.symbol == Terminal("1") new.symbol.name = "1+" self.relex(new) node = bos.next_term; assert node.symbol == Terminal("1") node = node.next_term; assert node.symbol == Terminal("+") node.symbol.name = "+2" self.relex(node) node = bos.next_term; assert node.symbol == Terminal("1") node = node.next_term; assert node.symbol == Terminal("+") node = node.next_term; assert node.symbol == Terminal("2")
def test_relex_return(self): ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal("123\r")) bos.insert_after(text) self.relex(text) last_return = eos.prev_term assert last_return.symbol.name == "\r" assert last_return.lookup == "<return>" new_number = TextNode(Terminal("3")) last_return.insert_after(new_number) self.relex(new_number) new = TextNode(Terminal("\r")) last_return.insert_after(new) self.relex(new) assert new.symbol == Terminal("\r") assert new.lookup == "<return>"
def test_lookahead(self): lexer = IncrementalLexer(""" "aaa":aaa "a":a "b":b """) ast = AST() ast.init() bos = ast.parent.children[0] eos = ast.parent.children[1] text = TextNode(Terminal("baab")) bos.insert_after(text) lexer.relex(text) assert ast.parent.children[1].symbol.name == "b" assert ast.parent.children[2].symbol.name == "a" assert ast.parent.children[3].symbol.name == "a" assert ast.parent.children[4].symbol.name == "b" ast.parent.children[1].symbol = None ast.parent.children[3].symbol.name = "aa" lexer.relex(ast.parent.children[3]) assert ast.parent.children[2].symbol.name == "aaa" assert ast.parent.children[3].symbol.name == "b"