def verify(self, src: str, kind: TokenType, value: TokenValue = None, nonl: bool = True): tk = Tokenizer(src, '<test>') token = tk.next(ignore_nl=nonl) self.assertEqual(kind, token.kind, 'wrong token kind: %s' % token) self.assertEqual(value, token.value, 'wrong token value: %s' % token) self.assertEqual(tk.next().kind, TokenType.End, 'should be EOF: %s' % token)
def _run_test(self, src, seq): tk = Tokenizer(src, '<test>') while seq and not tk.is_eof: token = tk.next() kind, value, row, col, srow, scol = seq[0] self.assertEqual(kind, token.kind, repr(seq[0])) self.assertEqual(value, token.value, repr(seq[0])) self.assertEqual(row, tk.save.row, repr(seq[0])) self.assertEqual(col, tk.save.col, repr(seq[0])) self.assertEqual(srow, tk.state.row, repr(seq[0])) self.assertEqual(scol, tk.state.col, repr(seq[0])) self.assertEqual(row, token.row, repr(seq[0])) self.assertEqual(col, token.col, repr(seq[0])) seq = seq[1:] self.assertTrue(not seq, 'not seq') self.assertTrue(tk.is_eof, 'tk.is_eof')
def test_line_number(self): src = r"""package parser import ( `io` `net/url` `reflect` `code.example.org/chenzhuoyu/infra-kernels/utils` _ `git.example.org/ee/people/infra/gateway/biz/dispatch/service` ) """ print(Parser(Tokenizer(src, 'test.go')).parse())
def test_ellipsis(self): tk = Tokenizer('a.....', '<test>') token = tk.next() self.assertEqual(TokenType.Name, token.kind) self.assertEqual('a', token.value) token = tk.next() self.assertEqual(TokenType.Operator, token.kind) self.assertEqual('...', token.value) token = tk.next() self.assertEqual(TokenType.Operator, token.kind) self.assertEqual('.', token.value) token = tk.next() self.assertEqual(TokenType.Operator, token.kind) self.assertEqual('.', token.value) token = tk.next() self.assertEqual(TokenType.LF, token.kind) token = tk.next() self.assertEqual(TokenType.End, token.kind)
def verify(self, src: str, kind: TokenType, value: TokenValue = None, nonl: bool = True): tk = Tokenizer(src, '<test>') token = tk.next() while token.kind == TokenType.Comments or (nonl and token.kind == TokenType.LF): if token.kind == TokenType.Comments and not nonl and '\n' in token.value: token = Token.eol(tk) break else: token = tk.next() self.assertEqual(kind, token.kind, 'wrong token kind: %s' % token) self.assertEqual(value, token.value, 'wrong token value: %s' % token) token = tk.next() while token.kind == TokenType.Comments or token.kind == TokenType.LF: token = tk.next() self.assertEqual(TokenType.End, token.kind, 'should be EOF: %s' % token)
def invalid(self, src: str, exc: str): self.assertRaisesRegex(SyntaxError, exc, lambda: Tokenizer(src, '<test>').next())
def test_const(self): print(Parser(Tokenizer(_const_src, 'test.go')).parse())
def test_import(self): Parser(Tokenizer(_import_src, 'test.go')).parse()
def main(): with open(FNAME, 'r', newline=None) as fp: Parser(Tokenizer(fp.read(), FNAME)).parse()
def main(): with open(FNAME, 'r', newline=None) as fp: tk = Tokenizer(fp.read(), FNAME) while not tk.is_eof: print(tk.next())