def setUp(self): self._lexer = Lexer() self._lexer.addTokenType(Literal.get()) self._lexer.addTokenType(MultiLineLiteral.get()) self._lexer.addTokenType(Word('[a-zA-Z_][a-zA-Z_0-9]*')) self._lexer.addTokenType(Separator('(')) self._lexer.addTokenType(Separator(')')) self._lexer.addTokenType(Separator(';')) self._lexer.addTokenType(Separator('+')) self._lexer.addTokenType(Separator('=')) self._lexer.addTokenType(Separator('.', whitespaceAllowed=False)) self._lexer.enableLineComments('#')
class LexerTest(unittest.TestCase): def setUp(self): self._lexer = Lexer() self._lexer.addTokenType(Literal.get()) self._lexer.addTokenType(MultiLineLiteral.get()) self._lexer.addTokenType(Word('[a-zA-Z_][a-zA-Z_0-9]*')) self._lexer.addTokenType(Separator('(')) self._lexer.addTokenType(Separator(')')) self._lexer.addTokenType(Separator(';')) self._lexer.addTokenType(Separator('+')) self._lexer.addTokenType(Separator('=')) self._lexer.addTokenType(Separator('.', whitespaceAllowed=False)) self._lexer.enableLineComments('#') def tearDown(self): self._lexer = None def testSeparatorInLiteral(self): code = "'1.23 '.toNumber();" self._lexer.setInputStream(StringInput(code)) tokens = [] token = self._lexer.getNextToken() while token: tokens.append(token) token = self._lexer.getNextToken() self._print(code, tokens) self.assertIsNot(tokens, []) self.assertEqual(len(tokens), 6) def testSeparatorBetweenLiterals(self): code = "'1.23'+'4.56';" self._lexer.setInputStream(StringInput(code)) tokens = [] token = self._lexer.getNextToken() while token: tokens.append(token) token = self._lexer.getNextToken() self._print(code, tokens) self.assertIsNot(tokens, []) self.assertEqual(len(tokens), 4) def testNoWSAllowedSeperator(self): code = "person.getAddress().street" self._lexer.setInputStream(StringInput(code)) tokens = [] token = self._lexer.getNextToken() while token: tokens.append(token) token = self._lexer.getNextToken() self._print(code, tokens) self.assertIsNot(tokens, []) self.assertEqual(len(tokens), 7) lastText = tokens[-1].getText() self.assertEqual(lastText, "street") def testMultLineLiterals(self): print("Multi-Line-Literals:") self._lexer.setInputStream(FileInput("multi-line-lit.txt")) token = self._lexer.getNextToken() while token: print(token.getText(), token.getTypes()) token = self._lexer.getNextToken() def _print(self, code, tokens): print("code: %s" % code) print() for t in tokens: print(t.getText(), t.getTypes()) print()