def test_number(self): self.assertSequenceEqual([('number', '0', 0), (None, '', 1)], tuple(tokens('0'))) self.assertSequenceEqual([('number', '123', 0), (None, '', 3)], tuple(tokens('123'))) self.assertSequenceEqual([('number', '123.45', 0), (None, '', 6)], tuple(tokens('123.45'))) with self.assertRaises(LexerException): tuple(tokens('123.')) with self.assertRaises(LexerException): tuple(tokens('.123'))
def test_symbol(self): self.assertSequenceEqual([('+', '+', 0), (None, '', 1)], tuple(tokens('+'))) self.assertSequenceEqual([('-', '-', 0), (None, '', 1)], tuple(tokens('-'))) self.assertSequenceEqual([('*', '*', 0), (None, '', 1)], tuple(tokens('*'))) self.assertSequenceEqual([('/', '/', 0), (None, '', 1)], tuple(tokens('/'))) self.assertSequenceEqual([('^', '^', 0), (None, '', 1)], tuple(tokens('^'))) self.assertSequenceEqual([('(', '(', 0), (None, '', 1)], tuple(tokens('('))) self.assertSequenceEqual([(')', ')', 0), (None, '', 1)], tuple(tokens(')'))) with self.assertRaises(LexerException): tuple(tokens('.'))
def readFile(): file = open(EXPRESIONS_FILE, "r") filas = (file.read().splitlines()) clearFile(RESULT_FILE) for exp in filas: result = lexer.tokens(exp) writeFile(result) lexer.lista = [] print(exp) file.close()
def test_tokens(self): self.assertSequenceEqual([('number', '2', 1), ('name', 'a', 2), ('+', '+', 4), ('name', 'b', 6), ('^', '^', 7), ('number', '2', 8), ('*', '*', 10), ('name', 'sin', 13), ('(', '(', 16), ('-', '-', 17), ('name', 'PI', 18), ('/', '/', 20), ('number', '2.0', 21), (')', ')', 24), (None, '', 26)], tuple(tokens(' 2a + b^2 * sin(-PI/2.0) ')))
def test_bareword(self): self.assertSequenceEqual([('name', 'some_token', 0), (None, '', 10)], tuple(tokens('some_token'))) self.assertSequenceEqual([('name', 'SomeToken', 0), (None, '', 9)], tuple(tokens('SomeToken')))
def parse(self, lexer): for token in lexer.tokens(): self.handleToken(token) while len(self.states) > 1: self.leaveState(self.states[-1])
import lexer import tokens program = """\ yo x iz 1 one does not simply get out until x iz 2 wow "x is not 2 yet" let x be 2 end wow "x is now 2" """ for token in lexer.tokens(program, tokens.MEME_DEFINITIONS): print(token)