def test_handles_string_literal(self): string1 = 'this is a string !@#$^woooo 1234' lut = Lexer('"{string1}"'.format(string1=string1)) result = lut.get_next_token() self.assertEqual(result.type, TOKEN_STRING_LITERAL) self.assertEqual(result.value, string1)
def test_handles_reserved_word(self): lut = Lexer('BEGIN') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_BEGINFUNCTION)
def test_collects_errors(self): lut = Lexer('&') lut.get_next_token() self.assertEqual(len(lut.errors()), 1)
def test_handles_valid_double_character_symbol(self): lut = Lexer('==') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_EQ)
def test_handles_empty_string(self): lut = Lexer('') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_EOF)
def test_decimal_constant(self): self.assertEqual(Lexer("4.2").check_tokens(), "<CONSTANT=4.2>\n")
def run_interpreter(stream): token_stream = Lexer.start(stream) parsed_stream = Parser.start(token_stream) return Interpreter.simulate_statements(parsed_stream)
def eat_tokens(line): lex = Lexer(line) line = "" while lex.get_next_token(): line += lex.peek_token().test() return line
def test_move_pointer_stream(self): tokenstream_oracle = TokenStream([PointerRight(0), PointerLeft(15)]) actual_tokenstream = Lexer.start('H4ck3rs Delight. Del1ghtH4ck3rs.') self.assertEqual(actual_tokenstream, tokenstream_oracle)
import lexer.tag as Tag from lexer.lexer import Lexer, LexError import utils.utils as utils testdir = utils.dirpath('lexer_test', root=utils.testdir) filelist = [file for file in os.listdir(testdir) if file.endswith('.em')] # Pass a "cmp" command line argument to make the script generate the compare files suffix = '.cmp' if len(sys.argv)>1 and sys.argv[1] == 'cmp' else '.out' for file in filelist: infile = testdir + os.path.sep + file outfile = infile[0:-3] + suffix ins = open(infile) lex = Lexer(ins.read()) ins.close() outs = open(outfile, 'w') #print 'Test:', file outstr_1 = '' outstr_2 = '' # set last token to EOL so we can skip the first EOL if it is # from the end of a comment lastTokenTag = '\n' while True:
def test_separate_integers_as_one_stream(self): tokenstream_oracle = TokenStream([Integer(0, 152546)]) actual_tokenstream = Lexer.start('152 546') self.assertEqual(actual_tokenstream, tokenstream_oracle)
def test_integer_stream(self): tokenstream_oracle = TokenStream([Integer(0, 152), InputCharCell(3), Integer(6, 546)]) actual_tokenstream = Lexer.start('152 DH@ 546') self.assertEqual(actual_tokenstream, tokenstream_oracle)
def test_io_char_stream(self): tokenstream_oracle = TokenStream([OutputCharCell(0), InputCharCell(3), OutputCharCell(18)]) actual_tokenstream = Lexer.start('HD@ Del1ghtH4ckers@ HD@') self.assertEqual(actual_tokenstream, tokenstream_oracle)
def test_basic_constant(self): self.assertEqual(Lexer("42").check_tokens(), "<CONSTANT=42>\n")
def test_basic_char(self): self.assertEqual( Lexer("'*'").get_next_token().test(), "<CHAR_CONST='*'>")
def test_minus_sign_constant(self): self.assertEqual(Lexer("-42").check_tokens(), "<MINUS><CONSTANT=42>\n")
def test_escaped_newline(self): self.assertEqual( Lexer("'\\n'").get_next_token().test(), "<CHAR_CONST='\\n'>")
def test_io_num_stream(self): tokenstream_oracle = TokenStream([OutputNumCell(0), InputNumCell(15)]) actual_tokenstream = Lexer.start('H4ckers Del1ght= DH=') self.assertEqual(actual_tokenstream, tokenstream_oracle)
def test_octal_char(self): self.assertEqual( Lexer("'\\042'").get_next_token().test(), "<CHAR_CONST='\\042'>")
def test_handles_single_character_symbol(self): lut = Lexer('+') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_ADD)
def test_hex_char(self): self.assertEqual( Lexer("'0x042'").get_next_token().test(), "<CHAR_CONST='0x042'>")
def test_handles_invalid_double_character_symbol(self): lut = Lexer('[]') self.assertEqual(lut.get_next_token().type, TOKEN_LBRACKET) self.assertEqual(lut.get_next_token().type, TOKEN_RBRACKET)
def test_error_newline_in_const(self): self.assertRaises(Lexer("'\n1'").get_next_token)
def test_handles_identifier_abc_123(self): lut = Lexer('abc_123') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_IDENTIFIER) self.assertEqual(result.value, 'abc_123')
def test_error_escaped_newline_followed_by_newline(self): self.assertRaises(Lexer("'\\n\n'").get_next_token)
def test_handles_whitespace(self): lut = Lexer(' ') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_EOF)
def test_error_unclosed_quote(self): self.assertRaises(Lexer("'A").get_next_token)
def test_handles_integer_literal(self): lut = Lexer('12345') result = lut.get_next_token() self.assertEqual(result.type, TOKEN_INTEGER_LITERAL) self.assertEqual(result.value, 12345)
def test_basic_escaped_string(self): self.assertEqual( Lexer('"Basic \\"string\\""').get_next_token().test(), '<STRING="Basic \\"string\\"">', )
def test_handles_inline_comment(self): comment = 'comment' lut = Lexer('//{comment}'.format(comment=comment)) result = lut.get_next_token() self.assertEqual(result.type, TOKEN_COMMENT) self.assertEqual(result.value, comment)
def test_escaped_string(self): self.assertEqual( Lexer('"Escaped \\\\\\"string\\\\\\\\\\"\\\\"').get_next_token().test(), '<STRING="Escaped \\\\\\"string\\\\\\\\\\"\\\\">', )
def test_plus_sign_constant(self): self.assertEqual(Lexer("+42").check_tokens(), "<PLUS><CONSTANT=42>\n")
def test_basic_string(self): self.assertEqual(Lexer('"Basic string"').get_next_token().test(), '<STRING="Basic string">')
def test_many_signs_constant(self): self.assertEqual( Lexer("+-42").check_tokens(), "<PLUS><MINUS><CONSTANT=42>\n")
import lexer.tag as Tag from lexer.lexer import Lexer, LexError import utils.utils as utils testdir = utils.dirpath('lexer_test', root=utils.testdir) filelist = [file for file in os.listdir(testdir) if file.endswith('.em')] # Pass a "cmp" command line argument to make the script generate the compare files suffix = '.cmp' if len(sys.argv) > 1 and sys.argv[1] == 'cmp' else '.out' for file in filelist: infile = testdir + os.path.sep + file outfile = infile[0:-3] + suffix ins = open(infile) lex = Lexer(ins.read()) ins.close() outs = open(outfile, 'w') #print 'Test:', file outstr_1 = '' outstr_2 = '' # set last token to EOL so we can skip the first EOL if it is # from the end of a comment lastTokenTag = '\n' while True:
def test_decimal_constant_starting_with_dot(self): self.assertEqual(Lexer(".42").check_tokens(), "<CONSTANT=.42>\n")
def test_edit_cell_stream(self): tokenstream_oracle = TokenStream([IncrementCell(0), IncrementCell(15), IncrementCell(18), DecrementCell(21)]) actual_tokenstream = Lexer.start('H4ck3rs Delight! HD! HD! Del1ghtH4ck3rs!') self.assertEqual(actual_tokenstream, tokenstream_oracle)