def test_lexer_end_to_end():
tokenizer = Tokenizer()
event_engine = EventEngine([
FileReader(),
AsciiCategorizer(),
tokenizer,
])
token_event = MagicMock()
tokenizer.set_external_event_handler(token_event)
event_engine.start(('open', base_dir / 'small_source.bas'))
token_event.assert_has_calls([
call(('number', '1')),
call(('identifier', 'LET')),
call(('variable', 'L')),
call(('special', '=')),
call(('number', '2000')),
call(('end_of_line', '\n')),
call(('number', '10')),
call(('identifier', 'DIM')),
call(('variable', 'S')),
call(('special', '(')),
call(('variable', 'L')),
call(('special', ')')),
call(('end_of_line', '\n')),
call(('number', '20')),
call(('identifier', 'LET')),
call(('variable', 'P')),
call(('special', '=')),
call(('number', '2')),
call(('end_of_line', '\n')),
])
def test_invalid_string(source_line):
add_external_event = MagicMock()
tokenizer = Tokenizer(add_external_event)
categorizer = AsciiCategorizer(tokenizer.handle_event)
categorizer.handle_event(source_line)
tokenizer.handle_event(('eof', None))
for event in categorizer:
categorizer.handle_event(event)
def test_filters_ascii_chars(source_line, tokens):
add_external_event = MagicMock()
tokenizer = Tokenizer(add_external_event)
categorizer = AsciiCategorizer(tokenizer.handle_event)
categorizer.handle_event(source_line)
tokenizer.handle_event(('eof', None))
for event in categorizer:
categorizer.handle_event(event)
add_external_event.assert_has_calls(tokens)
def create_event_engine():
return EventEngine([
FileReader(),
AsciiCategorizer(),
Tokenizer(),
SyntaxRecognizer(),
])
def test_string(source_line, expected_call):
add_external_event = MagicMock()
tokenizer = Tokenizer(add_external_event)
categorizer = AsciiCategorizer(tokenizer.handle_event)
categorizer.handle_event(source_line)
for event in categorizer:
categorizer.handle_event(event)
add_external_event.assert_has_calls([call(expected_call)])
def to_ir(filename):
engine = EventEngine([
FileReader(),
AsciiCategorizer(),
Tokenizer(),
SyntaxRecognizer(),
])
engine.start(('open', filename))
def test_fail_at_invalid_identifier():
add_external_event = MagicMock()
tokenizer = Tokenizer(add_external_event)
tokenizer.handle_event(('ascii_character', 'X'))
tokenizer.handle_event(('ascii_character', 'Y'))
tokenizer.handle_event(('ascii_digit', '0'))
tokenizer.handle_event(('ascii_ctrl', '\n'))