def test_translator(): string = '123\n12345\n12' translator = build_lc_translator(string) assert translator(1, 0) == 0 assert translator(2, 0) == 4 assert translator(2, 1) == 5 assert translator(3, 0) == 10 assert translator(3, 1) == 11
def generate_tokens(program_text): non_ws_tokens = _generate_tokens(StringIO(program_text).readline) translator = build_lc_translator(program_text) current_position = 0 for type, text, start_tup, stop_tup, _ in non_ws_tokens: start = translator(*start_tup) stop = translator(*stop_tup) if start != current_position and type != name_tok['ENDMARKER']: yield Token(current_position, start, WHITESPACE, program_text[current_position:start]) current_position = stop yield Token(start, stop, type, text)