def test_translator():
string = '123\n12345\n12'
translator = build_lc_translator(string)
assert translator(1, 0) == 0
assert translator(2, 0) == 4
assert translator(2, 1) == 5
assert translator(3, 0) == 10
assert translator(3, 1) == 11
def generate_tokens(program_text):
non_ws_tokens = _generate_tokens(StringIO(program_text).readline)
translator = build_lc_translator(program_text)
current_position = 0
for type, text, start_tup, stop_tup, _ in non_ws_tokens:
start = translator(*start_tup)
stop = translator(*stop_tup)
if start != current_position and type != name_tok['ENDMARKER']:
yield Token(current_position, start, WHITESPACE, program_text[current_position:start])
current_position = stop
yield Token(start, stop, type, text)