def translate_source(buf, filename='<string>'):
"""
Since we can't modify the parser in the builtin parser module we
must do token translation here. Luckily it does not affect line
numbers.
def foo [plain] (...): -> def _q_plain_template__foo(...):
def foo [html] (...): -> def _q_html_template__foo(...):
h'foo' -> f'<HSTRING_PREFIX>foo'
F'foo' -> f'<HSTRING_PREFIX>foo'
"""
assert isinstance(buf, bytes)
fp = io.BytesIO(buf)
try:
tokens = list(tokenize.tokenize(fp.readline))
except SyntaxError as exc:
exc_type = type(exc)
raise exc_type(str(exc), (filename, exc.lineno, exc.offset, exc.text))
translate_hstrings(tokens)
translate_defs(tokens)
ut = tokenize.Untokenizer()
src = ut.untokenize(tokens)
return ut.encoding, src
def translate(fn, verbose=False):
with open(fn, 'rb') as fp:
tokens = list(tokenize.tokenize(fp.readline))
translate_ptl(tokens)
ut = tokenize.Untokenizer()
src = ut.untokenize(tokens)
if verbose:
sys.stdout.write(src)
return ut.encoding, src