def get_pyxl_token(start_token, tokens):
ttype, tvalue, tstart, tend, tline = start_token
pyxl_parser = PyxlParser(tstart[0], tstart[1])
pyxl_parser.feed(start_token)
for token in tokens:
ttype, tvalue, tstart, tend, tline = token
if tvalue and tvalue[0] == '{':
if pyxl_parser.python_mode_allowed():
mid, right = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, mid)
pyxl_parser.feed_position_only((ttype, mid, tstart, division, tline))
tokens.rewind_and_retokenize((ttype, right, division, tend, tline))
python_tokens = list(transform_tokens(tokens))
close_curly = next(tokens)
ttype, tvalue, tstart, tend, tline = close_curly
close_curly_sub = (ttype, '', tend, tend, tline)
pyxl_parser.feed_python(python_tokens + [close_curly_sub])
continue
# else fallthrough to pyxl_parser.feed(token)
elif tvalue and ttype == tokenize.COMMENT:
if not pyxl_parser.python_comment_allowed():
tvalue, rest = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, tvalue)
tokens.unshift((tokenize.ERRORTOKEN, rest, division, tend, tline))
token = ttype, tvalue, tstart, division, tline
# fallthrough to pyxl_parser.feed(token)
else:
pyxl_parser.feed_comment(token)
continue
elif tvalue and tvalue[0] == '#':
# let the python tokenizer grab the whole comment token
tokens.rewind_and_retokenize(token)
continue
else:
sp = re.split('([#{])', tvalue, maxsplit=1)
if len(sp) > 1:
tvalue, mid, right = sp
division = get_end_pos(tstart, tvalue)
tokens.unshift((ttype, mid+right, division, tend, tline))
token = ttype, tvalue, tstart, division, tline
# fallthrough to pyxl_parser.feed(token)
pyxl_parser.feed(token)
if pyxl_parser.done(): break
if not pyxl_parser.done():
lines = ['<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
for tag_info in pyxl_parser.open_tags]
raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))
remainder = pyxl_parser.get_remainder()
if remainder:
tokens.rewind_and_retokenize(remainder)
return pyxl_parser.get_token()
def get_pyxl_token(start_token, tokens):
ttype, tvalue, tstart, tend, tline = start_token
pyxl_parser = PyxlParser(tstart[0], tstart[1])
pyxl_parser.feed(start_token)
for token in tokens:
ttype, tvalue, tstart, tend, tline = token
if tvalue and tvalue[0] == '{':
if pyxl_parser.python_mode_allowed():
mid, right = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, mid)
pyxl_parser.feed_position_only(
(ttype, mid, tstart, division, tline))
tokens.rewind_and_retokenize(
(ttype, right, division, tend, tline))
python_tokens = list(transform_tokens(tokens))
close_curly = tokens.next()
ttype, tvalue, tstart, tend, tline = close_curly
close_curly_sub = (ttype, '', tend, tend, tline)
pyxl_parser.feed_python(python_tokens + [close_curly_sub])
continue
# else fallthrough to pyxl_parser.feed(token)
elif tvalue and ttype == tokenize.COMMENT:
if not pyxl_parser.python_comment_allowed():
tvalue, rest = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, tvalue)
tokens.unshift(
(tokenize.ERRORTOKEN, rest, division, tend, tline))
token = ttype, tvalue, tstart, division, tline
# fallthrough to pyxl_parser.feed(token)
else:
pyxl_parser.feed_comment(token)
continue
elif tvalue and tvalue[0] == '#':
# let the python tokenizer grab the whole comment token
tokens.rewind_and_retokenize(token)
continue
else:
sp = re.split('([#{])', tvalue, maxsplit=1)
if len(sp) > 1:
tvalue, mid, right = sp
division = get_end_pos(tstart, tvalue)
tokens.unshift((ttype, mid + right, division, tend, tline))
token = ttype, tvalue, tstart, division, tline
# fallthrough to pyxl_parser.feed(token)
pyxl_parser.feed(token)
if pyxl_parser.done(): break
if not pyxl_parser.done():
lines = [
'<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
for tag_info in pyxl_parser.open_tags
]
raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))
remainder = pyxl_parser.get_remainder()
if remainder:
tokens.rewind_and_retokenize(remainder)
return pyxl_parser.get_token()
def get_pyxl_token(start_token, tokens, invertible, str_function):
ttype, tvalue, tstart, tend, tline = start_token
pyxl_parser = PyxlParser(tstart.row, tstart.col, str_function)
pyxl_parser.feed(start_token)
if invertible:
# In invertible mode, keep track of all the tokens we see in pyxl and
# each python fragment that appears
pyxl_tokens = [start_token]
python_fragments = []
for token in tokens:
ttype, tvalue, tstart, tend, tline = token
if tvalue and tvalue[0] == '{':
if pyxl_parser.python_mode_allowed():
# We've hit a python fragment
initial_tstart = tstart
mid, right = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, mid)
pyxl_parser.feed_position_only(
Token(ttype, mid, tstart, division, tline))
tokens.rewind_and_retokenize(
Token(ttype, right, division, tend, tline))
python_tokens = list(
transform_tokens(tokens, invertible, str_function))
close_curly = next(tokens)
ttype, tvalue, tstart, tend, tline = close_curly
close_curly_sub = Token(ttype, '', tend, tend, tline)
# strip comments in the invertible version
pyxl_parser.feed_python((strip_comments(python_tokens)
if invertible else python_tokens) +
[close_curly_sub])
if invertible:
# If we are doing invertible generation, put in a format placeholder
# into the collected pyxl tokens and collect python fragment separately.
# We keep each fragment wrapped in the curlies it came in (because just
# relying on the commas) doesn't work when there are undelimited commas.
# This also serves to preserve any internal whitespace in the fragment so
# it can be restored later.
open_curly = Token(ttype, '{', initial_tstart, division,
tline)
pyxl_tokens.append(
Token(ttype, '{}', initial_tstart, tend, ''))
python_fragments.append([open_curly] + python_tokens +
[close_curly])
continue
# else fallthrough to pyxl_parser.feed(token)
elif tvalue and ttype == tokenize.COMMENT:
if not pyxl_parser.python_comment_allowed():
tvalue, rest = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, tvalue)
tokens.unshift(
Token(tokenize.ERRORTOKEN, rest, division, tend, tline))
token = Token(ttype, tvalue, tstart, division, tline)
# fallthrough to pyxl_parser.feed(token)
else:
# strip comments in the invertible version
if not invertible:
pyxl_parser.feed_comment(token)
if invertible:
pyxl_tokens.append(sanitize_token(token))
continue
elif tvalue and tvalue[0] == '#':
# let the python tokenizer grab the whole comment token
tokens.rewind_and_retokenize(token)
continue
else:
sp = re.split('([#{])', tvalue, maxsplit=1)
if len(sp) > 1:
tvalue, mid, right = sp
division = get_end_pos(tstart, tvalue)
tokens.unshift(Token(ttype, mid + right, division, tend,
tline))
token = Token(ttype, tvalue, tstart, division, tline)
# fallthrough to pyxl_parser.feed(token)
pyxl_parser.feed(token)
if invertible:
pyxl_tokens.append(sanitize_token(token))
if pyxl_parser.done(): break
if not pyxl_parser.done():
lines = [
'<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
for tag_info in pyxl_parser.open_tags
]
raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))
remainder = pyxl_parser.get_remainder()
if remainder:
remainder = fix_token(remainder)
tokens.rewind_and_retokenize(remainder)
# Strip the remainder out from the last seen token
if invertible and remainder.value:
assert '{' not in remainder.value and '}' not in remainder.value
last = pyxl_tokens[-1]
pyxl_tokens[-1] = Token(last.ttype,
last.value[:-len(remainder[1])],
last.start, remainder.start, last.line)
pyxl_parser_start = Pos(*pyxl_parser.start)
if invertible:
# We want to include a trailing comment after a pyxl close in
# the pyxl block itself to prevent black from migrating it.
# (which matters a lot if it is a type: ignore.)
try:
peek = fix_token(next(tokens))
except StopIteration:
pass
else:
if peek.ttype == tokenize.COMMENT:
pyxl_tokens.append(peek)
else:
tokens.unshift(peek)
output = "html.PYXL('''{}''', {}, {}, {}{}{})".format(
untokenize(pyxl_tokens).replace('\\', '\\\\').replace("'", "\\'"),
# Include the real compiled pyxl so that tools can see all the gritty details
untokenize([pyxl_parser.get_token()]),
# Include the start column so we can shift it if needed
pyxl_parser_start.col,
# Include the columns of each python fragment so we can shift them if needed
''.join([
str(first_non_ws_token(x).start.col) + ', '
for x in python_fragments
]),
# When untokenizing python fragments, make sure to place them in their
# proper columns so that we don't detect a shift if there wasn't one.
''.join(
[untokenize_with_column(x) + ', ' for x in python_fragments]),
# Stick a final argument at the end so that all the real arguments are
# always terminated by commas (and don't pick up spurious whitespace
# with certain black formatting modes)
'0',
)
return Token(tokenize.STRING, output, pyxl_parser_start,
Pos(*pyxl_parser.end), '')
else:
return fix_token(pyxl_parser.get_token())