def get_pyxl_token(start_token, tokens):
ttype, tvalue, tstart, tend, tline = start_token
pyxl_parser = PyxlParser(tstart[0], tstart[1])
pyxl_parser.feed(start_token)
for token in tokens:
ttype, tvalue, tstart, tend, tline = token
if tvalue and tvalue[0] == '{':
if pyxl_parser.python_mode_allowed():
mid, right = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, mid)
pyxl_parser.feed_position_only((ttype, mid, tstart, division, tline))
tokens.rewind_and_retokenize((ttype, right, division, tend, tline))
python_tokens = list(transform_tokens(tokens))
close_curly = next(tokens)
ttype, tvalue, tstart, tend, tline = close_curly
close_curly_sub = (ttype, '', tend, tend, tline)
pyxl_parser.feed_python(python_tokens + [close_curly_sub])
continue
# else fallthrough to pyxl_parser.feed(token)
elif tvalue and ttype == tokenize.COMMENT:
if not pyxl_parser.python_comment_allowed():
tvalue, rest = tvalue[0], tvalue[1:]
division = get_end_pos(tstart, tvalue)
tokens.unshift((tokenize.ERRORTOKEN, rest, division, tend, tline))
token = ttype, tvalue, tstart, division, tline
# fallthrough to pyxl_parser.feed(token)
else:
pyxl_parser.feed_comment(token)
continue
elif tvalue and tvalue[0] == '#':
# let the python tokenizer grab the whole comment token
tokens.rewind_and_retokenize(token)
continue
else:
sp = re.split('([#{])', tvalue, maxsplit=1)
if len(sp) > 1:
tvalue, mid, right = sp
division = get_end_pos(tstart, tvalue)
tokens.unshift((ttype, mid+right, division, tend, tline))
token = ttype, tvalue, tstart, division, tline
# fallthrough to pyxl_parser.feed(token)
pyxl_parser.feed(token)
if pyxl_parser.done(): break
if not pyxl_parser.done():
lines = ['<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
for tag_info in pyxl_parser.open_tags]
raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))
remainder = pyxl_parser.get_remainder()
if remainder:
tokens.rewind_and_retokenize(remainder)
return pyxl_parser.get_token()
def get_pyxl_token(start_token, tokens):
ttype, tvalue, tstart, tend, tline = start_token
pyxl_parser = PyxlParser(tstart[0], tstart[1], tline)
pyxl_parser.feed(start_token)
for token in tokens:
ttype, tvalue, tstart, tend, tline = token
try:
pyxl_parser.feed(token)
except HTMLParseError, html_ex:
msg = 'HTMLParseError: %s (line:%d %s)' % (html_ex.msg, tstart[0], tline.strip())
raise Exception(msg)
except AssertionError, assert_ex:
msg = '%s (line:%d %s)' % (assert_ex, tstart[0], tline.strip())
raise Exception(msg)
def get_pyxl_token(start_token, tokens):
ttype, tvalue, tstart, tend, tline = start_token
pyxl_parser = PyxlParser(tstart[0], tstart[1], tline)
pyxl_parser.feed(start_token)
for token in tokens:
ttype, tvalue, tstart, tend, tline = token
try:
pyxl_parser.feed(token)
except HTMLParseError, html_ex:
msg = 'HTMLParseError: %s (line:%d: %s)' % (html_ex.msg, tstart[0],
tline.strip())
raise PyxlParseError(msg)
except AssertionError, assert_ex:
msg = '%s (line:%d: %s)' % (assert_ex, tstart[0], tline.strip())
raise PyxlParseError(msg)
