def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1]) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token if tvalue and tvalue[0] == '{': if pyxl_parser.python_mode_allowed(): mid, right = tvalue[0], tvalue[1:] division = get_end_pos(tstart, mid) pyxl_parser.feed_position_only((ttype, mid, tstart, division, tline)) tokens.rewind_and_retokenize((ttype, right, division, tend, tline)) python_tokens = list(transform_tokens(tokens)) close_curly = next(tokens) ttype, tvalue, tstart, tend, tline = close_curly close_curly_sub = (ttype, '', tend, tend, tline) pyxl_parser.feed_python(python_tokens + [close_curly_sub]) continue # else fallthrough to pyxl_parser.feed(token) elif tvalue and ttype == tokenize.COMMENT: if not pyxl_parser.python_comment_allowed(): tvalue, rest = tvalue[0], tvalue[1:] division = get_end_pos(tstart, tvalue) tokens.unshift((tokenize.ERRORTOKEN, rest, division, tend, tline)) token = ttype, tvalue, tstart, division, tline # fallthrough to pyxl_parser.feed(token) else: pyxl_parser.feed_comment(token) continue elif tvalue and tvalue[0] == '#': # let the python tokenizer grab the whole comment token tokens.rewind_and_retokenize(token) continue else: sp = re.split('([#{])', tvalue, maxsplit=1) if len(sp) > 1: tvalue, mid, right = sp division = get_end_pos(tstart, tvalue) tokens.unshift((ttype, mid+right, division, tend, tline)) token = ttype, tvalue, tstart, division, tline # fallthrough to pyxl_parser.feed(token) pyxl_parser.feed(token) if pyxl_parser.done(): break if not pyxl_parser.done(): lines = ['<%s> at (line:%d)' % (tag_info['tag'], tag_info['row']) for tag_info in pyxl_parser.open_tags] raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines)) remainder = pyxl_parser.get_remainder() if remainder: tokens.rewind_and_retokenize(remainder) return pyxl_parser.get_token()
def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1], tline) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token try: pyxl_parser.feed(token) except HTMLParseError, html_ex: msg = 'HTMLParseError: %s (line:%d %s)' % (html_ex.msg, tstart[0], tline.strip()) raise Exception(msg) except AssertionError, assert_ex: msg = '%s (line:%d %s)' % (assert_ex, tstart[0], tline.strip()) raise Exception(msg)
def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1], tline) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token try: pyxl_parser.feed(token) except HTMLParseError, html_ex: msg = 'HTMLParseError: %s (line:%d: %s)' % (html_ex.msg, tstart[0], tline.strip()) raise PyxlParseError(msg) except AssertionError, assert_ex: msg = '%s (line:%d: %s)' % (assert_ex, tstart[0], tline.strip()) raise PyxlParseError(msg)