def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1], tline) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token try: pyxl_parser.feed(token) except HTMLParseError, html_ex: msg = 'HTMLParseError: %s (line:%d %s)' % (html_ex.msg, tstart[0], tline.strip()) raise Exception(msg) except AssertionError, assert_ex: msg = '%s (line:%d %s)' % (assert_ex, tstart[0], tline.strip()) raise Exception(msg)
def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1], tline) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token try: pyxl_parser.feed(token) except HTMLParseError, html_ex: msg = 'HTMLParseError: %s (line:%d: %s)' % (html_ex.msg, tstart[0], tline.strip()) raise PyxlParseError(msg) except AssertionError, assert_ex: msg = '%s (line:%d: %s)' % (assert_ex, tstart[0], tline.strip()) raise PyxlParseError(msg)
def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1]) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token if tvalue and tvalue[0] == '{': if pyxl_parser.python_mode_allowed(): mid, right = tvalue[0], tvalue[1:] division = get_end_pos(tstart, mid) pyxl_parser.feed_position_only((ttype, mid, tstart, division, tline)) tokens.rewind_and_retokenize((ttype, right, division, tend, tline)) python_tokens = list(transform_tokens(tokens)) close_curly = next(tokens) ttype, tvalue, tstart, tend, tline = close_curly close_curly_sub = (ttype, '', tend, tend, tline) pyxl_parser.feed_python(python_tokens + [close_curly_sub]) continue # else fallthrough to pyxl_parser.feed(token) elif tvalue and ttype == tokenize.COMMENT: if not pyxl_parser.python_comment_allowed(): tvalue, rest = tvalue[0], tvalue[1:] division = get_end_pos(tstart, tvalue) tokens.unshift((tokenize.ERRORTOKEN, rest, division, tend, tline)) token = ttype, tvalue, tstart, division, tline # fallthrough to pyxl_parser.feed(token) else: pyxl_parser.feed_comment(token) continue elif tvalue and tvalue[0] == '#': # let the python tokenizer grab the whole comment token tokens.rewind_and_retokenize(token) continue else: sp = re.split('([#{])', tvalue, maxsplit=1) if len(sp) > 1: tvalue, mid, right = sp division = get_end_pos(tstart, tvalue) tokens.unshift((ttype, mid+right, division, tend, tline)) token = ttype, tvalue, tstart, division, tline # fallthrough to pyxl_parser.feed(token) pyxl_parser.feed(token) if pyxl_parser.done(): break if not pyxl_parser.done(): lines = ['<%s> at (line:%d)' % (tag_info['tag'], tag_info['row']) for tag_info in pyxl_parser.open_tags] raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines)) remainder = pyxl_parser.get_remainder() if remainder: tokens.rewind_and_retokenize(remainder) return pyxl_parser.get_token()
def get_pyxl_token(start_token, tokens): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart[0], tstart[1]) pyxl_parser.feed(start_token) for token in tokens: ttype, tvalue, tstart, tend, tline = token if tvalue and tvalue[0] == '{': if pyxl_parser.python_mode_allowed(): mid, right = tvalue[0], tvalue[1:] division = get_end_pos(tstart, mid) pyxl_parser.feed_position_only( (ttype, mid, tstart, division, tline)) tokens.rewind_and_retokenize( (ttype, right, division, tend, tline)) python_tokens = list(transform_tokens(tokens)) close_curly = tokens.next() ttype, tvalue, tstart, tend, tline = close_curly close_curly_sub = (ttype, '', tend, tend, tline) pyxl_parser.feed_python(python_tokens + [close_curly_sub]) continue # else fallthrough to pyxl_parser.feed(token) elif tvalue and ttype == tokenize.COMMENT: if not pyxl_parser.python_comment_allowed(): tvalue, rest = tvalue[0], tvalue[1:] division = get_end_pos(tstart, tvalue) tokens.unshift( (tokenize.ERRORTOKEN, rest, division, tend, tline)) token = ttype, tvalue, tstart, division, tline # fallthrough to pyxl_parser.feed(token) else: pyxl_parser.feed_comment(token) continue elif tvalue and tvalue[0] == '#': # let the python tokenizer grab the whole comment token tokens.rewind_and_retokenize(token) continue else: sp = re.split('([#{])', tvalue, maxsplit=1) if len(sp) > 1: tvalue, mid, right = sp division = get_end_pos(tstart, tvalue) tokens.unshift((ttype, mid + right, division, tend, tline)) token = ttype, tvalue, tstart, division, tline # fallthrough to pyxl_parser.feed(token) pyxl_parser.feed(token) if pyxl_parser.done(): break if not pyxl_parser.done(): lines = [ '<%s> at (line:%d)' % (tag_info['tag'], tag_info['row']) for tag_info in pyxl_parser.open_tags ] raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines)) remainder = pyxl_parser.get_remainder() if remainder: tokens.rewind_and_retokenize(remainder) return pyxl_parser.get_token()
def get_pyxl_token(start_token, tokens, invertible, str_function): ttype, tvalue, tstart, tend, tline = start_token pyxl_parser = PyxlParser(tstart.row, tstart.col, str_function) pyxl_parser.feed(start_token) if invertible: # In invertible mode, keep track of all the tokens we see in pyxl and # each python fragment that appears pyxl_tokens = [start_token] python_fragments = [] for token in tokens: ttype, tvalue, tstart, tend, tline = token if tvalue and tvalue[0] == '{': if pyxl_parser.python_mode_allowed(): # We've hit a python fragment initial_tstart = tstart mid, right = tvalue[0], tvalue[1:] division = get_end_pos(tstart, mid) pyxl_parser.feed_position_only( Token(ttype, mid, tstart, division, tline)) tokens.rewind_and_retokenize( Token(ttype, right, division, tend, tline)) python_tokens = list( transform_tokens(tokens, invertible, str_function)) close_curly = next(tokens) ttype, tvalue, tstart, tend, tline = close_curly close_curly_sub = Token(ttype, '', tend, tend, tline) # strip comments in the invertible version pyxl_parser.feed_python((strip_comments(python_tokens) if invertible else python_tokens) + [close_curly_sub]) if invertible: # If we are doing invertible generation, put in a format placeholder # into the collected pyxl tokens and collect python fragment separately. # We keep each fragment wrapped in the curlies it came in (because just # relying on the commas) doesn't work when there are undelimited commas. # This also serves to preserve any internal whitespace in the fragment so # it can be restored later. open_curly = Token(ttype, '{', initial_tstart, division, tline) pyxl_tokens.append( Token(ttype, '{}', initial_tstart, tend, '')) python_fragments.append([open_curly] + python_tokens + [close_curly]) continue # else fallthrough to pyxl_parser.feed(token) elif tvalue and ttype == tokenize.COMMENT: if not pyxl_parser.python_comment_allowed(): tvalue, rest = tvalue[0], tvalue[1:] division = get_end_pos(tstart, tvalue) tokens.unshift( Token(tokenize.ERRORTOKEN, rest, division, tend, tline)) token = Token(ttype, tvalue, tstart, division, tline) # fallthrough to pyxl_parser.feed(token) else: # strip comments in the invertible version if not invertible: pyxl_parser.feed_comment(token) if invertible: pyxl_tokens.append(sanitize_token(token)) continue elif tvalue and tvalue[0] == '#': # let the python tokenizer grab the whole comment token tokens.rewind_and_retokenize(token) continue else: sp = re.split('([#{])', tvalue, maxsplit=1) if len(sp) > 1: tvalue, mid, right = sp division = get_end_pos(tstart, tvalue) tokens.unshift(Token(ttype, mid + right, division, tend, tline)) token = Token(ttype, tvalue, tstart, division, tline) # fallthrough to pyxl_parser.feed(token) pyxl_parser.feed(token) if invertible: pyxl_tokens.append(sanitize_token(token)) if pyxl_parser.done(): break if not pyxl_parser.done(): lines = [ '<%s> at (line:%d)' % (tag_info['tag'], tag_info['row']) for tag_info in pyxl_parser.open_tags ] raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines)) remainder = pyxl_parser.get_remainder() if remainder: remainder = fix_token(remainder) tokens.rewind_and_retokenize(remainder) # Strip the remainder out from the last seen token if invertible and remainder.value: assert '{' not in remainder.value and '}' not in remainder.value last = pyxl_tokens[-1] pyxl_tokens[-1] = Token(last.ttype, last.value[:-len(remainder[1])], last.start, remainder.start, last.line) pyxl_parser_start = Pos(*pyxl_parser.start) if invertible: # We want to include a trailing comment after a pyxl close in # the pyxl block itself to prevent black from migrating it. # (which matters a lot if it is a type: ignore.) try: peek = fix_token(next(tokens)) except StopIteration: pass else: if peek.ttype == tokenize.COMMENT: pyxl_tokens.append(peek) else: tokens.unshift(peek) output = "html.PYXL('''{}''', {}, {}, {}{}{})".format( untokenize(pyxl_tokens).replace('\\', '\\\\').replace("'", "\\'"), # Include the real compiled pyxl so that tools can see all the gritty details untokenize([pyxl_parser.get_token()]), # Include the start column so we can shift it if needed pyxl_parser_start.col, # Include the columns of each python fragment so we can shift them if needed ''.join([ str(first_non_ws_token(x).start.col) + ', ' for x in python_fragments ]), # When untokenizing python fragments, make sure to place them in their # proper columns so that we don't detect a shift if there wasn't one. ''.join( [untokenize_with_column(x) + ', ' for x in python_fragments]), # Stick a final argument at the end so that all the real arguments are # always terminated by commas (and don't pick up spurious whitespace # with certain black formatting modes) '0', ) return Token(tokenize.STRING, output, pyxl_parser_start, Pos(*pyxl_parser.end), '') else: return fix_token(pyxl_parser.get_token())