Example #1
0
def get_pyxl_token(start_token, tokens):
    ttype, tvalue, tstart, tend, tline = start_token
    pyxl_parser = PyxlParser(tstart[0], tstart[1])
    pyxl_parser.feed(start_token)

    for token in tokens:
        ttype, tvalue, tstart, tend, tline = token

        if tvalue and tvalue[0] == '{':
            if pyxl_parser.python_mode_allowed():
                mid, right = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, mid)
                pyxl_parser.feed_position_only((ttype, mid, tstart, division, tline))
                tokens.rewind_and_retokenize((ttype, right, division, tend, tline))
                python_tokens = list(transform_tokens(tokens))

                close_curly = next(tokens)
                ttype, tvalue, tstart, tend, tline = close_curly
                close_curly_sub = (ttype, '', tend, tend, tline)

                pyxl_parser.feed_python(python_tokens + [close_curly_sub])
                continue
            # else fallthrough to pyxl_parser.feed(token)
        elif tvalue and ttype == tokenize.COMMENT:
            if not pyxl_parser.python_comment_allowed():
                tvalue, rest = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, tvalue)
                tokens.unshift((tokenize.ERRORTOKEN, rest, division, tend, tline))
                token = ttype, tvalue, tstart, division, tline
                # fallthrough to pyxl_parser.feed(token)
            else:
                pyxl_parser.feed_comment(token)
                continue
        elif tvalue and tvalue[0] == '#':
            # let the python tokenizer grab the whole comment token
            tokens.rewind_and_retokenize(token)
            continue
        else:
            sp = re.split('([#{])', tvalue, maxsplit=1)
            if len(sp) > 1:
                tvalue, mid, right = sp
                division = get_end_pos(tstart, tvalue)
                tokens.unshift((ttype, mid+right, division, tend, tline))
                token = ttype, tvalue, tstart, division, tline
                # fallthrough to pyxl_parser.feed(token)

        pyxl_parser.feed(token)

        if pyxl_parser.done(): break

    if not pyxl_parser.done():
        lines = ['<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
                 for tag_info in pyxl_parser.open_tags]
        raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))

    remainder = pyxl_parser.get_remainder()
    if remainder:
        tokens.rewind_and_retokenize(remainder)

    return pyxl_parser.get_token()
Example #2
0
def get_pyxl_token(start_token, tokens):
    ttype, tvalue, tstart, tend, tline = start_token
    pyxl_parser = PyxlParser(tstart[0], tstart[1])
    pyxl_parser.feed(start_token)

    for token in tokens:
        ttype, tvalue, tstart, tend, tline = token

        if tvalue and tvalue[0] == '{':
            if pyxl_parser.python_mode_allowed():
                mid, right = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, mid)
                pyxl_parser.feed_position_only(
                    (ttype, mid, tstart, division, tline))
                tokens.rewind_and_retokenize(
                    (ttype, right, division, tend, tline))
                python_tokens = list(transform_tokens(tokens))

                close_curly = tokens.next()
                ttype, tvalue, tstart, tend, tline = close_curly
                close_curly_sub = (ttype, '', tend, tend, tline)

                pyxl_parser.feed_python(python_tokens + [close_curly_sub])
                continue
            # else fallthrough to pyxl_parser.feed(token)
        elif tvalue and ttype == tokenize.COMMENT:
            if not pyxl_parser.python_comment_allowed():
                tvalue, rest = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, tvalue)
                tokens.unshift(
                    (tokenize.ERRORTOKEN, rest, division, tend, tline))
                token = ttype, tvalue, tstart, division, tline
                # fallthrough to pyxl_parser.feed(token)
            else:
                pyxl_parser.feed_comment(token)
                continue
        elif tvalue and tvalue[0] == '#':
            # let the python tokenizer grab the whole comment token
            tokens.rewind_and_retokenize(token)
            continue
        else:
            sp = re.split('([#{])', tvalue, maxsplit=1)
            if len(sp) > 1:
                tvalue, mid, right = sp
                division = get_end_pos(tstart, tvalue)
                tokens.unshift((ttype, mid + right, division, tend, tline))
                token = ttype, tvalue, tstart, division, tline
                # fallthrough to pyxl_parser.feed(token)

        pyxl_parser.feed(token)

        if pyxl_parser.done(): break

    if not pyxl_parser.done():
        lines = [
            '<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
            for tag_info in pyxl_parser.open_tags
        ]
        raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))

    remainder = pyxl_parser.get_remainder()
    if remainder:
        tokens.rewind_and_retokenize(remainder)

    return pyxl_parser.get_token()
Example #3
0
def get_pyxl_token(start_token, tokens, invertible, str_function):
    ttype, tvalue, tstart, tend, tline = start_token
    pyxl_parser = PyxlParser(tstart.row, tstart.col, str_function)
    pyxl_parser.feed(start_token)

    if invertible:
        # In invertible mode, keep track of all the tokens we see in pyxl and
        # each python fragment that appears
        pyxl_tokens = [start_token]
        python_fragments = []

    for token in tokens:
        ttype, tvalue, tstart, tend, tline = token

        if tvalue and tvalue[0] == '{':
            if pyxl_parser.python_mode_allowed():
                # We've hit a python fragment
                initial_tstart = tstart

                mid, right = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, mid)
                pyxl_parser.feed_position_only(
                    Token(ttype, mid, tstart, division, tline))
                tokens.rewind_and_retokenize(
                    Token(ttype, right, division, tend, tline))
                python_tokens = list(
                    transform_tokens(tokens, invertible, str_function))

                close_curly = next(tokens)
                ttype, tvalue, tstart, tend, tline = close_curly
                close_curly_sub = Token(ttype, '', tend, tend, tline)

                # strip comments in the invertible version
                pyxl_parser.feed_python((strip_comments(python_tokens)
                                         if invertible else python_tokens) +
                                        [close_curly_sub])

                if invertible:
                    # If we are doing invertible generation, put in a format placeholder
                    # into the collected pyxl tokens and collect python fragment separately.

                    # We keep each fragment wrapped in the curlies it came in (because just
                    # relying on the commas) doesn't work when there are undelimited commas.
                    # This also serves to preserve any internal whitespace in the fragment so
                    # it can be restored later.
                    open_curly = Token(ttype, '{', initial_tstart, division,
                                       tline)

                    pyxl_tokens.append(
                        Token(ttype, '{}', initial_tstart, tend, ''))

                    python_fragments.append([open_curly] + python_tokens +
                                            [close_curly])

                continue
            # else fallthrough to pyxl_parser.feed(token)
        elif tvalue and ttype == tokenize.COMMENT:
            if not pyxl_parser.python_comment_allowed():
                tvalue, rest = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, tvalue)
                tokens.unshift(
                    Token(tokenize.ERRORTOKEN, rest, division, tend, tline))
                token = Token(ttype, tvalue, tstart, division, tline)
                # fallthrough to pyxl_parser.feed(token)
            else:
                # strip comments in the invertible version
                if not invertible:
                    pyxl_parser.feed_comment(token)
                if invertible:
                    pyxl_tokens.append(sanitize_token(token))
                continue
        elif tvalue and tvalue[0] == '#':
            # let the python tokenizer grab the whole comment token
            tokens.rewind_and_retokenize(token)
            continue
        else:
            sp = re.split('([#{])', tvalue, maxsplit=1)
            if len(sp) > 1:
                tvalue, mid, right = sp
                division = get_end_pos(tstart, tvalue)
                tokens.unshift(Token(ttype, mid + right, division, tend,
                                     tline))
                token = Token(ttype, tvalue, tstart, division, tline)
                # fallthrough to pyxl_parser.feed(token)

        pyxl_parser.feed(token)
        if invertible:
            pyxl_tokens.append(sanitize_token(token))

        if pyxl_parser.done(): break

    if not pyxl_parser.done():
        lines = [
            '<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
            for tag_info in pyxl_parser.open_tags
        ]
        raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))

    remainder = pyxl_parser.get_remainder()
    if remainder:
        remainder = fix_token(remainder)
        tokens.rewind_and_retokenize(remainder)
        # Strip the remainder out from the last seen token
        if invertible and remainder.value:
            assert '{' not in remainder.value and '}' not in remainder.value
            last = pyxl_tokens[-1]
            pyxl_tokens[-1] = Token(last.ttype,
                                    last.value[:-len(remainder[1])],
                                    last.start, remainder.start, last.line)

    pyxl_parser_start = Pos(*pyxl_parser.start)
    if invertible:
        # We want to include a trailing comment after a pyxl close in
        # the pyxl block itself to prevent black from migrating it.
        # (which matters a lot if it is a type: ignore.)
        try:
            peek = fix_token(next(tokens))
        except StopIteration:
            pass
        else:
            if peek.ttype == tokenize.COMMENT:
                pyxl_tokens.append(peek)
            else:
                tokens.unshift(peek)

        output = "html.PYXL('''{}''', {}, {}, {}{}{})".format(
            untokenize(pyxl_tokens).replace('\\', '\\\\').replace("'", "\\'"),
            # Include the real compiled pyxl so that tools can see all the gritty details
            untokenize([pyxl_parser.get_token()]),
            # Include the start column so we can shift it if needed
            pyxl_parser_start.col,
            # Include the columns of each python fragment so we can shift them if needed
            ''.join([
                str(first_non_ws_token(x).start.col) + ', '
                for x in python_fragments
            ]),
            # When untokenizing python fragments, make sure to place them in their
            # proper columns so that we don't detect a shift if there wasn't one.
            ''.join(
                [untokenize_with_column(x) + ', ' for x in python_fragments]),
            # Stick a final argument at the end so that all the real arguments are
            # always terminated by commas (and don't pick up spurious whitespace
            # with certain black formatting modes)
            '0',
        )
        return Token(tokenize.STRING, output, pyxl_parser_start,
                     Pos(*pyxl_parser.end), '')
    else:
        return fix_token(pyxl_parser.get_token())