Python PyToken.PyToken Examples

Example #1

def get_chunks(code) -> Generator[Chunk, None, None]:
    g = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)
    chunk = Chunk()

    try:
        for item in g:
            t = PyToken(item)
            reuse = chunk.append(t)

            if chunk.complete:

                yield chunk
                chunk = Chunk()
                if reuse:
                    reuse = chunk.append(t)
                    # assert not reuse
                    if chunk.complete:
                        yield chunk
                        chunk = Chunk()

        yield chunk
    except tokenize.TokenError as e:
        if state.verbose:
            traceback.print_exc()
            print(e)

Example #2

def get_quote_type(code: str):
    from flynt.lexer.PyToken import PyToken
    g = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)
    next(g)
    token = PyToken(next(g))

    return token.get_quote_type()

Example #3

File: split.py Project: peterjc/flynt

def get_chunks(code) -> Generator[Chunk, None, None]:
    g = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)
    chunk = Chunk()

    for item in g:
        t = PyToken(item)
        reuse = chunk.append(t)

        if chunk.complete:

            yield chunk
            chunk = Chunk()
            if reuse:
                reuse = chunk.append(t)
                # assert not reuse
                if chunk.complete:
                    yield chunk
                    chunk = Chunk()

    yield chunk