Exemple #1
0
def generate_lib2to3_ast(code):
    from lib2to3.pgen2.driver import Driver
    from lib2to3.pgen2 import token as pgen2_token
    from lib2to3.pygram import python_symbols, python_grammar
    from lib2to3 import pytree
    from io import StringIO

    token_types = list(python_symbols.__dict__.items())
    token_types += list(pgen2_token.__dict__.items())

    def transform_ast(ast):
        transformed = {
            "node_type": next(n for n, t in token_types if t == ast.type)
        }
        if ast.children:
            transformed["children"] = [
                transform_ast(child) for child in ast.children
            ]
        if isinstance(ast, pytree.Leaf):
            if ast.value != "":
                transformed["value"] = ast.value
            if ast._prefix != "":
                transformed["prefix"] = ast._prefix
        return transformed

    driver = Driver(python_grammar, convert=pytree.convert)
    return transform_ast(driver.parse_stream(StringIO(code)))
Exemple #2
0
def parse(file):
    encoding = detect_encoding(file)
    file = codecs.lookup(encoding).streamreader(file)
    source = file.read()
    source += u'\n'  # necessary to fix weird parsing error
    features = _detect_future_features(source)
    if u'print_function' in features:
        grammar = pygram.python_grammar_no_print_statement
    else:
        grammar = pygram.python_grammar
    driver = Driver(grammar, convert=pytree.convert)
    return driver.parse_string(source)
Exemple #3
0
def parse(source):
    """String -> AST

    Parse the string and return its AST representation. May raise
    a ParseError exception.
    """
    # Modified from
    # https://gist.github.com/FZambia/876b724c329e864b6642adc52b577cdb
    drv = Driver(pygram.python_grammar, pytree.convert)
    result = drv.parse_string(source + "\n", True)
    if isinstance(result, Leaf):  # Always return a Node, not a Leaf.
        result = Node(pygram.python_symbols.file_input, [result])
    # Could track whether str() needs to remove the newline, but not worth it.
    return result
def generate_lib2to3_ast(code):
    from lib2to3.pgen2.driver import Driver
    from lib2to3.pgen2 import token as pgen2_token
    from lib2to3.pygram import python_symbols, python_grammar
    from lib2to3 import pytree
    from io import StringIO

    token_types = list(python_symbols.__dict__.items())
    token_types += list(pgen2_token.__dict__.items())

    def transform_ast(ast):
        transformed = {"node_type": next(n for n, t in token_types if t == ast.type)}
        if ast.children:
            transformed["children"] = [transform_ast(child) for child in ast.children]
        if isinstance(ast, pytree.Leaf):
            if ast.value != "":
                transformed["value"] = ast.value
            if ast._prefix != "":
                transformed["prefix"] = ast._prefix
        return transformed

    driver = Driver(python_grammar, convert=pytree.convert)
    return transform_ast(driver.parse_stream(StringIO(code)))
Exemple #5
0
class Flake8Checker(object):
    name = __name__
    version = __version__

    def __init__(self, tree, filename):
        self._filename = filename

    def run(self):
        errors = _process_file(self._filename)
        for line, column, error_code in errors:
            yield (line, column, '%s %s' % (error_code.name, error_code.value),
                   type(self))


_driver = Driver(
    grammar=python_grammar_no_print_statement,
    convert=pytree.convert,
)


def _process_file(filename):
    if filename == 'stdin':
        code = pycodestyle.stdin_get_value()
    else:
        with open(filename, 'rt') as f:
            code = f.read()
    return _process_code(code)


def _process_code(code):
    tree = _driver.parse_string(code)
    return _process_tree(tree)
Exemple #6
0
def suite(text):
    d = Driver(g)
    return d.parse_string(text)
Exemple #7
0
def suite(text):
    d = Driver(g )
    return d.parse_string(text)
Exemple #8
0
from lib2to3.pgen2.tokenize import generate_tokens, TokenError
from lib2to3.pgen2.driver import Driver
from lib2to3.pgen2.parse import ParseError
from lib2to3.pygram import python_grammar_no_print_statement
from lib2to3.pytree import convert as convert_tree
import logging
import itertools

import argparse

argparser = argparse.ArgumentParser(description='Retrieve a source file for parsing.')
argparser.add_argument('file', type=str, help='path to file')
args = argparser.parse_args()


driver = Driver(python_grammar_no_print_statement,
                convert=convert_tree)
                
                
with open(args.file) as f:

    def readline():
        while True:
            line = f.readline()
            
            if line.endswith('\n'):
                yield line
            else:
                yield line+'\n'
                yield ''
                break
Exemple #9
0
def parse_py(src_txt):
    drv = Driver(grammar, pytree.convert)
    result = drv.parse_string(src_txt, True)
    if isinstance(result, Leaf):
        result = Node(pygram.python_symbols.file_input, [result])
    return result