def generate_lib2to3_ast(code):
from lib2to3.pgen2.driver import Driver
from lib2to3.pgen2 import token as pgen2_token
from lib2to3.pygram import python_symbols, python_grammar
from lib2to3 import pytree
from io import StringIO
token_types = list(python_symbols.__dict__.items())
token_types += list(pgen2_token.__dict__.items())
def transform_ast(ast):
transformed = {
"node_type": next(n for n, t in token_types if t == ast.type)
}
if ast.children:
transformed["children"] = [
transform_ast(child) for child in ast.children
]
if isinstance(ast, pytree.Leaf):
if ast.value != "":
transformed["value"] = ast.value
if ast._prefix != "":
transformed["prefix"] = ast._prefix
return transformed
driver = Driver(python_grammar, convert=pytree.convert)
return transform_ast(driver.parse_stream(StringIO(code)))
def parse(file):
encoding = detect_encoding(file)
file = codecs.lookup(encoding).streamreader(file)
source = file.read()
source += u'\n' # necessary to fix weird parsing error
features = _detect_future_features(source)
if u'print_function' in features:
grammar = pygram.python_grammar_no_print_statement
else:
grammar = pygram.python_grammar
driver = Driver(grammar, convert=pytree.convert)
return driver.parse_string(source)
def parse(source):
"""String -> AST
Parse the string and return its AST representation. May raise
a ParseError exception.
"""
# Modified from
# https://gist.github.com/FZambia/876b724c329e864b6642adc52b577cdb
drv = Driver(pygram.python_grammar, pytree.convert)
result = drv.parse_string(source + "\n", True)
if isinstance(result, Leaf): # Always return a Node, not a Leaf.
result = Node(pygram.python_symbols.file_input, [result])
# Could track whether str() needs to remove the newline, but not worth it.
return result
def generate_lib2to3_ast(code):
from lib2to3.pgen2.driver import Driver
from lib2to3.pgen2 import token as pgen2_token
from lib2to3.pygram import python_symbols, python_grammar
from lib2to3 import pytree
from io import StringIO
token_types = list(python_symbols.__dict__.items())
token_types += list(pgen2_token.__dict__.items())
def transform_ast(ast):
transformed = {"node_type": next(n for n, t in token_types if t == ast.type)}
if ast.children:
transformed["children"] = [transform_ast(child) for child in ast.children]
if isinstance(ast, pytree.Leaf):
if ast.value != "":
transformed["value"] = ast.value
if ast._prefix != "":
transformed["prefix"] = ast._prefix
return transformed
driver = Driver(python_grammar, convert=pytree.convert)
return transform_ast(driver.parse_stream(StringIO(code)))
class Flake8Checker(object):
name = __name__
version = __version__
def __init__(self, tree, filename):
self._filename = filename
def run(self):
errors = _process_file(self._filename)
for line, column, error_code in errors:
yield (line, column, '%s %s' % (error_code.name, error_code.value),
type(self))
_driver = Driver(
grammar=python_grammar_no_print_statement,
convert=pytree.convert,
)
def _process_file(filename):
if filename == 'stdin':
code = pycodestyle.stdin_get_value()
else:
with open(filename, 'rt') as f:
code = f.read()
return _process_code(code)
def _process_code(code):
tree = _driver.parse_string(code)
return _process_tree(tree)
def suite(text):
d = Driver(g)
return d.parse_string(text)
from lib2to3.pgen2.tokenize import generate_tokens, TokenError
from lib2to3.pgen2.driver import Driver
from lib2to3.pgen2.parse import ParseError
from lib2to3.pygram import python_grammar_no_print_statement
from lib2to3.pytree import convert as convert_tree
import logging
import itertools
import argparse
argparser = argparse.ArgumentParser(description='Retrieve a source file for parsing.')
argparser.add_argument('file', type=str, help='path to file')
args = argparser.parse_args()
driver = Driver(python_grammar_no_print_statement,
convert=convert_tree)
with open(args.file) as f:
def readline():
while True:
line = f.readline()
if line.endswith('\n'):
yield line
else:
yield line+'\n'
yield ''
break
def parse_py(src_txt):
drv = Driver(grammar, pytree.convert)
result = drv.parse_string(src_txt, True)
if isinstance(result, Leaf):
result = Node(pygram.python_symbols.file_input, [result])
return result
