def parse_expression(input):
logger = logging.getLogger('parse_expression')
logger.debug('parse_expression(\'{input}\')'.format(input=input))
# Parse the input into an abstract syntax tree using the phython ast module.
module = ast.parse(input)
logger.debug('Input parsed as:\n{ast}'.format(
ast=astpretty.pformat(module, indent=' ', show_offsets=False)))
# The grammar used wraps the code in modules and a body array. In the context
# of this toy we can only process single expressions so we unwrap that and
# give up if it does not contain what we expect.
if not isinstance(module, ast.Module):
raise GvSError('Input does not parse as ast.Module')
if len(module.body) < 1:
raise GvSError('Module body is empty')
if len(module.body) > 1:
raise GvSError('Multiple module bodies')
if not isinstance(module.body[0], ast.Expr):
raise GvSError('Input does not parse as expression')
expr = module.body[0].value
# Expr now is the AST for a single expression. We will attempt
# to create code for the target machine from it. If the expression
# uses anything beyond the simplest of operators and operands that
# will fail and we give up.
logger.debug('Expression extracted as:\n{ast}'.format(
ast=astpretty.pformat(expr, indent=' ', show_offsets=False)))
return expr
def test_file(path):
with open(path) as f:
expected_ast = astpretty.pformat(ast.parse(f.read()), show_offsets=False)
before = time.time()
printer_output = subprocess.check_output(['./target/release/prettyprint', path])
total_time = time.time() - before
try:
received_ast = astpretty.pformat(ast.parse(printer_output), show_offsets=False)
except:
print('Error while parsing the output from {}:'.format(path))
raise
if expected_ast == received_ast:
print('({:03}ms) {}: ok'.format(int(total_time*1000), path))
return
print('========================')
print(path)
print('------------------------')
#for line in difflib.unified_diff(received_ast, expected_ast): # OMG so slow
# print(line)
with tempfile.NamedTemporaryFile('w+', prefix='expected-') as exp_file, \
tempfile.NamedTemporaryFile('w+', prefix='received-') as received_file:
exp_file.write(expected_ast)
exp_file.seek(0)
received_file.write(received_ast)
received_file.seek(0)
try:
subprocess.check_output(
['diff', '-u', received_file.name, exp_file.name],
universal_newlines=True)
except subprocess.CalledProcessError as e:
print(e.output)
else:
assert False, 'diff did not detect a different, but should have.'
print('========================')
exit(1)
def visit_If(self, node):
logger.debug('visit_If to remove it: ' + astpretty.pformat(node))
logger.debug('removed: ' +
astpretty.pformat(node.test, show_offsets=True))
for e in node.body:
logger.debug('removed: ' + astpretty.pformat(e))
return [self.visit(node.test)] + [self.visit(x) for x in node.body] + [
self.visit(x) for x in node.orelse
]
def visit_Call(self, node):
logger.debug('Call fullname:')
logger.info('Call node:' + astpretty.pformat(node))
# handle callsomething
if isinstance(node.func, ast.Name):
funcname = node.func.id
# get ClassDef if there have one
clssdef = self.table[funcname] if funcname in self.table.keys(
) else None
if isinstance(clssdef, ast.ClassDef):
logger.debug('class def get from global name binding: ' +
clssdef.name)
self.edges[node._upper._fullname].append(
namejoin(clssdef.name, '__init__'))
return clssdef
# handle self.callsomething()
elif isinstance(node.func, ast.Attribute):
func = node.func
method = func.attr
inst = func.value.id if isinstance(func.value, ast.Name) else ''
if inst == 'self':
clssnode = self.locate_self(node)
upper = node._upper if hasattr(node, '_upper') else None
if upper is not None:
self.edges[node._upper._fullname].append(
namejoin(clssnode._fullname, method))
elif len(inst) > 0:
realname = self.getrealname(inst, node)
clss = self.table.get(realname, None)
if isinstance(clss, ast.ClassDef):
# invoke the __init__() of class
self.edges[node._upper._fullname].append(
namejoin(clss.name, method))
def test_pformat_nested_node_without_line_information():
expected_38 = (
'Subscript(\n'
' lineno=1,\n'
' col_offset=0,\n'
' end_lineno=1,\n'
' end_col_offset=4,\n'
" value=Name(lineno=1, col_offset=0, end_lineno=1, end_col_offset=1, id='a', ctx=Load()),\n" # noqa: E501
' slice=Index(\n'
' value=Constant(lineno=1, col_offset=2, end_lineno=1, end_col_offset=3, value=0, kind=None),\n' # noqa: E501
' ),\n'
' ctx=Load(),\n'
')')
expected_lt38 = (
'Subscript(\n'
' lineno=1,\n'
' col_offset=0,\n'
" value=Name(lineno=1, col_offset=0, id='a', ctx=Load()),\n"
' slice=Index(\n'
' value=Num(lineno=1, col_offset=2, n=0),\n'
' ),\n'
' ctx=Load(),\n'
')')
expected = expected_38 if sys.version_info >= (3, 8) else expected_lt38
ret = astpretty.pformat(_to_expr_value('a[0]'))
assert ret == expected
def save_trees(args=None):
dst: Path = args["dst"]
trees = args["trees"]
dst_full = OUT_PATH.joinpath(dst)
dst_full.parent.mkdir(parents=True, exist_ok=True)
dst_full.touch(exist_ok=False)
# TODO: append "doctest.testmod(raise_on_error=True)"
trees = [ast.fix_missing_locations(tree) for tree in trees]
if SHOULD_SAVE_AST:
new_txt = "\n".join([str(astpretty.pformat(tree)) for tree in trees])
new_txt = f"""from ast import *
{new_txt}
"""
dst_full.with_suffix(".ast.py").write_text(new_txt)
new_txt = ""
if dst.name.startswith("test_"):
if "compatible" in str(dst):
new_txt += f"""
import {COMPATIBLE_MODULE}.unittest
"""
else:
new_txt += """
import oneflow.unittest
"""
new_txt += "\n".join([ast.unparse(tree) for tree in trees])
dst_full.write_text(new_txt)
def test_pformat_nested_attr_empty_list():
ret = astpretty.pformat(_to_module_body('if x: pass'), show_offsets=False)
assert ret == ('If(\n'
" test=Name(id='x', ctx=Load()),\n"
' body=[Pass()],\n'
' orelse=[],\n'
')')
def test_pformat_mixed_sub_nodes_and_primitives():
node = _to_module_body('from y import x')
ret = astpretty.pformat(node, show_offsets=False)
assert ret == ('ImportFrom(\n'
" module='y',\n"
" names=[alias(name='x', asname=None)],\n"
' level=0,\n'
')')
def visit_Attribute(self, node):
# we care if we're loading from this
if isinstance(node.ctx, ast.Load):
print("ctx:",self.context)
x = self.context[-1]
print("ctx", "\nctx: ".join(
astpretty.pformat(self.context[-1]).split("\n")))
#print("ctx:", astpretty.pprint(self.context[-1]))
self.generic_visit(node)
def test_pformat_nested():
ret = astpretty.pformat(_to_module_body('x = 5'))
assert ret == (
'Assign(\n'
' lineno=1,\n'
' col_offset=0,\n'
" targets=[Name(lineno=1, col_offset=0, id='x', ctx=Store())],\n"
' value=Num(lineno=1, col_offset=4, n=5),\n'
')')
def test_pformat_nested_attr_empty_list():
ret = astpretty.pformat(_to_module_body('if 1: pass'))
assert ret == ('If(\n'
' lineno=1,\n'
' col_offset=0,\n'
' test=Num(lineno=1, col_offset=3, n=1),\n'
' body=[Pass(lineno=1, col_offset=6)],\n'
' orelse=[],\n'
')')
def test_pformat_mixed_sub_nodes_and_primitives():
ret = astpretty.pformat(_to_module_body('from y import x'))
assert ret == ('ImportFrom(\n'
' lineno=1,\n'
' col_offset=0,\n'
" module='y',\n"
" names=[alias(name='x', asname=None)],\n"
' level=0,\n'
')')
def is_range_len(node: ast.For):
try:
logging.debug("check")
logging.debug(pformat(node))
return node.iter.func.id == 'range' and node.iter.args[
0].func.id == 'len'
except AttributeError as e:
logging.debug("attr error!" + str(e))
return False
def test_pformat_nested_multiple_elements():
ret = astpretty.pformat(_to_expr_value('[a, b, c]'), show_offsets=False)
assert ret == ('List(\n'
' elts=[\n'
" Name(id='a', ctx=Load()),\n"
" Name(id='b', ctx=Load()),\n"
" Name(id='c', ctx=Load()),\n"
' ],\n'
' ctx=Load(),\n'
')')
def visit(self, node):
try:
try:
self.lineno = node.lineno
except Exception as ex1:
pass
super().visit(node)
except Exception as ex:
raise ex
self.logger.error(_(str(ex), type=ex.__class__.__name__, pretty=astpretty.pformat(node)))
exit(1)
def test_pformat_integer_indent():
node = _to_expr_value('[a, b, c]')
ret = astpretty.pformat(node, indent=3, show_offsets=False)
assert ret == ('List(\n'
' elts=[\n'
" Name(id='a', ctx=Load()),\n"
" Name(id='b', ctx=Load()),\n"
" Name(id='c', ctx=Load()),\n"
' ],\n'
' ctx=Load(),\n'
')')
def test_pformat_custom_indent():
node = _to_expr_value('[a, b, c]')
ret = astpretty.pformat(node, indent='\t', show_offsets=False)
assert ret == ('List(\n'
'\telts=[\n'
"\t\tName(id='a', ctx=Load()),\n"
"\t\tName(id='b', ctx=Load()),\n"
"\t\tName(id='c', ctx=Load()),\n"
'\t],\n'
'\tctx=Load(),\n'
')')
def test_pformat_nested_multiple_elements():
ret = astpretty.pformat(_to_expr_value('[1, 2, 3]'))
assert ret == ('List(\n'
' lineno=1,\n'
' col_offset=0,\n'
' elts=[\n'
' Num(lineno=1, col_offset=1, n=1),\n'
' Num(lineno=1, col_offset=4, n=2),\n'
' Num(lineno=1, col_offset=7, n=3),\n'
' ],\n'
' ctx=Load(),\n'
')')
def test_pformat_nested_node_without_line_information():
ret = astpretty.pformat(_to_expr_value('a[0]'))
assert ret == (
'Subscript(\n'
' lineno=1,\n'
' col_offset=0,\n'
" value=Name(lineno=1, col_offset=0, id='a', ctx=Load()),\n"
' slice=Index(\n'
' value=Num(lineno=1, col_offset=2, n=0),\n'
' ),\n'
' ctx=Load(),\n'
')')
def test_pformat_custom_indent():
ret = astpretty.pformat(_to_expr_value('[1, 2, 3]'), indent='\t')
assert ret == ('List(\n'
'\tlineno=1,\n'
'\tcol_offset=0,\n'
'\telts=[\n'
'\t\tNum(lineno=1, col_offset=1, n=1),\n'
'\t\tNum(lineno=1, col_offset=4, n=2),\n'
'\t\tNum(lineno=1, col_offset=7, n=3),\n'
'\t],\n'
'\tctx=Load(),\n'
')')
def main():
"""
This function represents the main logic of the program. It scans through a directory, and creates an
Abstract Syntax Tree for each python file it finds, and outputs it to a new file.
The system prematurely exits if there is no directory given, if the given directory does not exist or is a file,
or if no python files could be found from the directory.
Parameters
------------
sys.argv[1] : str
the directory to search through in the program
"""
# if no arugments were given to the function, exit prematurely
if len(sys.argv) < 2:
print("Missing Arguments Error: Please provide a path to a directory.")
sys.exit(1)
# set the directory to the first argument given
rootpath = sys.argv[1]
# determine if the directory is valid to use or not
if not FileHandler.validateDirectory(rootpath):
sys.exit(1)
# get all pyhton files found from the directory
python_files = FileHandler.getPythonFiles(rootpath)
# if the directory and its subdirectories contain no python files, alert the user and exit.
if len(python_files) == 0:
print("No python files could be found from the directory given.")
sys.exit(0)
# create an AST for each python file and export the tree
for file in python_files:
# open the file and extract the contents
file_contents = FileHandler.getFileContents(file)
try:
# parse and create the AST from the file contents
head_node = ast.parse(file_contents)
# use astpretty to create detailed output form of the AST
output_string = astpretty.pformat(head_node,
indent=" ",
show_offsets=False)
# output this AST to a new file
FileHandler.outputNewFile(file, output_string)
except SyntaxError as err:
# in case the parser cannot parse due to a syntax error, alert the user
print(
"Error: Syntax Error found while parsing file " + file +
" -- ", err)
def test_pformat_py35_regression():
expected = ('Dict(\n'
' keys=[\n'
" Name(id='a', ctx=Load()),\n"
' None,\n'
' ],\n'
' values=[\n'
" Name(id='b', ctx=Load()),\n"
" Name(id='k', ctx=Load()),\n"
' ],\n'
')')
s = '{a: b, **k}'
assert astpretty.pformat(_to_expr_value(s), show_offsets=False) == expected
def oldvisit_Compare(self, node):
if len(node.ops) != 1:
expr = { 'type': 'TSUndefinedKeyword' }
self.cur_node = expr
self.collect_output_node(expr)
self.generic_visit(node)
return
assert len(node.ops) == 1, 'node ops 1'
# fix me!
op = node.ops[0]
operator = None
negate = False
if isinstance(op, (ast.Eq, ast.Is)):
operator = '==='
elif isinstance(op, (ast.NotEq, ast.IsNot)):
operator = '!=='
elif isinstance(op, (ast.In)):
operator = 'in'
elif isinstance(op, (ast.Gt)):
operator = '>'
elif isinstance(op, (ast.Lt)):
operator = '<'
elif isinstance(op, (ast.LtE)):
operator = '<='
elif isinstance(op, (ast.GtE)):
operator = '>='
elif isinstance(op, (ast.NotEq)):
operator = '!=='
elif isinstance(op, (ast.NotIn)):
negate = True
operator = 'in'
else:
print(astpretty.pformat(node))
exit(2)
comparator = node.comparators[0]
v = ValueCollector("compator", True, parent=self)
v.do_visit(comparator)
comparator = v.finished_output_nodes[-1].pop()
v2 = ValueCollector("left", True, parent=self)
v2.do_visit(node.left)
left = v2.finished_output_nodes[-1].pop()
expr = { 'type': 'BinaryExpression', 'operator': operator, 'left': left, 'right': comparator, 'comments': comments_for(node) }
if negate:
expr = { 'type': 'UnaryExpression', 'operator': '!', 'prefix': True, 'operand': expr }
self.cur_node = expr
self.collect_output_node(expr)
self.generic_visit(node, False)
def test_pformat_py35_regression():
expected = ('Dict(\n'
' lineno=1,\n'
' col_offset=0,\n'
' keys=[\n'
' Num(lineno=1, col_offset=1, n=1),\n'
' None,\n'
' ],\n'
' values=[\n'
' Num(lineno=1, col_offset=4, n=2),\n'
" Name(lineno=1, col_offset=9, id='k', ctx=Load()),\n"
' ],\n'
')')
assert astpretty.pformat(_to_expr_value('{1: 2, **k}')) == expected
def keystep_node(tree,fArg):
try:
treePrintout=astpretty.pformat(tree)
except Exception as e:
treePrintout=ast.dump(tree)
fArgString=pp.pformat(fArg)
return Call(func=PRINT_AND_EVAL_NODE,
args=[tree,Str(s=treePrintout),Str(s=fArgString)],
keywords=[])
def hls_debug(msg='', title=None, indent=0):
if not hls_debug_log_enabled():
return None
if title is not None:
hls_debug_header(title)
if isinstance(msg, dict):
msg = pprint.pformat(msg)
elif isinstance(msg, ast.AST):
import astpretty
msg = astpretty.pformat(msg)
else:
msg = str(msg)
if title is not None:
msg = textwrap.indent(msg, ' ')
hls_log().debug(textwrap.indent(msg, ' ' * indent))
def test_pformat_nested_with_offsets():
expected_38 = (
'Assign(\n'
' lineno=1,\n'
' col_offset=0,\n'
' end_lineno=1,\n'
' end_col_offset=5,\n'
" targets=[Name(lineno=1, col_offset=0, end_lineno=1, end_col_offset=1, id='x', ctx=Store())],\n" # noqa: E501
' value=Constant(lineno=1, col_offset=4, end_lineno=1, end_col_offset=5, value=5, kind=None),\n' # noqa: E501
' type_comment=None,\n'
')')
expected_lt38 = (
'Assign(\n'
' lineno=1,\n'
' col_offset=0,\n'
" targets=[Name(lineno=1, col_offset=0, id='x', ctx=Store())],\n"
' value=Num(lineno=1, col_offset=4, n=5),\n'
')')
expected = expected_38 if sys.version_info >= (3, 8) else expected_lt38
ret = astpretty.pformat(_to_module_body('x = 5'))
assert ret == expected
def visit_Module(self, node):
self.write("\nvisit_Module ", mynote=1)
# Dump source code to logh file
self.logh.out_wrap_in_html(
add_line_numbers(node.root.source_code),
style_class="dump1",
heading="Module Source Code...",
)
# Dump ast structure to logh file
s = astpretty.pformat(node.root) # .root is a property I added to each node of the ast tree
if sys.version_info < (3, 0):
s = s.encode("utf-8") # unicode to str
self.logh.out_wrap_in_html(
s, style_class="dump_ast", heading="AST..."
) # better than self.write(s, mynote=1)
self.generic_visit(node) # need this to keep the visiting going...
# After whole module is done...
self.write("visit_Module complete", mynote=1)
def test_pformat_nested_no_offsets():
ret = astpretty.pformat(_to_module_body('x = 5'), show_offsets=False)
assert ret == ('Assign(\n'
" targets=[Name(id='x', ctx=Store())],\n"
' value=Num(n=5),\n'
')')
def test_pformat_node():
ret = astpretty.pformat(_to_expr_value('x'))
assert ret == "Name(lineno=1, col_offset=0, id='x', ctx=Load())"