def test_ast_tree_edit_distance() -> None:
node1 = ast.parse("a=1")
node2 = ast.parse("a=1")
assert ast_tree_edit_distance(node1, node2) == 0
node2 = ast.parse("b=1")
assert ast_tree_edit_distance(node1, node2) == 1
node2 = ast.parse("b=2")
assert ast_tree_edit_distance(node1, node2) == 1.5
node1 = Assign(
targets=[Name(id="a", ctx=Store())],
value=Constant(value=1, kind=None),
type_comment=None,
)
node2 = Assign(
targets=[Name(id="a", ctx=Store())],
value=Constant(value=1, kind=None),
type_comment=None,
)
assert ast_tree_edit_distance(node1, node2) == 0
node1 = ast.parse("")
node2 = ast.parse("")
assert ast_tree_edit_distance(node1, node2) == 0
node1 = ast.parse("a")
node2 = ast.parse("a")
assert ast_tree_edit_distance(node1, node2) == 0
node1 = Expr(Name("a", Load()))
node2 = Expr(Name("a", Load()))
assert ast_tree_edit_distance(node1, node2) == 0
node1 = Name("a", Load())
node2 = Name("a", Load())
assert ast_tree_edit_distance(node1, node2) == 0
def blocks(it, lvl=0):
cur = lvl
expr = Expr()
blk = Block(expr)
prefix = str(lvl)+" "*(lvl-1)
for t in it:
log.indent(prefix, "considering", t)
if isinstance(t, DENT):
cur = t.value
if cur == lvl and expr:
log.indent(prefix, "got newline, starting new expr")
expr = Expr()
blk.append(expr)
continue
elif cur > lvl:
log.indent(prefix, ">>> calling nested block")
r, cur = blocks(it, cur)
log.indent(prefix, "<<< got %s from it with level %s" % (r, cur))
expr.append(r)
if cur == lvl:
log.indent(prefix, "!!! starting new expression")
expr = Expr()
blk.append(expr)
else:
log.indent(prefix, "adding %s to expr %s" % (t,expr))
expr.append(t)
if cur < lvl:
log.indent(prefix, "<== %s < %s: time to return" % (cur, lvl))
return blk, cur
return blk, lvl
def expr(self):
self.dbg_msg(" EXPR ")
val1 = self.term()
res = val1
ptok = self.peek()
if ptok.kind in EzhilToken.ADDSUB:
binop = self.dequeue()
if (ptok.kind == EzhilToken.MINUS):
val2 = self.term()
else:
val2 = self.expr()
[l, c] = binop.get_line_col()
res = Expr(val1, binop, val2, l, c, self.debug)
elif ptok.kind == EzhilToken.LPAREN:
## function call
if (res.__class__ != Identifier):
raise ParseException("invalid function call" + str(ptok))
[l, c] = ptok.get_line_col()
vallist = self.valuelist()
res = ExprCall(res, vallist, l, c, self.debug)
ptok = self.peek()
while ptok.kind in EzhilToken.BINOP:
binop = self.dequeue()
[l, c] = binop.get_line_col()
res = Expr(res, binop, self.expr(), l, c, self.debug)
ptok = self.peek()
return res
def _convertExprStmt2Expression(expr_stmt:ast.Expr)->Any:
expr_stmt = copy.deepcopy(expr_stmt)
expr_stmt.lineno = 0
expr_stmt.col_offset = 0
expr_stmt_value = expr_stmt.value
stmt_expression = ast.Expression(expr_stmt_value, lineno=0, col_offset=0)
return stmt_expression
def visit_Expr(self, node):
"""Handle visiting an expression."""
if self.is_in_scope(node):
return node
else:
newnode = self.try_subproc_toks(node)
if not isinstance(newnode, Expr):
newnode = Expr(value=newnode,
lineno=node.lineno,
col_offset=node.col_offset)
if hasattr(node, 'max_lineno'):
newnode.max_lineno = node.max_lineno
newnode.max_col = node.max_col
return newnode
def test_param2argparse_param_default_ast_expr_with_list(self) -> None:
"""
Tests that param2argparse_param works to change the type based on the default
whence said default is an ast.List inside an ast.Expr
"""
run_ast_test(
gen_ast=param2argparse_param(
(
"byo",
{
"default": Expr(
List(
elts=[],
ctx=Load(),
expr=None,
),
expr_value=None,
),
"typ": "str",
},
),
),
gold=argparse_add_argument_expr,
test_case_instance=self,
)
def test_param2argparse_param_default_simple_type(self) -> None:
"""
Tests that param2argparse_param works to change the type based on the default
"""
run_ast_test(
gen_ast=param2argparse_param(
("byo", {"default": 5, "typ": "str"}),
),
gold=Expr(
Call(
args=[set_value("--byo")],
func=Attribute(
Name("argument_parser", Load()),
"add_argument",
Load(),
),
keywords=[
keyword(arg="type", value=Name("int", Load()), identifier=None),
keyword(arg="required", value=set_value(True), identifier=None),
keyword(arg="default", value=set_value(5), identifier=None),
],
expr=None,
expr_func=None,
)
),
test_case_instance=self,
)
def visit_Module(self, node):
logging.info("Adding " + str(len(self.globalvars)) +
" global variables, " + str(len(self.methods)) + " methods")
node_body = node.body
node.body = []
for n in node_body:
if type(n) is ast.ImportFrom or type(n) is ast.Import:
node.body.append(n)
node_body.remove(n)
node.body.append(
Import(names=[alias(name='sys', asname=None)])
)
node.body.append(Expr(value=Call(func=Attribute(
value=Attribute(value=Name(id='sys', ctx=Load()),
attr='path', ctx=Load()),
attr='append',
ctx=Load()),
args=[Constant(value=path_to_tracer, kind=None)],
keywords=[]))
)
node.body.append(Import(names=[alias(name='Tracer', asname=None)]))
for globalvar in self.globalvars:
node.body.append(globalvar)
for method in self.methods:
node.body.append(method)
node.body.extend(node_body)
return node
def test_param2argparse_param_none_default(self) -> None:
"""
Tests that param2argparse_param works to reparse the default
"""
run_ast_test(
gen_ast=param2argparse_param(("yup", {
"default": NoneStr
})),
gold=Expr(
Call(
args=[set_value("--yup")],
func=Attribute(
Name("argument_parser", Load()),
"add_argument",
Load(),
),
keywords=[
keyword(arg="type",
value=Name("str", Load()),
identifier=None),
keyword(arg="default",
value=set_value(None),
identifier=None),
],
expr=None,
expr_func=None,
)),
test_case_instance=self,
)
def compile_call(self, srcnode, parent):
fn = srcnode.function
if '(' not in fn:
fn += '()'
call = parse_and_strip(fn)[0].value
leftovers = im.ContainerNode()
for item in srcnode.children:
if isinstance(item, im.CallKeyword):
funcname = self.unique_id(item.name)
self._compile_function(item, parent, funcname)
call.keywords.append(keyword(arg=item.name,
value=LoadName(funcname)))
elif isinstance(item, im.TextNode) and re.match('^\s*$', item.content, re.S):
continue
else:
leftovers.children.append(item)
if leftovers.children:
funcname = self.unique_id()
self._compile_function(leftovers, parent, funcname)
call.args.append(make_arg(funcname))
parent.body.append(
self.annotate_runtime_errors(
Expr(value=Yield(Call(func=LoadName('str'),
args=[call],
starargs=None,
kwargs=None,
keywords=[]))), srcnode))
def _expand(self, syntax, target, macroname, tree, kw=None):
"""
Transform `target` node, replacing it with the expansion result of
aplying the named macro on the proper node and recursively treat the
expansion as well.
"""
macro = self.bindings[macroname]
kw = kw or {}
kw.update({
'syntax': syntax,
'to_source': unparse,
'expand_macros': self.visit
})
expansion = _apply_macro(macro, tree, kw)
if syntax == 'block':
# I'm not sure why is all this mess
#
# Strategy 1: Make the last line cover the whole block.
# Result: Covers the "with document" line, but not the last one.
# copy_location(expansion[-1], target)
#
# Strategy 2: Make the second line cover the whole block.
# Result: Covers all, unless the block is just 2 lines.
# copy_location(expansion[1], target) # Lo mejor para largo > 2
#
# Strategy 3: Insert a second dummy line covering the whole block.
# Result: Works
dummy = Expr(value=Call(func=Name(id="id", ctx=Load()), args=[Constant(value="bogus", kind=None)], keywords=[]),
lineno=target.lineno)
copy_location(dummy, target)
expansion.insert(1, dummy)
expansion = self._visit_expansion(expansion, target)
return expansion
def _ensure_all_functions_yield(module):
"""
All generated functions should contain at least one yield statement.
This walks the ast to insert a "yield ''" in functions that
don't otherwise produce output (eg in the case of '<py:def
function="a"></py:def>')
"""
functions = {}
if YieldFrom is not None:
yield_classes = (Yield, YieldFrom)
else:
yield_classes = (Yield,)
for node, ancestors in astwalk(module):
if isinstance(node, FunctionDef):
functions.setdefault(node, False)
elif isinstance(node, yield_classes):
f = next(a for a in reversed(ancestors)
if isinstance(a, FunctionDef))
functions[f] = True
for f in functions:
if not functions[f]:
f.body.append(Expr(Yield(Str(s=''))))
return module
def test_ast_equal() -> None:
a = Name(id="print", ctx=Load())
b = Name(id="print", ctx=Load())
assert ast_deep_equal(a, b)
a = Expr(value=Call(
func=Name(id="print", ctx=Load()),
args=[Constant(value="hello, world")],
keywords=[],
))
b = Expr(value=Call(
func=Name(id="print", ctx=Load()),
args=[Constant(value="hello, world")],
keywords=[],
))
assert ast_deep_equal(a, b)
def visit_Expr(self, node):
"""Handle visiting an expression."""
if isdescendable(node.value):
node.value = self.visit(node.value) # this allows diving into BoolOps
if self.is_in_scope(node):
return node
else:
newnode = self.try_subproc_toks(node)
if not isinstance(newnode, Expr):
newnode = Expr(value=newnode,
lineno=node.lineno,
col_offset=node.col_offset)
if hasattr(node, 'max_lineno'):
newnode.max_lineno = node.max_lineno
newnode.max_col = node.max_col
return newnode
def add_method_call(
body: List, instance: str, method: str, args: List, kwargs: List, returns: List[str], index: Optional[int] = None
) -> None:
"""Adds method call to be body of a container. By default, it appends. When
index is specified, it inserts.
:param body:
:param instance:
:param method:
:param args:
:param kwargs:
:param index:
:param returns:
:return:
"""
call = Call(func=get_name_attr(instance, method), args=args, keywords=kwargs)
cont: Union[Expr, Assign, None] = None
if not returns:
cont = Expr(value=call)
elif len(returns) == 1:
# TODO AnnAssign??
cont = Assign(targets=[Name(id=returns[0], ctx=Store())], value=call)
else:
cont = Assign(targets=[Tuple(elts=[Name(id=ret, ctx=Store()) for ret in returns], ctx=Store())], value=call)
if index is None:
body.append(cont)
else:
body.insert(index, cont)
def test_parse_out_param_fails(self) -> None:
""" Test that parse_out_param throws NotImplementedError when unsupported type given """
self.assertRaises(
NotImplementedError,
lambda: parse_out_param(
Expr(
Call(
args=[set_value("--num")],
func=Attribute(
Name("argument_parser", Load()),
"add_argument",
Load(),
),
keywords=[
keyword(
arg="type",
value=Subscript(
expr_context_ctx=None,
expr_slice=None,
expr_value=None,
),
identifier=None,
),
keyword(
arg="required",
value=set_value(True),
identifier=None,
),
],
expr=None,
expr_func=None,
))),
)
def visitblock(self,body,kind): if not self.enableCodeCoverage: return if self.hot == None: return for i in reversed(range(len(body)+1)): versneaky0 = Expr(value=Call(func=Attribute(value=Name(id='madscience_debug_context', ctx=Load()), attr='log', ctx=Load()), args=[Num(n=self.scopes[self.hot]+i)], keywords=[])) body.insert(i,versneaky0) self.scopes[self.hot]+=len(body)
def visit_Expr(self, node: Expr) -> Expr:
node.value = Call(func=Name(id='__autoexpr__', ctx=Load()),
args=[node.value],
keywords=[])
fix_missing_locations(node)
return node
def visit_Expr(self, node):
"""Handle visiting an expression."""
if isdescendable(node.value):
node.value = self.visit(node.value) # this allows diving into BoolOps
if self.is_in_scope(node):
return node
else:
newnode = self.try_subproc_toks(node)
if not isinstance(newnode, Expr):
newnode = Expr(value=newnode,
lineno=node.lineno,
col_offset=node.col_offset)
if hasattr(node, 'max_lineno'):
newnode.max_lineno = node.max_lineno
newnode.max_col = node.max_col
return newnode
def compile_extendsnode(self, srcnode, parent):
if '$' in srcnode.href:
value = _interpolated_str_to_ast_value(srcnode.href)
parent.body.append(Assign(targets=[StoreName('__piglet_tmp')],
value=value))
loadcode = '__piglet_rt.load(__piglet_template, __piglet_tmp)\n'
else:
loadcode = ('__piglet_rt.load(__piglet_template, "{}")\n'
.format(srcnode.href))
parent.body.extend(parse_and_strip(
'__piglet_parent = {}'
'__piglet_bases = [__piglet_parent] + __piglet_bases\n'
.format(loadcode)))
for n in srcnode.children:
if isinstance(n, im.BlockNode):
self.compile_blockreplacenode(n, parent)
elif isinstance(n, im.DefNode):
self._compile(n, parent)
block_ids = [make_block_name(n.name)
for n in srcnode.find(im.BlockNode)]
parent_template_call = Call(
func=LoadAttribute('__piglet_parent', '__piglet_root__'),
args=[],
starargs=None,
kwargs=None,
keywords=([keyword(arg='__piglet_bases',
value=LoadName('__piglet_bases'))] +
[keyword(arg=str(b), value=LoadName(str(b)))
for b in block_ids])
)
add_kwarg(parent_template_call, '__piglet_extra_blocks')
if YieldFrom is not None:
parent.body.append(
Expr(value=YieldFrom(value=parent_template_call)))
else:
loopvar = self.unique_id('loop')
parent.body.append(
For(target=Name(id=loopvar, ctx=Store()),
iter=parent_template_call,
body=[Expr(value=Yield(Name(id=loopvar, ctx=Load())))],
orelse=[]))
def compile_translationnode(self, srcnode, parent):
translated = Call(func=LoadName('_'),
args=[Str(srcnode.get_msgstr())],
starargs=None,
kwargs=None,
keywords=[])
named_children = [(name, node)
for name, node in srcnode.named_children()
if name is not None]
if not named_children:
# Simple case - no dynamic children for placeholder replacement
parent.body.append(Expr(value=Yield(translated)))
return
parent.body.append(
Assign(targets=[StoreName('__piglet_places')],
value=Dict([], []))
)
for name, node in named_children:
with self.collect_output(parent) as ACC:
self._compile(node, parent)
parent.body.append(
Assign(targets=[Subscript(value=LoadName('__piglet_places'),
slice=Index(value=Str(name)),
ctx=Store())],
value=Call(func=Attribute(value=Str(s=''), attr='join', ctx=Load()),
args=[LoadName(ACC)],
starargs=None,
kwargs=None,
keywords=[]))
)
for name, node in named_children:
translated = Call(
func=Attribute(value=translated, attr='replace', ctx=Load()),
args=[Str('${{{}}}'.format(name)),
Subscript(value=LoadName('__piglet_places'),
slice=Index(value=Str(name)),
ctx=Load())],
starargs=None,
kwargs=None,
keywords=[])
set_pos(translated, srcnode)
parent.body.append(Expr(value=Yield(translated)))
def visit_Return(self,node:ast.Return):
if self.hot == None or (not self.hotHasReturnCheck and self.funcNames[self.hot] not in self.exitpatterns): return node
# print("Assign: ",self.funcNames[self.hot])
sin = [
Assign(targets=[Name(id='_dbg_ret_var', ctx=Store())], value=node.value),
Return(value=Name(id='_dbg_ret_var', ctx=Load()))
]
if self.hotHasReturnCheck:
expattern = self.funcparams[self.hot]
sin.insert(1,Expr(value=Call(func=Name(id='_dbgExit', ctx=Load()), args=[Name(id=pn+'_dbg_str_var_'+str(self.hot),ctx=Load()) for pn in expattern]+[Name(id='_dbg_ret_var', ctx=Load())], keywords=[])))
if self.funcNames[self.hot] in self.exitpatterns:
expattern = self.exitpatterns[self.funcNames[self.hot]]
sin.insert(1,Expr(value=Call(func=Name(id='_dbgExit_'+self.funcNames[self.hot],ctx=Load()), args=[Name(id=pn+'_dbg_str_var_'+str(self.hot),ctx=Load()) for pn in expattern]+[Name(id='_dbg_ret_var', ctx=Load())], keywords=[])))
for s in sin:
ast.copy_location(s, node)
ast.fix_missing_locations(s)
return sin
def test_selector_standalone():
from ast import Expr, Num # use python's builtin ast library
Expr._priority = 0
Num._priority = 1
node = Expr(value=Num(n=1))
sel = Selector(Num)
sel.visit(node)
assert isinstance(sel.out[0], Num)
def start(self, tree):
from ast import Module, expr, Expr, Call, fix_missing_locations
stmts = [
Expr(s) if isinstance(s, expr) else s
for s in reversed(self.statements)
]
if not stmts:
stmts.append(tree[0])
return fix_missing_locations(Module(body=stmts, type_ignores=[]))
def end_loop(self, lineno, loop_start): loop = [self.update(lineno)] cleanup = Expr(value=Call(func=Name(id=Environment.cleanup_loop_func, ctx=Load()), args=[Num(n=lineno), Num(n=1), Name(id=Environment.iter_num, ctx=Load())])) loop.append(cleanup) delete = Delete(targets=[Name(id=Environment.iter_num, ctx=Del())]) loop.append(delete) return loop
def sub_expr(expr: ast.Expr, env: List[Dict[str, str]]) -> ast.Expr:
if type(expr) == ast.Name:
return ast.Str(env[expr.id])
elif type(expr) == ast.Compare:
expr = copy.deepcopy(expr)
expr.left = sub_expr(expr.left, env)
expr.comparators = [sub_expr(x, env) for x in expr.comparators]
return expr
elif type(expr) == ast.Str:
return expr
elif type(expr) == ast.Subscript:
if type(expr.value) == ast.Name:
if type(expr.slice) == ast.Index:
return ast.Str(env[expr.value.id][expr.slice.value.n])
elif type(expr.slice) == ast.Slice:
return ast.Str(
env[expr.value.id][expr.slice.lower.n:expr.slice.upper.n])
raise Exception(ast.dump(expr))
def parseSwitchStmt(self, exp):
## @ <ID/EXPR> SWITCH @( expr ) CASE {stmtlist} @( expr ) CASE {stmtlist} OTHERWISE {stmtlist} END
## implement as an if-elseif-else statement
self.dbg_msg("parsing SWITCH statement")
sw_tok = self.dequeue()
[l, c] = sw_tok.get_line_col()
self.inside_if = True
lhs = exp[0]
# enter this if-statement always
ifstmt = IfStmt(Number(1), None, None, l, c, self.debug)
self.if_stack.append(ifstmt)
self.dbg_msg("parsing SWITCH-body") #self.dbg_msg
ptok = self.peek()
equality_token = EzhilLexeme("=", EzhilToken.EQUALITY)
while (ptok.kind == EzhilToken.ATRATEOF
or ptok.kind == EzhilToken.OTHERWISE):
self.inside_if = True
[l, c] = ptok.get_line_col()
if (ptok.kind == EzhilToken.ATRATEOF):
# parse elseif branch
self.dbg_msg("parsing CASE")
self.match(EzhilToken.ATRATEOF)
exp = self.valuelist()
self.dbg_msg("parsing CASE EXPR")
self.match(EzhilToken.CASE)
next_stmt = self.stmtlist()
expr = Expr(lhs, equality_token, exp[0], l, c, self.debug)
self.dbg_msg("building an Expr " + str(expr))
if not ifstmt.body:
ifstmt.expr = expr
ifstmt.body = next_stmt
else:
case_stmt = IfStmt(expr, next_stmt, None, l, c, self.debug)
ifstmt.append_stmt(case_stmt)
elif (ptok.kind == EzhilToken.OTHERWISE):
#parse else branch
self.dbg_msg("parsing OTHERWISE: ")
self.match(EzhilToken.OTHERWISE)
self.dbg_msg("parsing OTHERWISE-Body")
self.inside_if = False
body = self.stmtlist()
else_stmt = ElseStmt(body, l, c, self.debug)
if not ifstmt.body:
ifstmt.body = else_stmt
else:
ifstmt.append_stmt(else_stmt)
break
else:
self.inside_if = False
raise ParseError(
"SWITCH-CASE-OTHERWISE statement syntax is messed up")
ptok = self.peek()
self.dbg_msg("parsing SWITCH-CASE next bits " + str(ptok))
self.match(EzhilToken.END)
self.inside_if = False
self.dbg_msg("parsing -SWITCH-CASE- complete")
return ifstmt
def visit_Assign(self, node: Assign) -> Any:
unused = all(n for n in node.targets
if isinstance(n, Name) and hasattr(n, "_pyo_unused"))
if unused:
if has_side_effects(node.value):
return Expr(node.value)
else:
return Pass()
return node
def test_param2argparse_param_default_ast_expr_with_list(self) -> None:
"""
Tests that param2argparse_param works to change the type based on the default
whence said default is an ast.List inside an ast.Expr
"""
run_ast_test(
gen_ast=param2argparse_param((
"byo",
{
"default":
Expr(
List(
elts=[],
ctx=Load(),
expr=None,
),
expr_value=None,
),
"typ":
"str",
},
), ),
gold=Expr(
Call(
args=[set_value("--byo")],
func=Attribute(
Name("argument_parser", Load()),
"add_argument",
Load(),
),
keywords=[
keyword(arg="action",
value=set_value("append"),
identifier=None),
keyword(arg="required",
value=set_value(True),
identifier=None),
],
expr=None,
expr_func=None,
)),
test_case_instance=self,
)
def visit_Expr(self, node):
if self.is_in_scope(node):
return node
else:
newnode = self.try_subproc_toks(node)
if not isinstance(newnode, Expr):
newnode = Expr(value=newnode,
lineno=node.lineno,
col_offset=node.col_offset)
return newnode
def replace_node(old, new):
"""
Replaces a node by another.
If the new node is an expression while the old one is a statement, this function
wraps the expression in an Expr node.
"""
if isinstance(old, stmt) and isinstance(new, expr):
new = Expr(value=new)
for field, values in iter_fields(old.parent):
try:
values[values.index(old)] = new
break
except Exception:
if old is values:
setattr(old.parent, field, new)
break
new.parent = old.parent
old.parent = None
return new