import ast
from ast import Assign, Load, Module, Name, Print, Store
node_map = {
'Assignment': dict(pynode='Assign',
mappers={'targets': 'node', 'value': 'expr'},
types={'left': 'list'},
args={'ctx': ast.Store(lineno=0, col_offset=0)}),
'Echo': dict(pynode='Print',
mappers={'values': 'nodes'},
types={'left': list},
args={'nl': True, 'ctx': ast.Load(lineno=0, col_offset=0)}),
'Variable': dict(pynode='Name',
mappers={'id': 'name'},
types={'left': 'var'},
args={'ctx': ast.Store(lineno=0, col_offset=0)}),
}
body = []
def eval_node(phpnode, ctx=None):
node_type, el = phpnode.generic()
mapdef = node_map[node_type]
py_ast_class = getattr(ast, mapdef['pynode'])
instance = py_ast_class(lineno=0, col_offset=0,
**mapdef.get('args', {}))
if ctx:
def assign(self, expr: ast.expr) -> ast.Name:
"""Give *expr* a name."""
name = self.variable()
self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr))
return ast.Name(name, ast.Load())
other = Decimal(other)
quotient, remainder = super(Decimal, other).__divmod__(self)
return Decimal(quotient), Decimal(remainder)
def __pow__(self, power, modulo=None, context=None):
if isinstance(power, (int, float)):
power = Decimal(power)
return Decimal(super(Decimal, self).__pow__(power, modulo))
def __rpow__(self, other, context=None):
if isinstance(other, (int, float)):
other = Decimal(other)
return Decimal(super(Decimal, other).__pow__(self))
TYPE_STORE = type(ast.Store())
TYPE_LOAD = type(ast.Load())
TYPE_DEL = type(ast.Del())
TYPE_EXPR = type(ast.Expr())
class BadSyntax(Exception):
pass
BINOP_TABLE: Dict[Any, Callable[[Any, Any], Any]] = {
_ast.Add: lambda a, b: a + b,
_ast.BitAnd: lambda a, b: a & b,
_ast.BitOr: lambda a, b: a | b,
_ast.BitXor: lambda a, b: a ^ b,
_ast.Div: lambda a, b: a / b,
def translate_to_tptv1(parsed_model, data_batch, hypers):
parsed_model = u.replace_hypers(parsed_model, hypers)
parsed_model = AssignmentAndFunctionInliner().visit(parsed_model)
input_dependents = get_input_dependent_vars(parsed_model)
idx_var_name = "input_idx"
idx_var = ast.Name(idx_var_name, ast.Load())
input_number = len(data_batch['instances'])
range_expr = ast.Num(input_number)
input_vars = set()
output_vars = set()
var_decls = []
input_stmts = []
general_stmts = []
output_stmts = []
for stmt in parsed_model.body:
if isinstance(stmt, ast.Assign) and is_input_declaration(stmt.value):
input_vars.add(get_var_name(stmt.targets[0]))
var_decls.append(stmt)
elif isinstance(stmt, ast.Assign) and is_output_declaration(
stmt.value):
output_vars.add(get_var_name(stmt.targets[0]))
var_decls.append(stmt)
elif isinstance(stmt, ast.Assign) and is_var_declaration(stmt.value):
var_decls.append(stmt)
elif ast_uses_varset(stmt, input_dependents):
input_stmts.append(
add_input_indices(stmt, input_dependents, idx_var))
elif ast_uses_varset(stmt, output_vars):
output_stmts.append(stmt)
else:
general_stmts.append(stmt)
input_init = []
output_observation = []
for input_idx, instance in enumerate(data_batch['instances']):
for var_name, val in instance.iteritems():
if var_name in input_vars:
input_init.extend(
generate_io_stmt(input_idx, var_name, val,
"set_to_constant"))
elif var_name in output_vars:
output_observation.extend(
generate_io_stmt(input_idx, var_name, val,
"observe_value"))
extended_var_decls = []
for var_decl in var_decls:
# If input-dependent, extend dimension by one
if get_var_name(var_decl.targets[0]) in input_dependents:
new_decl = copy.deepcopy(var_decl)
if isinstance(new_decl.value, ast.Subscript):
new_decl.value = extend_subscript_for_input(
new_decl.value, ast.Num(input_number))
else:
new_decl.value = ast.Subscript(
new_decl.value, ast.Index(ast.Num(input_number)),
ast.Load())
extended_var_decls.append(new_decl)
else:
extended_var_decls.append(var_decl)
input_loop = ast.For(
ast.Name(idx_var_name, ast.Store()),
ast.Call(ast.Name("range", ast.Load()), [range_expr], [], None, None),
input_stmts, [])
parsed_model.body = general_stmts + extended_var_decls + input_init + [
input_loop
] + output_stmts + output_observation
ast.fix_missing_locations(parsed_model)
return parsed_model
def ex_lvalue(name):
"""A variable load expression."""
return ast.Name(name, ast.Store())
def multiphase_expand(tree,
*,
filename,
self_module,
dexpander=None,
_optimize=-1):
"""Macro-expand an AST in multiple phases, controlled by `with phase[n]`.
Primarily meant to be called with `tree` the AST of a module that
uses macros, but works with any `tree` that has a `body` attribute,
where that `body` is a `list` of statement AST nodes.
At each phase `k >= 1`, we inject a temporary module into `sys.modules`, so that
the next phase can import macros from it, using the self-macro-import syntax.
Once phase `k = 0` is reached and the code has been macro-expanded, the temporary
entry is deleted from `sys.modules`.
`filename`: Full path to the `.py` file being compiled.
`self_module`: Absolute dotted module name of the module being compiled.
Used for temporarily injecting the temporary, higher-phase
modules into `sys.modules`, as well as resolving `__self__`
in self-macro-imports (`from __self__ import macros, ...`).
`dexpander`: The `DialectExpander` instance to use for dialect AST transforms.
If not provided, dialect processing is skipped.
`_optimize`: Passed on to Python's built-in `compile` function, when compiling
the temporary higher-phase modules. Has no effect on the final result.
Return value is the final phase-0 `tree`, after macro expansion.
"""
n = detect_highest_phase(tree)
debug = isdebug(tree)
c, CS = setcolor, ColorScheme
# If this module is already in `sys.modules` (e.g. created by Python's
# import system as a blank module instance, to be filled in by an exec
# after compilation is done), record the original module object, so we
# can reinstate it once we're done.
if self_module in sys.modules:
original_module = sys.modules[self_module]
else:
original_module = None
if debug:
print(
f"{c(CS.HEADING)}**Multi-phase compiling module {c(CS.TREEID)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.TREEID)}){c()}",
file=sys.stderr)
# Inject temporary module into `sys.modules`.
#
# Note that if the module already exists in `sys.modules`, its entry will be overwritten.
# We don't touch the parent module (if any), so that if it already refers to an old (but
# phase-0, fully compiled) version of this one, the reference won't be clobbered with one
# pointing to the temporary module.
#
# We must inject the temporary module only once, to keep the already compiled higher-phase
# stuff available in the module's namespace while the next phase is being compiled.
# (This matters when there are at least 3 phases, see `demo/let.py`.)
module = compiler.create_module(dotted_name=self_module,
filename=filename,
update_parent=False)
for k in range(n, -1, -1): # phase 0 is what a regular compile would do
if debug:
print(
f"{c(CS.HEADING)}**AST for {c(CS.ATTENTION)}PHASE {k}{c(CS.HEADING)} of module {c(CS.TREEID)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.TREEID)}){c()}",
file=sys.stderr)
phase_k_tree = extract_phase(tree, phase=k)
if phase_k_tree.body:
# inject `__phase__ = k` for introspection (at run time of the phase being compiled now)
tgt = ast.Name(id="__phase__",
ctx=ast.Store(),
lineno=1,
col_offset=1)
val = ast.Constant(value=k, lineno=1, col_offset=13)
assignment = ast.Assign(targets=[tgt],
value=val,
lineno=1,
col_offset=1)
if getdocstring(phase_k_tree.body):
docstring, *body = phase_k_tree.body
phase_k_tree.body = [docstring, assignment] + body
else: # no docstring
phase_k_tree.body = [assignment] + phase_k_tree.body
if debug:
print(unparse_with_fallbacks(phase_k_tree,
debug=True,
color=True),
file=sys.stderr)
# Once we hit the final phase, no more temporary modules - let the import system take over.
if k == 0:
expansion = compiler.singlephase_expand(
phase_k_tree,
filename=filename,
self_module=self_module,
dexpander=dexpander)
break
# Compile the current tree, and run it in the namespace of the temporary module. This
# gives us the temporary higher-phase module, which allows us to compile the next phase.
# At intermediate phases, some definitions overwrite previous ones - this is ok.
compiler.run(phase_k_tree, module)
# restore `sys.modules` to how it was before we began the multi-phase compile for this module
if original_module:
sys.modules[self_module] = original_module
else:
try:
del sys.modules[self_module]
except KeyError:
pass
return expansion
def gen_unpack_wrapper(self, node, target, ctx='store'):
"""Helper function to protect tuple unpacks.
node: used to copy the locations for the new nodes.
target: is the tuple which must be protected.
ctx: Defines the context of the returned temporary node.
It returns a tuple with two element.
Element 1: Is a temporary name node which must be used to
replace the target.
The context (store, param) is defined
by the 'ctx' parameter..
Element 2: Is a try .. finally where the body performs the
protected tuple unpack of the temporary variable
into the original target.
"""
# Generate a tmp name to replace the tuple with.
tmp_name = self.gen_tmp_name()
# Generates an expressions which protects the unpack.
# converter looks like 'wrapper(tmp_name)'.
# 'wrapper' takes care to protect sequence unpacking with _getiter_.
converter = self.protect_unpack_sequence(
target, ast.Name(tmp_name, ast.Load()))
# Assign the expression to the original names.
# Cleanup the temporary variable.
# Generates:
# try:
# # converter is 'wrapper(tmp_name)'
# arg = converter
# finally:
# del tmp_arg
try_body = [ast.Assign(targets=[target], value=converter)]
finalbody = [self.gen_del_stmt(tmp_name)]
if IS_PY2:
cleanup = ast.TryFinally(body=try_body, finalbody=finalbody)
else:
cleanup = ast.Try(body=try_body,
finalbody=finalbody,
handlers=[],
orelse=[])
if ctx == 'store':
ctx = ast.Store()
elif ctx == 'param':
ctx = ast.Param()
else: # pragma: no cover
# Only store and param are defined ctx.
raise NotImplementedError(
'Unsupported context type: "{name}".'.format(name=type(ctx)), )
# This node is used to catch the tuple in a tmp variable.
tmp_target = ast.Name(tmp_name, ctx)
copy_locations(tmp_target, node)
copy_locations(cleanup, node)
return (tmp_target, cleanup)
async def meval(code, globs, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globs.copy()
# This code saves __name__ and __package into a kwarg passed to the func.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there's no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, "exec")
code = root.body
ret_name = "_ret"
ok = False
while True:
if ret_name in globs.keys():
ret_name = "_" + ret_name
continue
for node in ast.walk(root):
if isinstance(node, ast.Name) and node.id == ret_name:
ret_name = "_" + ret_name
break
ok = True
if ok:
break
if not code:
return None
if not any(isinstance(node, ast.Return) for node in code):
for i in range(len(code)):
if isinstance(code[i], ast.Expr):
if (i == len(code) - 1
or not isinstance(code[i].value, ast.Call)):
code[i] = ast.copy_location(
ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(
id=ret_name, ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[code[i].value],
keywords=[])), code[-1])
else:
for node in code:
if isinstance(node, ast.Return):
node.value = ast.List(elts=[node.value], ctx=ast.Load())
code.append(
ast.copy_location(
ast.Return(value=ast.Name(id=ret_name, ctx=ast.Load())), code[-1]))
# globals().update(**<global_args>)
glob_copy = ast.Expr(
ast.Call(
func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals",
ctx=ast.Load()),
args=[],
keywords=[]),
attr="update",
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))
]))
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
ret_decl = ast.Assign(targets=[ast.Name(id=ret_name, ctx=ast.Store())],
value=ast.List(elts=[], ctx=ast.Load()))
ast.fix_missing_locations(ret_decl)
code.insert(1, ret_decl)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
ast.fix_missing_locations(a)
args += [a]
args = ast.arguments(args=[],
vararg=None,
kwonlyargs=args,
kwarg=None,
defaults=[],
kw_defaults=[None for i in range(len(args))])
args.posonlyargs = []
fun = ast.AsyncFunctionDef(name="tmp",
args=args,
body=code,
decorator_list=[],
returns=None)
ast.fix_missing_locations(fun)
mod = ast.parse("")
mod.body = [fun]
comp = compile(mod, "<string>", "exec")
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
for i in range(len(r)):
if hasattr(r[i], "__await__"):
r[i] = await r[i] # workaround for 3.5
i = 0
while i < len(r) - 1:
if r[i] is None:
del r[i]
else:
i += 1
if len(r) == 1:
[r] = r
elif not r:
r = None
return r
def visit_FunctionDef(self, node):
node.parent = self.fundef
self.fundef = node
if len(
list(
filter(
lambda n: isinstance(n, ast.Name) and n.id is
'rep_fun', node.decorator_list))) > 0:
self.recs.append(node.name)
self.generic_visit(node)
r_args = {}
for arg in node.args.args:
arg_name = arg.arg
try:
if self.fundef.locals[arg_name] > 1:
r_args[arg_name] = self.freshName('x')
# self.fundef.locals[arg_name] += 1
except KeyError as e:
pass
# generate code to pre-initialize staged vars
# we stage all vars that are written to more than once
inits = [ast.Assign(targets=[ast.Name(id=id, ctx=ast.Store())],
value=ast.Call(
func=ast.Name(id='_var', ctx=ast.Load()),
args=[],
keywords=[])) \
for id in node.locals if node.locals[id] > 1]
a_nodes = [ast.Expr(
ast.Call(
func=ast.Name(id='_assign', ctx=ast.Load()),
args=[ast.Name(id=arg,ctx=ast.Load()), ast.Name(id=r_args[arg],ctx=ast.Load())],
keywords=[])) \
for arg in r_args]
new_node = ast.FunctionDef(
name=node.name,
args=ast.arguments(args=[
ast.arg(arg=r_args[arg.arg], annotation=None)
if arg.arg in r_args else arg for arg in node.args.args
],
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[]), # node.args,
body=[
ast.Try(body=inits + a_nodes + node.body,
handlers=[
ast.ExceptHandler(
type=ast.Name(id='NonLocalReturnValue',
ctx=ast.Load()),
name='r',
body=[
ast.Return(value=ast.Attribute(
value=ast.Name(id='r', ctx=ast.Load()),
attr='value',
ctx=ast.Load()))
])
],
orelse=[],
finalbody=[])
],
decorator_list=list(
filter(
lambda n: isinstance(n, ast.Name) and n.id != 'lms' and n.
id != 'rep_fun', node.decorator_list)))
ast.copy_location(new_node, node)
ast.fix_missing_locations(new_node)
self.fundef = node.parent
return new_node
def visit_FunctionDef(self, node):
after_nodes = []
if self.curprim is None:
self.curprim = self.pdp
self.curchild = -1
if isinstance(node.decorator_list[0], ast.Call):
self.module_name = node.decorator_list[0].func.value.id
else:
self.module_name = node.decorator_list[0].value.id
# Strip decorator
del node.decorator_list[0]
oldchild = self.curchild
oldprim = self.curprim
else:
if len(node.decorator_list) == 0:
return self.generic_visit(node)
dec = node.decorator_list[0]
if isinstance(dec, ast.Call):
decname = astparser.rname(dec.func.attr)
else:
decname = astparser.rname(dec.attr)
if decname in [
'map', 'async_map', 'reduce', 'async_reduce', 'consume',
'async_consume', 'tasklet', 'async_tasklet', 'iterate',
'loop', 'conditional'
]:
self.curchild += 1
oldchild = self.curchild
oldprim = self.curprim
self.curprim = self.curprim.children[self.curchild]
self.curchild = -1
if isinstance(self.curprim, astnodes._MapNode):
newnode = \
_copy_location(ast.For(target=ast.Tuple(ctx=ast.Store(),
elts=[ast.Name(id=name, ctx=ast.Store()) for name in self.curprim.params]),
iter=ast.parse('%s.ndrange(%s)' % (self.module_name, self.curprim.range.pystr())).body[0].value,
body=node.body, orelse=[]),
node)
node = newnode
elif isinstance(self.curprim, astnodes._ConsumeNode):
stream = self.curprim.stream
if isinstance(self.curprim.stream, ast.AST):
stream = unparse(self.curprim.stream)
if '[' not in stream:
stream += '[0]'
newnode = \
_copy_location(ast.While(
test=ast.parse('len(%s) > 0' % stream).body[0].value,
body=node.body, orelse=[]),
node)
node = newnode
node.body.insert(
0,
_copy_location(
ast.parse('%s = %s.popleft()' % (str(
self.curprim.params[0]), stream)).body[0],
node))
elif isinstance(self.curprim, astnodes._TaskletNode):
# Strip decorator
del node.decorator_list[0]
newnode = \
_copy_location(ast.parse('if True: pass').body[0], node)
newnode.body = node.body
newnode = ast.fix_missing_locations(newnode)
node = newnode
elif isinstance(self.curprim, astnodes._ReduceNode):
in_memlet = self.curprim.inputs['input']
out_memlet = self.curprim.outputs['output']
# Create reduction call
params = [unparse(p) for p in node.decorator_list[0].args]
params.extend([
unparse(kp) for kp in node.decorator_list[0].keywords
])
reduction = ast.parse(
'%s.simulator.simulate_reduce(%s, %s)' %
(self.module_name, node.name,
', '.join(params))).body[0]
reduction = _copy_location(reduction, node)
reduction = ast.increment_lineno(reduction,
len(node.body) + 1)
reduction = ast.fix_missing_locations(reduction)
# Strip decorator
del node.decorator_list[0]
after_nodes.append(reduction)
elif isinstance(self.curprim, astnodes._IterateNode):
newnode = \
_copy_location(ast.For(target=ast.Tuple(ctx=ast.Store(),
elts=[ast.Name(id=name, ctx=ast.Store()) for name in self.curprim.params]),
iter=ast.parse('%s.ndrange(%s)' % (self.module_name, self.curprim.range.pystr())).body[0].value,
body=node.body, orelse=[]),
node)
newnode = ast.fix_missing_locations(newnode)
node = newnode
elif isinstance(self.curprim, astnodes._LoopNode):
newnode = \
_copy_location(ast.While(test=node.decorator_list[0].args[0],
body=node.body, orelse=[]),
node)
newnode = ast.fix_missing_locations(newnode)
node = newnode
else:
raise RuntimeError('Unimplemented primitive %s' % decname)
else:
return self.generic_visit(node)
newbody = []
end_stmts = []
substitute_stmts = []
# Incrementally build new body from original body
for stmt in node.body:
if isinstance(stmt, ast.Expr):
res, append, prepend = self.VisitTopLevelExpr(stmt)
if res is not None:
newbody.append(res)
if append is not None:
end_stmts.extend(append)
if prepend is not None:
substitute_stmts.extend(prepend)
else:
subnodes = self.visit(stmt)
if subnodes is not None:
if isinstance(subnodes, list):
newbody.extend(subnodes)
else:
newbody.append(subnodes)
node.body = newbody + end_stmts
self.curchild = oldchild
self.curprim = oldprim
substitute_stmts.append(node)
if len(after_nodes) > 0:
return substitute_stmts + after_nodes
return substitute_stmts
def VisitTopLevelExpr(self, node):
# DaCe memlet expression
if isinstance(node.value, ast.BinOp):
rhs = node.value.right
lhs = node.value.left
arrays = self.curprim.arrays()
if isinstance(node.value.op, ast.LShift):
# Dynamic access. Emit nothing and load memory on encounter
if isinstance(rhs, ast.Call) and ast.literal_eval(
rhs.args[0]) == -1:
array_name = rhs.func.id
stripped_subscript = '%s[:]' % (array_name)
self.storeOnAssignment[node.value.left.id] = \
ast.parse(stripped_subscript).body[0].value
return None, None, None
if isinstance(rhs, ast.Subscript) and isinstance(
rhs.value, ast.Call):
# Dynamic access. Emit nothing and load memory on encounter
if ast.literal_eval(rhs.value.args[0]) == -1:
array_name = rhs.value.func.id
stripped_subscript = '%s[%s]' % (array_name,
unparse(rhs.slice))
self.storeOnAssignment[node.value.left.id] = \
ast.parse(stripped_subscript).body[0].value
return None, None, None
rhs = ast.Subscript(
value=rhs.value.func, ctx=ast.Load(), slice=rhs.slice)
result = _copy_location(
ast.Assign(targets=[node.value.left], value=rhs), node)
result.targets[0].ctx = ast.Store()
return result, None, None
# END of "a << b"
elif isinstance(node.value.op, ast.RShift):
# If the memlet refers to a sub-array (view), also add an expression to initialize it
init_expr = None
result = None
prefix = []
if isinstance(rhs, ast.Subscript):
# Index subscript expression ("tmp >> b(1, sum)[i,j,k,l]")
if isinstance(rhs.value, ast.Call):
# Only match expressions with possible write-conflict resolution, such as "A(...)[...]"
array_name = rhs.value.func.id
stripped_subscript = '%s[%s]' % (array_name,
unparse(rhs.slice))
# WCR initialization with identity value
if len(rhs.value.args) >= 3:
prefix.append(
_copy_location(
ast.parse(
'%s = %s' %
(stripped_subscript,
unparse(rhs.value.args[2]))).body[0],
node))
# Dynamic access. Emit nothing and store memory on assignment
if ast.literal_eval(rhs.value.args[0]) == -1:
if len(rhs.value.args) >= 2:
self.accumOnAssignment[node.value.left.id] = \
(stripped_subscript, rhs.value.args[1])
else:
self.storeOnAssignment[node.value.left.id] = \
ast.parse(stripped_subscript).body[0].value
return init_expr, None, prefix
# Make sure WCR function exists
if len(rhs.value.args) >= 2:
result = ast.parse(
'%s = (%s)(%s, %s)' %
(stripped_subscript, unparse(
rhs.value.args[1]), stripped_subscript,
node.value.left.id)).body[0]
result = _copy_location(result, node)
else:
result = ast.parse(
'%s = %s' % (stripped_subscript,
node.value.left.id)).body[0]
result = _copy_location(result, node)
else:
array_name = rhs.value.id
if not isinstance(rhs.slice, ast.Index):
init_expr = _copy_location(
ast.Assign(
targets=[
ast.Name(
id=node.value.left.id, ctx=ast.Store())
],
value=ast.Subscript(
value=ast.Name(
id=array_name, ctx=ast.Load()),
slice=rhs.slice,
ctx=ast.Load())), node)
elif not isinstance(rhs, ast.Subscript):
if isinstance(rhs, ast.Call):
array_name = rhs.func
else:
array_name = rhs
lhs_name = lhs.id
# In case of "tmp >> array", write "array[:]"
if node.value.left.id in self.curprim.transients:
init_expr = None
# If reading from a single stream ("b << stream")
elif (array_name.id in arrays
and isinstance(arrays[array_name.id], data.Stream)):
if arrays[array_name.id].shape == [1]:
init_expr = _copy_location(
ast.parse('{v} = {q}[0]'.format(
v=lhs_name, q=array_name.id)).body[0],
node)
return init_expr, None, []
else:
init_expr = _copy_location(
ast.Assign(
targets=[
ast.Name(id=lhs_name, ctx=ast.Store())
],
value=ast.Subscript(
value=ast.Name(
id=array_name.id, ctx=ast.Load()),
slice=ast.Slice(
lower=None, upper=None, step=None),
ctx=ast.Load())), node)
# If we are setting a stream's sink
if lhs_name in arrays and isinstance(
arrays[lhs_name], data.Stream):
result = ast.parse(
'{arr}[0:len({q}[0])] = list({q}[0])'.format(
arr=rhs.id, q=lhs.id)).body[0]
result = _copy_location(result, node)
# If WCR function exists
elif isinstance(rhs, ast.Call) and len(rhs.args) >= 2:
# WCR initialization with identity value
if len(rhs.args) >= 3:
prefix.append(
_copy_location(
ast.parse('%s[:] = %s' %
(array_name.id,
unparse(rhs.args[2]))).body[0],
node))
# Dynamic access. Emit nothing and store memory on assignment
if ast.literal_eval(rhs.args[0]) == -1:
self.accumOnAssignment[lhs.id] = (array_name.id,
rhs.args[1])
return init_expr, None, prefix
result = ast.parse(
'%s[:] = (%s)(%s[:], %s)' %
(array_name.id, unparse(rhs.args[1]),
array_name.id, node.value.left.id)).body[0]
result = _copy_location(result, node)
else:
result = _copy_location(
ast.Assign(
targets=[
ast.Subscript(
value=ast.Name(
id=array_name.id, ctx=ast.Load()),
slice=ast.Slice(
lower=None, upper=None, step=None),
ctx=ast.Store())
],
value=node.value.left), node)
if result is None:
result = _copy_location(
ast.Assign(
targets=[node.value.right], value=node.value.left),
node)
result.targets[0].ctx = ast.Store()
return init_expr, [result], prefix
# END of "a >> b"
return self.generic_visit(node), [], None
def instantiateFunction(self, filename, lineNumber, memberDictionary,
file_text):
"""Instantiate a function instance."""
memberDictionary = {
k: v
for k, v in memberDictionary.iteritems()
if not isinstance(v, Exceptions.PyforaNameError)
}
objectOrNone = self.moduleLevelObject(filename, lineNumber)
if objectOrNone is not None:
return objectOrNone
sourceAst = PyAstUtil.pyAstFromText(file_text)
functionAst = PyAstUtil.functionDefOrLambdaOrWithBlockAtLineNumber(
sourceAst, lineNumber)
outputLocals = {}
globalScope = {}
globalScope.update(memberDictionary)
self.importModuleMagicVariables(globalScope, filename)
if isinstance(functionAst, ast.Lambda):
expr = ast.FunctionDef()
expr.name = '__pyfora_lambda_builder__'
expr.args = ast.arguments()
expr.args.args = []
expr.args.defaults = []
expr.args.vararg = None
expr.args.kwarg = None
expr.decorator_list = []
expr.lineno = functionAst.lineno - 1
expr.col_offset = functionAst.col_offset
return_statement = ast.Return(functionAst)
expr.body = [return_statement]
expr = updatePyAstMemberChains(ast.Module([expr],
lineno=1,
col_offset=0),
globalScope,
isClassContext=True)
code = compile(expr, filename, 'exec')
exec code in globalScope, outputLocals
return list(outputLocals.values())[0]()
elif isinstance(functionAst, ast.With):
expr = ast.FunctionDef()
expr.name = '__pyfora_with_block_as_function__'
expr.args = ast.arguments()
expr.args.args = []
expr.args.defaults = []
expr.args.vararg = None
expr.args.kwarg = None
expr.decorator_list = []
#make sure we copy the list - if we use the existing one, we will mess up the
#cached copy!
expr.body = list(functionAst.body)
expr.lineno = functionAst.lineno - 1
expr.col_offset = functionAst.col_offset
bound_variables = PyAstFreeVariableAnalyses.collectBoundValuesInScope(
expr)
return_dict_list_src = "[" + ",".join(
"'%s'" % k for k in bound_variables) + "]"
return_dict_expr_src = "{{k: __locals[k] for k in {return_dict_list_src} if k in __locals}}".format(
return_dict_list_src=return_dict_list_src)
return_dict_expr_src = "(lambda __locals: {return_dict_expr_src})(dict(locals()))".format(
return_dict_expr_src=return_dict_expr_src)
return_dict_expr = ast.parse(return_dict_expr_src).body[0].value
return_statement = ast.Return(
ast.Tuple(
[return_dict_expr,
ast.Num(0), ast.Num(0)], ast.Load()))
return_statement_exception = ast.Return(
ast.Tuple([
return_dict_expr,
ast.Call(
ast.Name("__pyfora_get_exception_traceback__",
ast.Load()), [], [], None, None),
ast.Name("__pyfora_exception_var__", ast.Load())
], ast.Load()))
expr.body.append(return_statement)
handler = ast.ExceptHandler(
None, ast.Name("__pyfora_exception_var__", ast.Store()),
[return_statement_exception])
#now wrap in a try-catch block
curBody = list(expr.body)
expr.body = [ast.TryExcept(curBody, [handler], [])]
#for every incoming variable 'x' that's also assigned to, create a dummy '__pyfora_var_guard_x' that actually
#takes the value in from the surrounding scope, and immediately assign it
for var in memberDictionary:
if var in bound_variables:
newVar = "__pyfora_var_guard_" + var
var_copy_expr = ast.Assign(
targets=[ast.Name(var, ast.Store())],
value=ast.Name(newVar, ast.Load()))
globalScope[newVar] = globalScope[var]
del globalScope[var]
expr.body = [var_copy_expr] + expr.body
expr = updatePyAstMemberChains(expr,
globalScope,
isClassContext=True)
ast.fix_missing_locations(expr)
def extractTrace():
return sys.exc_info()[2]
globalScope['__pyfora_get_exception_traceback__'] = extractTrace
code = compile(ast.Module([expr]), filename, 'exec')
exec code in globalScope, outputLocals
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
else:
functionAst = updatePyAstMemberChains(ast.Module([functionAst],
lineno=1,
col_offset=0),
globalScope,
isClassContext=False)
code = compile(functionAst, filename, 'exec')
exec code in globalScope, outputLocals
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
def visit_Return(self, node):
return ast.Assign([ast.Name(f"__magma_ssa_return_value", ast.Store())],
node.value)
def visit_Return(self, node):
self.counter += 1
name = f"__magma_ssa_return_value_{self.counter}"
return ast.Assign([ast.Name(name, ast.Store())], node.value)
def visit_Expr(self, node):
'Change last expression into an assignment'
return ast.Assign([ast.Name('result', ast.Store())],
self.generic_visit(node.value))
def _store_ex_dict(self, node: ex_ast.ExDict):
node.ctx = self.ctx
for each in node.values:
self.visit(each)
visit_Name = _store_simply
visit_Subscript = _store_simply
visit_Attribute = _store_simply
visit_Tuple = _store_recursively
visit_List = _store_recursively
visit_ExDict = _store_ex_dict
visit_Starred = _store_recursively
_fix_store = ExprContextFixer(ast.Store()).visit
_fix_del = ExprContextFixer(ast.Del()).visit
class Loc:
def __matmul__(self, other: t.Union[ast.AST, Tokenizer]):
return {
'lineno': other.lineno,
'col_offset':
other.col_offset if hasattr(other, 'col_offset') else other.colno
}
class LocatedError(Exception):
def _makeAttrAssign(name, attr, value, lineno=1, indent=""):
name = ast.Name(id=name, lineno=lineno, col_offset=len(indent), ctx=ast.Load())
attr = ast.Attribute(attr=attr, value=name, lineno=lineno, col_offset = len(f"{indent}{name}."), ctx=ast.Store())
value = ast.Name(id=value, lineno=lineno, col_offset=len(f"{indent}{name}.{attr} = "), ctx=ast.Load())
assign = ast.Assign(targets=[attr], value=value, lineno=lineno, col_offset=len(f"{indent}{name}.{attr} "))
return assign
def _translate_all_expression_to_a_module(
generator_exp: ast.GeneratorExp, generated_function_name: str,
name_to_value: Mapping[str, Any]) -> ast.Module:
"""
Generate the AST of the module to trace an all quantifier on an generator expression.
:param generator_exp: generator expression to be translated
:param generated_function_name: UUID of the tracing function to be used in the code
:param name_to_value:
mapping of all resolved values to the variable names
(passed as arguments to the function so that the generation can access them)
:return: translation to a module
"""
assert generated_function_name not in name_to_value
assert not hasattr(builtins, generated_function_name)
# Collect all the names involved in the generation
relevant_names = _collect_stored_names(
generator.target for generator in generator_exp.generators)
assert generated_function_name not in relevant_names
# Work backwards, from the most-inner block outwards
result_id = 'icontract_tracing_all_result_{}'.format(uuid.uuid4().hex)
result_assignment = ast.Assign(
targets=[ast.Name(id=result_id, ctx=ast.Store())],
value=generator_exp.elt)
exceptional_return = ast.Return(
ast.Tuple(elts=[
ast.Name(id=result_id, ctx=ast.Load()),
ast.Tuple(elts=[
ast.Tuple(elts=[
ast.Constant(value=relevant_name, kind=None),
ast.Name(id=relevant_name, ctx=ast.Load())
],
ctx=ast.Load()) for relevant_name in relevant_names
],
ctx=ast.Load())
],
ctx=ast.Load()))
# While happy return shall not be executed, we add it here for robustness in case
# future refactorings forget to check for that edge case.
happy_return = ast.Return(
ast.Tuple(elts=[
ast.Name(id=result_id, ctx=ast.Load()),
ast.Constant(value=None, kind=None)
],
ctx=ast.Load()))
critical_if: If = ast.If(test=ast.Name(id=result_id, ctx=ast.Load()),
body=[ast.Pass()],
orelse=[exceptional_return])
# Previous inner block to be added as body to the next outer block
block = None # type: Optional[List[ast.stmt]]
for i, comprehension in enumerate(reversed(generator_exp.generators)):
if i == 0:
# This is the inner-most comprehension.
block = [result_assignment, critical_if]
assert block is not None
for condition in reversed(comprehension.ifs):
block = [ast.If(test=condition, body=block, orelse=[])]
if not comprehension.is_async:
block = [
ast.For(target=comprehension.target,
iter=comprehension.iter,
body=block,
orelse=[])
]
else:
block = [
ast.AsyncFor(target=comprehension.target,
iter=comprehension.iter,
body=block,
orelse=[])
]
assert block is not None
block.append(happy_return)
# Now we are ready to generate the function.
is_async = any(comprehension.is_async
for comprehension in generator_exp.generators)
args = [
ast.arg(arg=name, annotation=None)
for name in sorted(name_to_value.keys())
]
if sys.version_info < (3, 5):
raise NotImplementedError(
"Python versions below 3.5 not supported, got: {}".format(
sys.version_info))
if not is_async:
if sys.version_info < (3, 8):
func_def_node = ast.FunctionDef(
name=generated_function_name,
args=ast.arguments(args=args,
kwonlyargs=[],
kw_defaults=[],
defaults=[],
vararg=None,
kwarg=None),
decorator_list=[],
body=block
) # type: Union[ast.FunctionDef, ast.AsyncFunctionDef]
module_node = ast.Module(body=[func_def_node])
else:
func_def_node = ast.FunctionDef(name=generated_function_name,
args=ast.arguments(args=args,
posonlyargs=[],
kwonlyargs=[],
kw_defaults=[],
defaults=[],
vararg=None,
kwarg=None),
decorator_list=[],
body=block)
module_node = ast.Module(body=[func_def_node], type_ignores=[])
else:
if sys.version_info < (3, 8):
async_func_def_node = ast.AsyncFunctionDef(
name=generated_function_name,
args=ast.arguments(args=args,
kwonlyargs=[],
kw_defaults=[],
defaults=[],
vararg=None,
kwarg=None),
decorator_list=[],
body=block)
module_node = ast.Module(body=[async_func_def_node])
else:
async_func_def_node = ast.AsyncFunctionDef(
name=generated_function_name,
args=ast.arguments(args=args,
posonlyargs=[],
kwonlyargs=[],
kw_defaults=[],
defaults=[],
vararg=None,
kwarg=None),
decorator_list=[],
body=block)
module_node = ast.Module(body=[async_func_def_node],
type_ignores=[])
ast.fix_missing_locations(module_node)
return module_node
def visit_Assert(self, assert_):
"""Return the AST statements to replace the ast.Assert instance.
This rewrites the test of an assertion to provide
intermediate values and replace it with an if statement which
raises an assertion error with a detailed explanation in case
the expression is false.
"""
if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1:
from _pytest.warning_types import PytestAssertRewriteWarning
import warnings
warnings.warn_explicit(
PytestAssertRewriteWarning(
"assertion is always true, perhaps remove parentheses?"),
category=None,
filename=str(self.module_path),
lineno=assert_.lineno,
)
self.statements = []
self.variables = []
self.variable_counter = itertools.count()
self.stack = []
self.on_failure = []
self.push_format_context()
# Rewrite assert into a bunch of statements.
top_condition, explanation = self.visit(assert_.test)
# If in a test module, check if directly asserting None, in order to warn [Issue #3191]
if self.module_path is not None:
self.statements.append(
self.warn_about_none_ast(top_condition,
module_path=self.module_path,
lineno=assert_.lineno))
# Create failure message.
body = self.on_failure
negation = ast.UnaryOp(ast.Not(), top_condition)
self.statements.append(ast.If(negation, body, []))
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
explanation = "\n>assert " + explanation
else:
assertmsg = ast.Str("")
explanation = "assert " + explanation
template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
msg = self.pop_format_context(template)
fmt = self.helper("_format_explanation", msg)
err_name = ast.Name("AssertionError", ast.Load())
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
body.append(raise_)
# Clear temporary variables by setting them to None.
if self.variables:
variables = [
ast.Name(name, ast.Store()) for name in self.variables
]
clear = ast.Assign(variables, _NameConstant(None))
self.statements.append(clear)
# Fix line numbers.
for stmt in self.statements:
set_location(stmt, assert_.lineno, assert_.col_offset)
return self.statements
def execute_code_block(compiler, block, example_globals, script_vars,
gallery_conf):
"""Executes the code block of the example file"""
blabel, bcontent, lineno = block
# If example is not suitable to run, skip executing its blocks
if not script_vars['execute_script'] or blabel == 'text':
return ''
cwd = os.getcwd()
# Redirect output to stdout and
orig_stdout, orig_stderr = sys.stdout, sys.stderr
src_file = script_vars['src_file']
# First cd in the original example dir, so that any file
# created by the example get created in this directory
captured_std = StringIO()
os.chdir(os.path.dirname(src_file))
sys_path = copy.deepcopy(sys.path)
sys.path.append(os.getcwd())
sys.stdout = sys.stderr = LoggingTee(captured_std, logger, src_file)
try:
dont_inherit = 1
if sys.version_info >= (3, 8):
ast_Module = partial(ast.Module, type_ignores=[])
else:
ast_Module = ast.Module
code_ast = ast_Module([bcontent])
code_ast = compile(bcontent, src_file, 'exec',
ast.PyCF_ONLY_AST | compiler.flags, dont_inherit)
ast.increment_lineno(code_ast, lineno - 1)
# capture output if last line is expression
is_last_expr = False
if len(code_ast.body) and isinstance(code_ast.body[-1], ast.Expr):
is_last_expr = True
last_val = code_ast.body.pop().value
# exec body minus last expression
_, mem_body = _memory_usage(
_exec_once(compiler(code_ast, src_file, 'exec'),
example_globals), gallery_conf)
# exec last expression, made into assignment
body = [
ast.Assign(targets=[ast.Name(id='___', ctx=ast.Store())],
value=last_val)
]
last_val_ast = ast_Module(body=body)
ast.fix_missing_locations(last_val_ast)
_, mem_last = _memory_usage(
_exec_once(compiler(last_val_ast, src_file, 'exec'),
example_globals), gallery_conf)
# capture the assigned variable
___ = example_globals['___']
mem_max = max(mem_body, mem_last)
else:
_, mem_max = _memory_usage(
_exec_once(compiler(code_ast, src_file, 'exec'),
example_globals), gallery_conf)
script_vars['memory_delta'].append(mem_max)
except Exception:
sys.stdout.flush()
sys.stderr.flush()
sys.stdout, sys.stderr = orig_stdout, orig_stderr
except_rst = handle_exception(sys.exc_info(), src_file, script_vars,
gallery_conf)
code_output = u"\n{0}\n\n\n\n".format(except_rst)
# still call this even though we won't use the images so that
# figures are closed
save_figures(block, script_vars, gallery_conf)
else:
sys.stdout.flush()
sys.stderr.flush()
sys.stdout, orig_stderr = orig_stdout, orig_stderr
sys.path = sys_path
os.chdir(cwd)
last_repr = None
repr_meth = None
if gallery_conf['capture_repr'] != () and is_last_expr:
for meth in gallery_conf['capture_repr']:
try:
last_repr = getattr(___, meth)()
# for case when last statement is print()
if last_repr is None or last_repr == 'None':
repr_meth = None
else:
repr_meth = meth
except Exception:
pass
else:
if isinstance(last_repr, str):
break
captured_std = captured_std.getvalue().expandtabs()
# normal string output
if repr_meth in ['__repr__', '__str__'] and last_repr:
captured_std = u"{0}\n{1}".format(captured_std, last_repr)
if captured_std and not captured_std.isspace():
captured_std = CODE_OUTPUT.format(indent(captured_std, u' ' * 4))
else:
captured_std = ''
images_rst = save_figures(block, script_vars, gallery_conf)
# give html output its own header
if repr_meth == '_repr_html_':
captured_html = html_header.format(indent(last_repr, u' ' * 8))
else:
captured_html = ''
code_output = u"\n{0}\n\n{1}\n{2}\n\n".format(images_rst, captured_std,
captured_html)
finally:
os.chdir(cwd)
sys.path = sys_path
sys.stdout, sys.stderr = orig_stdout, orig_stderr
return code_output
def _compile_directive_call_assets(self, el, options):
""" This special 't-call' tag can be used in order to aggregate/minify javascript and css assets"""
if len(el):
raise SyntaxError("t-call-assets cannot contain children nodes")
# nodes = self._get_asset(xmlid, options, css=css, js=js, debug=values.get('debug'), async=async, values=values)
#
# for index, (tagName, t_attrs, content) in enumerate(nodes):
# if index:
# append('\n ')
# append('<')
# append(tagName)
#
# self._post_processing_att(tagName, t_attrs, options)
# for name, value in t_attrs.items():
# if value or isinstance(value, string_types)):
# append(u' ')
# append(name)
# append(u'="')
# append(escape(pycompat.to_text((value)))
# append(u'"')
#
# if not content and tagName in self._void_elements:
# append('/>')
# else:
# append('>')
# if content:
# append(content)
# append('</')
# append(tagName)
# append('>')
#
space = el.getprevious() is not None and el.getprevious(
).tail or el.getparent().text
sep = u'\n' + space.rsplit('\n').pop()
return [
ast.Assign(
targets=[ast.Name(id='nodes', ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(value=ast.Name(id='self',
ctx=ast.Load()),
attr='_get_asset',
ctx=ast.Load()),
args=[
ast.Str(el.get('t-call-assets')),
ast.Name(id='options', ctx=ast.Load()),
],
keywords=[
ast.keyword('css',
self._get_attr_bool(el.get('t-css',
True))),
ast.keyword('js',
self._get_attr_bool(el.get('t-js', True))),
ast.keyword(
'debug',
ast.Call(func=ast.Attribute(value=ast.Name(
id='values', ctx=ast.Load()),
attr='get',
ctx=ast.Load()),
args=[ast.Str('debug')],
keywords=[],
starargs=None,
kwargs=None)),
ast.keyword(
'async',
self._get_attr_bool(el.get('async', False))),
ast.keyword('values',
ast.Name(id='values', ctx=ast.Load())),
],
starargs=None,
kwargs=None)),
ast.For(
target=ast.Tuple(elts=[
ast.Name(id='index', ctx=ast.Store()),
ast.Tuple(elts=[
ast.Name(id='tagName', ctx=ast.Store()),
ast.Name(id='t_attrs', ctx=ast.Store()),
ast.Name(id='content', ctx=ast.Store())
],
ctx=ast.Store())
],
ctx=ast.Store()),
iter=ast.Call(func=ast.Name(id='enumerate', ctx=ast.Load()),
args=[ast.Name(id='nodes', ctx=ast.Load())],
keywords=[],
starargs=None,
kwargs=None),
body=[
ast.If(test=ast.Name(id='index', ctx=ast.Load()),
body=[self._append(ast.Str(sep))],
orelse=[]),
self._append(ast.Str(u'<')),
self._append(ast.Name(id='tagName', ctx=ast.Load())),
] + self._append_attributes() + [
ast.If(test=ast.BoolOp(
op=ast.And(),
values=[
ast.UnaryOp(ast.Not(),
ast.Name(id='content', ctx=ast.Load()),
lineno=0,
col_offset=0),
ast.Compare(
left=ast.Name(id='tagName', ctx=ast.Load()),
ops=[ast.In()],
comparators=[
ast.Attribute(value=ast.Name(
id='self', ctx=ast.Load()),
attr='_void_elements',
ctx=ast.Load())
]),
]),
body=[self._append(ast.Str(u'/>'))],
orelse=[
self._append(ast.Str(u'>')),
ast.If(test=ast.Name(id='content',
ctx=ast.Load()),
body=[
self._append(
ast.Name(id='content',
ctx=ast.Load()))
],
orelse=[]),
self._append(ast.Str(u'</')),
self._append(
ast.Name(id='tagName', ctx=ast.Load())),
self._append(ast.Str(u'>')),
])
],
orelse=[])
]
def test_assign_multiple_slice(self):
self.v.visit(
ast.Assign([
ast.Subscript(ast_load('foo'), ast.Slice(), ast.Store()),
ast_store('bar'),
], ast.Str('test')))
def name(v, store=False):
return ast.Name(id=v, ctx=ast.Load() if not store else ast.Store(), lineno=1, col_offset=0)
import ast
from anoky.expansion.expansion_context import ExpansionContext
from anoky.macros.macro import Macro
from anoky.special_forms.special_form import SpecialForm
macrostore_init_code = [
# import anoky.module as __anoky__
ast.Import([ast.alias(name="anoky.module", asname="__aky__")]),
# __macros__ = {} (and for id macros and special forms)
ast.Assign(targets=[ast.Name(id="__macros__", ctx=ast.Store())],
value=ast.Dict([], [])),
ast.Assign(targets=[ast.Name(id="__id_macros__", ctx=ast.Store())],
value=ast.Dict([], [])),
ast.Assign(targets=[ast.Name(id="__special_forms__", ctx=ast.Store())],
value=ast.Dict([], []))
]
def generate_macro_store_code(mac_name, mac:Macro):
# __macros__[mac_name]
store_as_code = ast.Subscript(value=ast.Name(id="__macros__", ctx=ast.Load()),
slice=ast.Index(ast.Str(mac_name)),
ctx=ast.Store())
# ~~RHS~~
#macro_gen_code =
raise NotImplementedError()
def res_python_setup(res: Property) -> Tuple[Callable[[EntityFixup], object], str]:
variables = {}
variable_order = []
code = None
result_var = None
for child in res:
if child.name.startswith('$'):
if child.value.casefold() not in FUNC_GLOBALS:
raise Exception('Invalid variable type! ({})'.format(child.value))
variables[child.name[1:]] = child.value.casefold()
variable_order.append(child.name[1:])
elif child.name == 'op':
code = child.value
elif child.name == 'resultvar':
result_var = child.value
else:
raise Exception('Invalid key "{}"'.format(child.real_name))
if not code:
raise Exception('No operation specified!')
if not result_var:
raise Exception('No destination specified!')
for name in variables:
if name.startswith('_'):
raise Exception('"{}" is not permitted as a variable name!'.format(name))
# Allow $ in the variable names..
code = code.replace('$', '')
# Now process the code to convert it into a function taking variables
# and returning them.
# We also need to whitelist operations for security.
expression = ast.parse(
code,
'<bee2_op>',
mode='eval',
).body
Checker(variable_order).visit(expression)
# For each variable, do
# var = func(_fixup['var'])
statements: List[ast.AST] = [
ast.Assign(
targets=[ast.Name(id=var_name, ctx=ast.Store())],
value=ast.Call(
func=ast.Name(id=variables[var_name], ctx=ast.Load()),
args=[
ast.Subscript(
value=ast.Name(id='_fixup', ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=var_name)),
ctx=ast.Load(),
),
],
keywords=[],
starargs=None,
kwargs=None,
)
)
for line_num, var_name in enumerate(
variable_order, start=1,
)
]
# The last statement returns the target expression.
statements.append(ast.Return(expression, lineno=len(variable_order)+1, col_offset=0))
args = ast.arguments(
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
)
# Py 3.8+, make it pos-only.
if 'posonlyargs' in args._fields:
args.posonlyargs = [ast.arg('_fixup', None)]
args.args = []
else: # Just make it a regular arg.
args.args = [ast.arg('_fixup', None)]
func = ast.Module([
ast.FunctionDef(
name='_bee2_generated_func',
args=args,
body=statements,
decorator_list=[],
),
],
lineno=1,
col_offset=0,
)
# Python 3.8 also
if 'type_ignores' in func._fields:
func.type_ignores = []
# Fill in lineno and col_offset
ast.fix_missing_locations(func)
ns: Dict[str, Any] = {}
eval(compile(func, '<bee2_op>', mode='exec'), FUNC_GLOBALS, ns)
compiled_func = ns['_bee2_generated_func']
compiled_func.__name__ = '<bee2_func>'
return compiled_func, result_var
def generate_idmacro_store_code(idmac_name, idmac:Macro):
store_as_code = ast.Subscript(value=ast.Name(id="__id_macros__", ctx=ast.Load()),
slice=ast.Index(ast.Str(idmac_name)),
ctx=ast.Store())
def visit_Assert(self, assert_: ast.Assert) -> List[ast.stmt]:
"""Return the AST statements to replace the ast.Assert instance.
This rewrites the test of an assertion to provide
intermediate values and replace it with an if statement which
raises an assertion error with a detailed explanation in case
the expression is false.
"""
if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1:
from _pytest.warning_types import PytestAssertRewriteWarning
import warnings
# TODO: This assert should not be needed.
assert self.module_path is not None
warnings.warn_explicit(
PytestAssertRewriteWarning(
"assertion is always true, perhaps remove parentheses?"
),
category=None,
filename=os.fspath(self.module_path),
lineno=assert_.lineno,
)
self.statements: List[ast.stmt] = []
self.variables: List[str] = []
self.variable_counter = itertools.count()
if self.enable_assertion_pass_hook:
self.format_variables: List[str] = []
self.stack: List[Dict[str, ast.expr]] = []
self.expl_stmts: List[ast.stmt] = []
self.push_format_context()
# Rewrite assert into a bunch of statements.
top_condition, explanation = self.visit(assert_.test)
negation = ast.UnaryOp(ast.Not(), top_condition)
if self.enable_assertion_pass_hook: # Experimental pytest_assertion_pass hook
msg = self.pop_format_context(ast.Str(explanation))
# Failed
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
gluestr = "\n>assert "
else:
assertmsg = ast.Str("")
gluestr = "assert "
err_explanation = ast.BinOp(ast.Str(gluestr), ast.Add(), msg)
err_msg = ast.BinOp(assertmsg, ast.Add(), err_explanation)
err_name = ast.Name("AssertionError", ast.Load())
fmt = self.helper("_format_explanation", err_msg)
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
statements_fail = []
statements_fail.extend(self.expl_stmts)
statements_fail.append(raise_)
# Passed
fmt_pass = self.helper("_format_explanation", msg)
orig = self._assert_expr_to_lineno()[assert_.lineno]
hook_call_pass = ast.Expr(
self.helper(
"_call_assertion_pass",
ast.Num(assert_.lineno),
ast.Str(orig),
fmt_pass,
)
)
# If any hooks implement assert_pass hook
hook_impl_test = ast.If(
self.helper("_check_if_assertion_pass_impl"),
self.expl_stmts + [hook_call_pass],
[],
)
statements_pass = [hook_impl_test]
# Test for assertion condition
main_test = ast.If(negation, statements_fail, statements_pass)
self.statements.append(main_test)
if self.format_variables:
variables = [
ast.Name(name, ast.Store()) for name in self.format_variables
]
clear_format = ast.Assign(variables, ast.NameConstant(None))
self.statements.append(clear_format)
else: # Original assertion rewriting
# Create failure message.
body = self.expl_stmts
self.statements.append(ast.If(negation, body, []))
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
explanation = "\n>assert " + explanation
else:
assertmsg = ast.Str("")
explanation = "assert " + explanation
template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
msg = self.pop_format_context(template)
fmt = self.helper("_format_explanation", msg)
err_name = ast.Name("AssertionError", ast.Load())
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
body.append(raise_)
# Clear temporary variables by setting them to None.
if self.variables:
variables = [ast.Name(name, ast.Store()) for name in self.variables]
clear = ast.Assign(variables, ast.NameConstant(None))
self.statements.append(clear)
# Fix line numbers.
for stmt in self.statements:
set_location(stmt, assert_.lineno, assert_.col_offset)
return self.statements
def node():
"""Get an ast.Assign node of 'x = 1'."""
return ast.Assign(targets=[ast.Name(id='x', ctx=ast.Store())],
value=ast.Num(n=1))
def visit_Assign(self, node):
assert (len(node.targets) == 1)
self.generic_visit(node)
decorated = isinstance(node.value, ast.Call) and isinstance(
node.value.func, ast.Attribute) and isinstance(node.value.func.value, ast.Name) \
and node.value.func.value.id == 'ti'
is_static_assign = False
if decorated:
attr = node.value.func
if attr.attr == 'static':
is_static_assign = True
else:
pass # eg. x = ti.cast(xx) will reach here, but they're not decorators, so no raising errors here
if is_static_assign:
return node
if isinstance(node.targets[0], ast.Tuple):
targets = node.targets[0].elts
# Create
stmts = []
holder = self.parse_stmt('__tmp_tuple = ti.expr_init_list(0, '
f'{len(targets)})')
holder.value.args[0] = node.value
stmts.append(holder)
def tuple_indexed(i):
indexing = self.parse_stmt('__tmp_tuple[0]')
self.set_subscript_index(indexing.value,
self.parse_expr("{}".format(i)))
return indexing.value
for i, target in enumerate(targets):
is_local = isinstance(target, ast.Name)
if is_local and self.is_creation(target.id):
var_name = target.id
target.ctx = ast.Store()
# Create
init = ast.Attribute(value=ast.Name(id='ti',
ctx=ast.Load()),
attr='expr_init',
ctx=ast.Load())
rhs = ast.Call(
func=init,
args=[tuple_indexed(i)],
keywords=[],
)
self.create_variable(var_name)
stmts.append(ast.Assign(targets=[target], value=rhs))
else:
# Assign
target.ctx = ast.Load()
func = ast.Attribute(value=target,
attr='assign',
ctx=ast.Load())
call = ast.Call(func=func,
args=[tuple_indexed(i)],
keywords=[])
stmts.append(ast.Expr(value=call))
for stmt in stmts:
ast.copy_location(stmt, node)
stmts.append(self.parse_stmt('del __tmp_tuple'))
return self.make_single_statement(stmts)
else:
is_local = isinstance(node.targets[0], ast.Name)
if is_local and self.is_creation(node.targets[0].id):
var_name = node.targets[0].id
# Create
init = ast.Attribute(value=ast.Name(id='ti', ctx=ast.Load()),
attr='expr_init',
ctx=ast.Load())
rhs = ast.Call(
func=init,
args=[node.value],
keywords=[],
)
self.create_variable(var_name)
return ast.copy_location(
ast.Assign(targets=node.targets, value=rhs), node)
else:
# Assign
node.targets[0].ctx = ast.Load()
func = ast.Attribute(value=node.targets[0],
attr='assign',
ctx=ast.Load())
call = ast.Call(func=func, args=[node.value], keywords=[])
return ast.copy_location(ast.Expr(value=call), node)
def p_target(t):
'''target : IDENTIFIER'''
identifier = ast.Name(t[1], ast.Store())
identifier.lineno = t.lineno(1)
identifier.col_offset = -1 # XXX
t[0] = [identifier]
