def collect_var_assignments(pat, val):
"""This helper function ensures that the pattern is used to
properly assign all subfields of the given AST for use
in the clause body
E.g., for PatternConstructor(A, PatternVar(v), PatternWildcard(),
PatternConstructor(B, PatternVar(w)))
we would want to have
v = a.fields[0]
w = a.fields[2].fields[0]
"""
if isinstance(pat, relay.PatternWildcard):
return []
if isinstance(pat, relay.PatternVar):
return [Assign([self.include_var(pat.var, assign=True)], val)]
# constructor pattern: assign each field of the value
# based on subpatterns
assignments = []
for i in range(len(pat.patterns)):
# we want the assignments for val.fields[i]
field = ast.Subscript(ast.Attribute(val, 'fields', Load()),
ast.Index(Num(i)), Load())
assignments += collect_var_assignments(pat.patterns[i], field)
return assignments
def rewrite_array_iteration(self, node):
"""
Convert 1D array iteration to for-range and indexing:
for value in my_array:
...
becomes
for i in my_array.shape[0]:
value = my_array[i]
...
"""
logger.debug(ast.dump(node))
orig_target = node.target
orig_iter = node.iter
#--------------------------------------------------------------------
# Replace node.target with a temporary
#--------------------------------------------------------------------
target_name = orig_target.id + '.idx'
target_temp = nodes.TempNode(Py_ssize_t)
node.target = target_temp.store()
#--------------------------------------------------------------------
# Create range(A.shape[0])
#--------------------------------------------------------------------
call_func = ast.Name(id='range', ctx=ast.Load())
nodes.typednode(call_func, typesystem.range_)
shape_index = ast.Index(nodes.ConstNode(0, typesystem.Py_ssize_t))
shape_index.type = typesystem.npy_intp
stop = ast.Subscript(value=nodes.ShapeAttributeNode(orig_iter),
slice=shape_index,
ctx=ast.Load())
nodes.typednode(stop, npy_intp)
#--------------------------------------------------------------------
# Create range iterator and replace node.iter
#--------------------------------------------------------------------
call_args = [
nodes.ConstNode(0, typesystem.Py_ssize_t),
nodes.CoercionNode(stop, typesystem.Py_ssize_t),
nodes.ConstNode(1, typesystem.Py_ssize_t),
]
node.iter = ast.Call(func=call_func, args=call_args)
nodes.typednode(node.iter, call_func.type)
node.index = target_temp.load(invariant=True)
#--------------------------------------------------------------------
# Add assignment to new target variable at the start of the body
#--------------------------------------------------------------------
index = ast.Index(value=node.index)
index.type = target_temp.type
subscript = ast.Subscript(value=orig_iter, slice=index, ctx=ast.Load())
nodes.typednode(subscript, get_type(orig_iter).dtype)
#--------------------------------------------------------------------
# Add assignment to new target variable at the start of the body
#--------------------------------------------------------------------
assign = ast.Assign(targets=[orig_target], value=subscript)
node.body = [assign] + node.body
#--------------------------------------------------------------------
# Specialize new for loop through range iteration
#--------------------------------------------------------------------
return self.visit(node)
def get_ast(self, *arg_asts, **kwarg_asts):
"""Transform this object into a Python AST. When serialized and
executed, the AST will do exactly the same as calling this
object."""
if Primitive._DEBUG:
debug_output(repr(self))
debug_output(" arg_asts: " + repr(arg_asts))
new_prim = self.fill_slots(arg_asts, kwarg_asts, convert_to_ast=True)
if not new_prim.are_slots_filled():
raise PyExportError("not enough arguments")
if Primitive._DEBUG:
debug_output(" new_prim.arg_descs: " + repr(new_prim.arg_descs))
# extract the actual values from the (now constant) arguments
(new_arg_asts, new_kwarg_asts) = new_prim.get_values_of_filled_slots(
exportable_only=True)
if Primitive._DEBUG:
debug_output(" new_arg_asts: " + repr(new_arg_asts))
debug_output("end " + repr(self))
# SPECIAL HANDLING #
# loops
if self == LogoCode.prim_loop:
controller = self._get_loop_controller()
if controller == Primitive.controller_repeat:
# 'repeat' loop
num_repetitions = new_arg_asts[0]
if num_repetitions.func.id == 'controller_repeat':
num_repetitions = num_repetitions.args[0]
repeat_iter = get_call_ast("range", [num_repetitions])
# TODO use new variable name in nested loops
loop_ast = ast.For(target=ast.Name(id="i", ctx=ast.Store),
iter=repeat_iter,
body=new_arg_asts[1],
orelse=[])
return loop_ast
else:
if controller == Primitive.controller_forever:
condition_ast = ast.Name(id="True", ctx=ast.Load)
elif controller == Primitive.controller_while:
condition_ast = new_arg_asts[0].args[0]
elif controller == Primitive.controller_until:
pos_cond_ast = new_arg_asts[0].args[0]
condition_ast = ast.UnaryOp(op=ast.Not,
operand=pos_cond_ast)
else:
raise PyExportError("unknown loop controller: " +
repr(controller))
loop_ast = ast.While(test=condition_ast,
body=new_arg_asts[1],
orelse=[])
# Until always executes its body once.
if controller == Primitive.controller_until:
loop_list = []
for arg_ast in new_arg_asts[1]:
loop_list.append(arg_ast)
loop_list.append(loop_ast)
return loop_list
else:
return loop_ast
# conditionals
elif self in (LogoCode.prim_if, LogoCode.prim_ifelse):
test = new_arg_asts[0]
body = new_arg_asts[1]
if len(new_arg_asts) > 2:
orelse = new_arg_asts[2]
else:
orelse = []
if_ast = ast.If(test=test, body=body, orelse=orelse)
return if_ast
# boxes
elif self == LogoCode.prim_set_box:
target_ast = ast.Subscript(value=BOX_AST,
slice=ast.Index(value=new_arg_asts[0]),
ctx=ast.Store)
return ast.Assign(targets=[target_ast], value=new_arg_asts[1])
elif self == LogoCode.prim_get_box:
return ast.Subscript(value=BOX_AST,
slice=ast.Index(value=new_arg_asts[0]),
ctx=ast.Load)
# action stacks
elif self == LogoCode.prim_define_stack:
return
elif self == LogoCode.prim_invoke_stack:
stack_func = ast.Subscript(
value=ACTION_AST,
slice=ast.Index(value=new_arg_asts[0]), ctx=ast.Load)
call_ast = get_call_ast('logo.icall', [stack_func])
return [call_ast, ast_yield_true()]
# stop stack
elif self == LogoCode.prim_stop_stack:
return ast.Return()
# sleep/ wait
elif self == LogoCode.prim_wait:
return [get_call_ast('sleep', new_arg_asts), ast_yield_true()]
# standard operators
elif self.func.__name__ in Primitive.STANDARD_OPERATORS:
op = Primitive.STANDARD_OPERATORS[self.func.__name__]
# 'divide': prevent unwanted integer division
if self == Primitive.divide:
def _is_float(x):
return get_type(x)[0] == TYPE_FLOAT
if (not _is_float(new_arg_asts[0]) and
not _is_float(new_arg_asts[1])):
new_arg_asts[0] = get_call_ast('float', [new_arg_asts[0]],
return_type=TYPE_FLOAT)
if len(new_arg_asts) == 1:
if isinstance(op, tuple):
op = op[0]
return ast.UnaryOp(op=op, operand=new_arg_asts[0])
elif len(new_arg_asts) == 2:
if isinstance(op, tuple):
op = op[1]
(left, right) = new_arg_asts
if issubclass(op, ast.boolop):
return ast.BoolOp(op=op, values=[left, right])
elif issubclass(op, ast.cmpop):
return ast.Compare(left=left, ops=[op],
comparators=[right])
else:
return ast.BinOp(op=op, left=left, right=right)
# f(x)
elif self == LogoCode.prim_myfunction:
param_asts = []
for id_ in ['x', 'y', 'z'][:len(new_arg_asts)-1]:
param_asts.append(ast.Name(id=id_, ctx=ast.Param))
func_ast = ast_extensions.LambdaWithStrBody(
body_str=new_arg_asts[0].s, args=param_asts)
return get_call_ast(func_ast, new_arg_asts[1:],
return_type=self.return_type)
# square root
elif self == Primitive.square_root:
return get_call_ast('sqrt', new_arg_asts, new_kwarg_asts,
return_type=self.return_type)
# random
elif self in (Primitive.random_char, Primitive.random_int):
uniform_ast = get_call_ast('uniform', new_arg_asts)
round_ast = get_call_ast('round', [uniform_ast, ast.Num(n=0)])
int_ast = get_call_ast('int', [round_ast], return_type=TYPE_INT)
if self == Primitive.random_char:
chr_ast = get_call_ast('chr', [int_ast], return_type=TYPE_CHAR)
return chr_ast
else:
return int_ast
# identity
elif self == Primitive.identity:
return new_arg_asts[0]
# constant
elif self == CONSTANTS.get:
return TypedSubscript(value=ast.Name(id='CONSTANTS', ctx=ast.Load),
slice_=ast.Index(value=new_arg_asts[0]),
return_type=self.return_type)
# group of Primitives or sandwich-clamp block
elif self in (Primitive.group, LogoCode.prim_clamp):
ast_list = []
for prim in new_arg_asts[0]:
if export_me(prim):
new_ast = value_to_ast(prim)
if isinstance(new_ast, ast.AST):
ast_list.append(new_ast)
return ast_list
# set turtle
elif self == LogoCode.prim_turtle:
text = 'turtle = turtles.get_active_turtle()'
return [get_call_ast('logo.prim_turtle', new_arg_asts),
ast_extensions.ExtraCode(text)]
elif self == LogoCode.active_turtle:
text = 'turtle = turtles.get_active_turtle()'
return ast_extensions.ExtraCode(text)
# comment
elif self == Primitive.comment:
if isinstance(new_arg_asts[0], ast.Str):
text = ' ' + str(new_arg_asts[0].s)
else:
text = ' ' + str(new_arg_asts[0])
return ast_extensions.Comment(text)
# print
elif self == TurtleArtWindow.print_:
func_name = self.get_name_for_export()
call_ast = get_call_ast(func_name, new_arg_asts)
print_ast = ast.Print(values=new_arg_asts[:1], dest=None, nl=True)
return [call_ast, print_ast]
# heap
elif self == LogoCode.get_heap:
return TypedName(id_='logo.heap', return_type=self.return_type)
elif self == LogoCode.reset_heap:
target_ast = ast.Name(id='logo.heap', ctx=ast.Store)
value_ast = ast.List(elts=[], ctx=ast.Load)
return ast.Assign(targets=[target_ast], value=value_ast)
# NORMAL FUNCTION CALL #
else:
func_name = self.get_name_for_export()
return get_call_ast(func_name, new_arg_asts, new_kwarg_asts,
return_type=self.return_type)
def _make_for_loops_while(parent_node, names_in_use):
"""Converts for loops into while loops.
Creates an index variable and a call to the len() function as a test
condition for the while loop.
All for loop iterators must be indexable.
DOES NOT SUPPOT NONINDEXABLE ITERATORS.
Parameters:
parent_node: ast node
Returns:
parent node with updates"""
# Get every index of for loop objects in the body of parent node.
# Could be done cleaner with a numpy .where, but we'd have to import numpy
# pretty much just for that.
try:
indxs_for_loops = []
for indx in range(len(parent_node.body)):
current = parent_node.body[indx]
if isinstance(current, ast.For):
indxs_for_loops.append(indx)
if hasattr(current, "body"):
is_module = isinstance(current, ast.Module)
is_func_def = isinstance(current, ast.FunctionDef)
if not is_func_def and not is_module:
current, names_in_use = _make_for_loops_while(
current, names_in_use)
except AttributeError:
# node has no body. No for loops in it.
return parent_node, names_in_use
num_lines_inserted = 0
for for_loop_index in indxs_for_loops:
for_loop_index = for_loop_index + num_lines_inserted
for_loop = parent_node.body[for_loop_index]
# Make loop incrementor variable.
name_incrementor_variable = _new_name('loop_index', names_in_use)
names_in_use[name_incrementor_variable] = 1
# Make a call to built in len() function with the iterator
# provided in the for loop.
len_builtin_function = ast.Name(id='len', ctx=ast.Load)
len_function_call = ast.Call(func=len_builtin_function,
args=[for_loop.iter],
keywords=[])
# Test for while loop.
left = ast.Name(id=name_incrementor_variable, ctx=ast.Load)
compare_op = ast.Compare(left=left,
ops=[ast.Lt()],
comparators=[len_function_call])
# Assign current value of loop to for loop target.
index = ast.Index(ast.Name(id=name_incrementor_variable, ctx=ast.Load))
value = ast.Subscript(for_loop.iter, index)
target = [for_loop.target]
assign_to_for_loop_target = ast.Assign(target, value)
# Increment index variable.
name = ast.Name(id=name_incrementor_variable)
add_1_to_index_variable = ast.AugAssign(name, ast.Add(), ast.Num(1))
# Construct while loop.
while_loop = [assign_to_for_loop_target] + \
for_loop.body + [add_1_to_index_variable]
while_loop = ast.While(test=compare_op, body=while_loop, orelse=[])
# Replace for with while loop.
parent_node.body[for_loop_index] = while_loop
# Insert loop incrementor variable before while loop and set to 0.
inc_name = ast.Name(id=name_incrementor_variable, ctx=ast.Store)
inc_0 = ast.Assign([inc_name], ast.Num(0))
parent_node.body.insert(for_loop_index, inc_0)
# Not the total lines inserted, only the lines inserted into the
# parent's body. (so not the lines inside the loop)
num_lines_inserted += 1
return parent_node, names_in_use
def gen_subscript(var_name, index):
""" Generates code like variable[1] """
return ast.Expr(
value=ast.Subscript(value=ast.Name(id=var_name, ctx=ast.Load()),
slice=ast.Index(value=ast.Num(n=index)),
ctx=ast.Load()))
def name__NAME(p):
return ast.Subscript(
value=ast.Name(id='context', ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=p[0].getstr()), ctx=ast.Load()),
ctx=ast.Load(),
)
def adjust_slice(s):
if isinstance(s, (ast.Slice, ast.Ellipsis)):
return s
else:
return ast.Index(s)
class HaskellASTVisitor(ast.NodeVisitor):
_globalVariables = {
'fst':
ast.FunctionDef(
'fst', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()), ast.Index(ast.Num(0)),
ast.Load()), None),
'snd':
ast.FunctionDef(
'snd', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()), ast.Index(ast.Num(1)),
ast.Load()), None),
'take':
ast.FunctionDef(
'take', [ast.Name('n', ast.Store()),
ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()),
ast.Slice(None, ast.Name('n', ast.Load), None),
ast.Load()), None),
'drop':
ast.FunctionDef(
'drop', [ast.Name('n', ast.Store()),
ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()),
ast.Slice(ast.Name('n', ast.Load), None, None),
ast.Load()), None),
'reverse':
ast.FunctionDef(
'reverse', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()),
ast.Slice(None, None, ast.Num(-1)), ast.Load()),
None),
'init':
ast.FunctionDef(
'init', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()),
ast.Slice(None, ast.Num(-1), None), ast.Load()),
None),
'tail':
ast.FunctionDef(
'tail', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()),
ast.Slice(ast.Num(1), None, None), ast.Load()),
None),
'last':
ast.FunctionDef(
'last', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()), ast.Index(ast.Num(-1)),
ast.Load()), None),
'head':
ast.FunctionDef(
'head', [ast.Name('l', ast.Store())],
ast.Subscript(ast.Name('l', ast.Load()), ast.Index(ast.Num(0)),
ast.Load()), None),
'length':
ast.FunctionDef(
'length', [ast.Name('l', ast.Store())],
ast.Attribute(ast.Name('l', ast.Load()), '!length', ast.Load()),
None),
'null':
ast.FunctionDef(
'null', [ast.Name('l', ast.Store())],
ast.Compare(ast.Name('l', ast.Load()), [ast.Eq],
[ast.List([], ast.Store())]), None),
'replicate':
ast.FunctionDef(
'replicate',
[ast.Name('n', ast.Store()),
ast.Name('l', ast.Store())],
ast.BinOp(ast.List([ast.Name('l', ast.Load())], ast.Store()),
ast.Mult, ast.Name('n', ast.Load())), None),
'cycle':
ast.FunctionDef(
'cycle', [ast.Name('l', ast.Store())],
ast.List(LoopedList(ast.Name('l', ast.Load())), ast.Store()),
None),
'repeat':
ast.FunctionDef(
'repeat', [ast.Name('l', ast.Store())],
ast.List(LoopedList([ast.Name('l', ast.Load())]), ast.Store()),
None),
}
_funVariablesStack = []
def generic_visit(self, node):
return node
def visit_Assign(self, node):
result = self.visit(node.value)
self._globalVariables[node.targets.id] = result
return result
def visit_Name(self, node):
name = None
try:
name = self._funVariablesStack[0][node.id]
except IndexError:
name = self._globalVariables.get(node.id, None)
except KeyError:
name = self._globalVariables.get(node.id, None)
return name
def visit_BinOp(self, node):
operator = node.op
result = None
if operator == ast.Add:
result = self.visit(node.left) + self.visit(node.right)
elif operator == ast.Sub:
if isinstance(node.left, str):
result = ord(self.visit(node.left)) - ord(
self.visit(node.right))
else:
result = self.visit(node.left) - self.visit(node.right)
elif operator == ast.Div:
result = self.visit(node.left) / self.visit(node.right)
elif operator == ast.Mult:
result = self.visit(node.left) * self.visit(node.right)
return result
def visit_Num(self, node):
return self.visit(node.n) if not isinstance(node.n, int) else node.n
def visit_BoolOp(self, node):
if len(node.values) < 1:
return None
if node.op == None:
return node.values[0]
elif node.op == ast.And:
return self.visit(node.values[0]) and self.visit(node.values[1])
elif node.op == ast.Or:
return self.visit(node.values[0]) or self.visit(node.values[1])
elif node.op == ast.Not:
return not self.visit(node.values[0])
def visit_Compare(self, node):
operator = node.ops[0]
left = self.visit(node.left)
right = self.visit(node.comparators[0])
if operator == ast.Gt: return left > right
elif operator == ast.GtE: return left >= right
elif operator == ast.Lt: return left < right
elif operator == ast.LtE: return left <= right
elif operator == ast.Eq: return left == right
elif operator == ast.NotEq: return left != right
def visit_FunctionDef(self, node):
if node.name.id in (self._globalVariables.keys() +
self._globalVariables.keys()):
print "Name already in use"
return None
self._globalVariables[node.name.id] = node
return node.name.id
def visit_Call(self, node):
fun = self.visit(node.func)
parameters = map(lambda x: self.visit(x), node.args)
args = map(lambda x: x.id, fun.args)
self._funVariablesStack.insert(0, dict(zip(args, parameters)))
retval = self.visit(fun.body)
self._funVariablesStack.pop(0)
return retval
def visit_Tuple(self, node):
return tuple(map(lambda x: self.visit(x), node.elts))
def visit_List(self, node):
if isinstance(node.elts, LoopedList):
return LoopedList([self.visit(x) for x in self.visit(node.elts.l)],
[self.visit(x) for x in self.visit(node.elts.h)])
else:
return map(lambda x: self.visit(x), node.elts)
def visit_Subscript(self, node):
if node.value is None:
(lower, upper, step) = self.visit(node.slice)
if isinstance(lower, str):
lower = ord(lower)
upper = ord(upper)
return map(lambda x: chr(x), range(lower, upper + 1, step))
else:
return range(lower, upper + 1, step)
if isinstance(node.slice, ast.Index):
return self.visit(node.value)[self.visit(node.slice)]
if isinstance(node.slice, ast.Slice):
(lower, upper, step) = self.visit(node.slice)
return self.visit(node.value)[lower:upper:step]
def visit_Index(self, node):
return self.visit(node.value)
def visit_If(self, node):
if self.visit(node.test):
return self.visit(node.body)
else:
return self.visit(node.orelse)
def visit_Slice(self, node):
lower = self.visit(node.lower)
upper = self.visit(node.upper)
step = self.visit(node.step)
return [lower, upper, step]
def visit_Attribute(self, node):
if node.attr == '!length':
return len(self.visit(node.value))
def test_str_subscript(self):
value = ast.Name("foo", ast.Load)
slice = ast.Index(ast.Str("bar"))
subscript = ast.Subscript(value, slice, ast.Load)
assert name_to_str(subscript) == "foo['bar']"
MethodIntr(),
"keys":
MethodIntr(),
"pop":
MethodIntr(),
"popitem":
MethodIntr(),
"setdefault":
MethodIntr(lambda self, node: len(node.args) == 3 and self.combine(
node.args[0],
node.args[1],
unary_op=lambda x: cxxtypes.DictType(x, self.result[node.args[2]]),
register=True,
aliasing_type=True),
return_alias=lambda node: {
ast.Subscript(node.args[0], ast.Index(node.args[1]),
ast.Load())
}),
"update":
MethodIntr(update_effects),
"values":
MethodIntr(),
"viewitems":
MethodIntr(),
"viewkeys":
MethodIntr(),
"viewvalues":
MethodIntr(),
},
"file": {
# Member variables
def handleNode(self, node, parent, scope):
# Scope variables
if isinstance(node, ast.Name) and not isinstance(node.ctx, ast.Param):
return ast.Subscript(value=ast.Name(id="scope%s" % scope,
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=node.id)),
ctx=node.ctx)
# Global
if isinstance(node, ast.Global):
res = []
for name in node.names:
res.append(
ast.Expr(value=ast.Call(func=ast.Attribute(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
attr="inheritVariable",
ctx=ast.Load()),
args=[
ast.Name(id="scope0",
ctx=ast.Load()),
ast.Str(s=name)
],
keywords=[],
starargs=None,
kwargs=None)))
return res
# Import
if isinstance(node, ast.Import) or isinstance(node, ast.ImportFrom):
none = ast.Name(id="None", ctx=ast.Load())
names = ast.List(elts=[
ast.Tuple(elts=[
ast.Str(s=name.name),
ast.Str(s=name.asname) if name.asname else none
],
ctx=ast.Load()) for name in node.names
],
ctx=ast.Load())
if isinstance(node, ast.ImportFrom):
from_ = ast.Str(node.module) if node.module else none
level = ast.Num(node.level or 0)
else:
from_ = none
level = none
return ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(
id="scope%s" % scope, ctx=ast.Load()),
attr="import_",
ctx=ast.Load()),
args=[names, from_, level],
keywords=[],
starargs=None,
kwargs=None))
if isinstance(node, ast.FunctionDef):
scope += 1
if isinstance(node, ast.Lambda):
# Handle arguments (default values) in parent scope
args = self.handleNode(node.args, node, scope)
if args is not None:
node.args = args
# Now increment scope and handle body
scope += 1
body = self.handleNode(node.body, node, scope)
if body is not None:
node.body = body
else:
for fieldname, value in ast.iter_fields(node):
if isinstance(value, ast.AST):
res = self.handleNode(value, node, scope)
if res is not None:
setattr(node, fieldname, res)
elif isinstance(value, list):
result = []
for child in value:
val = self.handleNode(child, node, scope)
if val is None:
result.append(child)
elif isinstance(val, list):
result += val
else:
result.append(val)
setattr(node, fieldname, result)
# Add scope to functions
if isinstance(node, ast.FunctionDef):
node.body.insert(
0,
ast.Assign(
targets=[ast.Name(id="scope%s" % scope, ctx=ast.Store())],
value=ast.Call(func=ast.Attribute(value=ast.Name(
id="scope%s" % (scope - 1), ctx=ast.Load()),
attr="inherit",
ctx=ast.Load()),
args=[],
keywords=[],
starargs=None,
kwargs=None)))
# Arguments
for arg in node.args.args:
node.body.insert(
1,
ast.Assign(targets=[
ast.Subscript(value=ast.Name(id="scope%s" % scope,
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=arg.id)),
ctx=ast.Store())
],
value=ast.Name(id=arg.id, ctx=ast.Load())))
# Vararg
if node.args.vararg is not None:
node.body.insert(
1,
ast.Assign(targets=[
ast.Subscript(
value=ast.Name(id="scope%s" % scope,
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=node.args.vararg)),
ctx=ast.Store())
],
value=ast.Name(id=node.args.vararg,
ctx=ast.Load())))
# Kwarg
if node.args.kwarg is not None:
node.body.insert(
1,
ast.Assign(targets=[
ast.Subscript(
value=ast.Name(id="scope%s" % scope,
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=node.args.kwarg)),
ctx=ast.Store())
],
value=ast.Name(id=node.args.kwarg,
ctx=ast.Load())))
# Save functions (not methods) to scope
if isinstance(node, ast.FunctionDef) and not isinstance(
parent, ast.ClassDef):
oldname = node.name
node.name = "_user_%s_%s_" % (oldname, scope)
return [
node,
ast.Assign(targets=[
ast.Subscript(value=ast.Name(id="scope%s" % (scope - 1),
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=oldname)),
ctx=ast.Store())
],
value=ast.Name(id=node.name, ctx=ast.Load()))
]
# Save classes to scope
if isinstance(node,
ast.ClassDef) and not isinstance(parent, ast.ClassDef):
oldname = node.name
node.name = "_user_%s_%s_" % (oldname, scope)
return [
node,
ast.Assign(targets=[
ast.Subscript(value=ast.Name(id="scope%s" % scope,
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=oldname)),
ctx=ast.Store())
],
value=ast.Name(id=node.name, ctx=ast.Load()))
]
# Print
if isinstance(node, ast.Print):
true = ast.Name(id="True", ctx=ast.Load())
false = ast.Name(id="False", ctx=ast.Load())
none = ast.Name(id="None", ctx=ast.Load())
return ast.Expr(value=ast.Call(
func=ast.Subscript(value=ast.Attribute(value=ast.Name(
id="scope0", ctx=ast.Load()),
attr="inherits",
ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s="print")),
ctx=ast.Load()),
args=node.values,
keywords=[
ast.keyword(arg="nl", value=true if node.nl else false),
ast.keyword(arg="dest", value=node.dest or none)
],
starargs=None,
kwargs=None))
# Add scope to lambdas
if isinstance(node, ast.Lambda):
# lambda a: a
# ->
# lambda a: (lambda scope1: scope1["a"])(scope0.extend({"a": a}))
# We save everything to dict, don't assign automatically
dct = ast.Dict(keys=[], values=[])
# Arguments
for arg in node.args.args:
dct.keys.append(ast.Str(s=arg.id))
dct.values.append(ast.Name(id=arg.id, ctx=ast.Load()))
# Vararg
if node.args.vararg is not None:
dct.keys.append(ast.Str(s=node.args.vararg))
dct.values.append(ast.Name(id=node.args.vararg,
ctx=ast.Load()))
# Kwarg
if node.args.kwarg is not None:
dct.keys.append(ast.Str(s=node.args.kwarg))
dct.values.append(ast.Name(id=node.args.kwarg, ctx=ast.Load()))
node.body = ast.Call(func=ast.Lambda(args=ast.arguments(
args=[ast.Name(id="scope%s" % scope, ctx=ast.Load())],
vararg=None,
kwarg=None,
defaults=[]),
body=node.body),
args=[
ast.Call(func=ast.Attribute(
value=ast.Name(id="scope%s" %
(scope - 1),
ctx=ast.Load()),
attr="extend",
ctx=ast.Load()),
args=[dct],
keywords=[],
starargs=None,
kwargs=None)
],
keywords=[],
starargs=None,
kwargs=None)
# Now do something to prevent object.__subclasses__() hacks and others
if (isinstance(node, ast.Attribute) and
((node.attr.startswith("__") and node.attr.endswith("__")) or
(node.attr.startswith("func_")))):
return ast.Subscript(value=ast.Call(func=ast.Attribute(
value=ast.Name(id="scope0", ctx=ast.Load()),
attr="safeAttr",
ctx=ast.Load()),
args=[node.value],
keywords=[],
starargs=None,
kwargs=None),
slice=ast.Index(value=ast.Str(node.attr)),
ctx=node.ctx)
def _annotation_list(self, type_):
return ast.Subscript(
value=ast.Name(id="List"), slice=ast.Index(value=ast.Name(id=type_[0]))
)
def IndexedIdentifier(self, sub):
return ast.Subscript(
value=self(Identifier(sub.id)),
slice=ast.Index(value=self(NumberConstant(sub.index))),
ctx=ast.Load(),
**_linearg)
def p_load_subscript(p):
'expression : ID SUBSCRIPT ID'
p[0] = ast.Subscript(value=ast.Name(id=p[1], ctx=ast.Load(), lineno=p.lineno(1), col_offset=p.lexpos(1)),
slice=ast.Index(value=ast.Str(s=p[3], lineno=p.lineno(3), col_offset=p.lexpos(3))),
ctx=ast.Load(),
lineno=p.lineno(1), col_offset=p.lexpos(1))
def res_python_setup(
res: Property) -> Tuple[Callable[[EntityFixup], object], str]:
variables = {}
variable_order = []
code = None
result_var = None
for child in res:
if child.name.startswith('$'):
if child.value.casefold() not in FUNC_GLOBALS:
raise Exception('Invalid variable type! ({})'.format(
child.value))
variables[child.name[1:]] = child.value.casefold()
variable_order.append(child.name[1:])
elif child.name == 'op':
code = child.value
elif child.name == 'resultvar':
result_var = child.value
else:
raise Exception('Invalid key "{}"'.format(child.real_name))
if not code:
raise Exception('No operation specified!')
if not result_var:
raise Exception('No destination specified!')
for name in variables:
if name.startswith('_'):
raise Exception(
'"{}" is not permitted as a variable name!'.format(name))
# Allow $ in the variable names..
code = code.replace('$', '')
# Now process the code to convert it into a function taking variables
# and returning them.
# We also need to whitelist operations for security.
expression = ast.parse(
code,
'<bee2_op>',
mode='eval',
).body
Checker(variable_order).visit(expression)
# For each variable, do
# var = func(_fixup['var'])
statements: List[ast.AST] = [
ast.Assign(targets=[ast.Name(id=var_name, ctx=ast.Store())],
value=ast.Call(
func=ast.Name(id=variables[var_name], ctx=ast.Load()),
args=[
ast.Subscript(
value=ast.Name(id='_fixup', ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=var_name)),
ctx=ast.Load(),
),
],
keywords=[],
starargs=None,
kwargs=None,
)) for line_num, var_name in enumerate(
variable_order,
start=1,
)
]
# The last statement returns the target expression.
statements.append(
ast.Return(expression, lineno=len(variable_order) + 1, col_offset=0))
args = ast.arguments(
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
)
# Py 3.8+, make it pos-only.
if 'posonlyargs' in args._fields:
args.posonlyargs = [ast.arg('_fixup', None)]
args.args = []
else: # Just make it a regular arg.
args.args = [ast.arg('_fixup', None)]
func = ast.Module(
[
ast.FunctionDef(
name='_bee2_generated_func',
args=args,
body=statements,
decorator_list=[],
),
],
lineno=1,
col_offset=0,
)
# Python 3.8 also
if 'type_ignores' in func._fields:
func.type_ignores = []
# Fill in lineno and col_offset
ast.fix_missing_locations(func)
ns: Dict[str, Any] = {}
eval(compile(func, '<bee2_op>', mode='exec'), FUNC_GLOBALS, ns)
compiled_func = ns['_bee2_generated_func']
compiled_func.__name__ = '<bee2_func>'
return compiled_func, result_var
def test_ExtSlice(self):
slice1 = ast.Index(ast.Num(42))
slice2 = ast.Slice(None, None, ast.Num(6))
self.verify(ast.ExtSlice([slice1, slice2]), '42,::6')
def parse_nodes(nodes, ctx_klass=ast.Load):
out = []
while len(nodes) > 0:
node = nodes.pop(0)
if node["type"] == "name" and node["name"] == "_G":
out.append(
ast.Call(
func=ast.Name(id='globals', ctx=ast.Load()),
args=[],
keywords=[],
)
)
continue
if node["type"] == "tuple":
expressions = parse_nodes(node["value"], ctx_klass=ctx_klass)
out.append(
ast.Tuple(
elts=expressions,
ctx=ctx_klass(),
)
)
continue
if node["type"] == "table":
argument_nodes = []
keyword_nodes = []
for x in node["value"]:
if not (x["type"] == "call" and x["name"] == "="):
argument_nodes.append(x)
continue
keyword_nodes.append(x)
key_nodes = [x["args"][0] for x in keyword_nodes]
# Convert name references to strings
key_nodes = [
{"type": "string", "value": x["name"]}
if x["type"] == "name" else x
for x in key_nodes
]
value_nodes = [x["args"][1] for x in keyword_nodes]
value_nodes = [x[0] for x in value_nodes]
value_nodes = parse_nodes(value_nodes)
keywords = []
for x in (zip(key_nodes, value_nodes)):
name_node, value_node = x
name = name_node["value"]
# Apply __ to make sure its casted in Table
if name_node["type"] == "number":
name = "__{0}".format(name)
keywords.append(
ast.keyword(arg=name, value=value_node)
)
out.append(
ast.Call(
func=ast.Name(id='Table', ctx=ast.Load()),
args=parse_nodes(argument_nodes),
keywords=keywords,
)
)
continue
if node["type"] == "string":
out.append(ast.Str(s=node["value"]))
continue
if node["type"] == "boolean":
value = node["value"]
value = True if value == "true" else value
value = False if value == "false" else value
out.append(ast.NameConstant(value=value))
continue
if node["type"] == "number":
value = node["value"]
value = float(value) if "." in value else int(value)
out.append(ast.Num(n=value))
continue
if node["type"] == "nil":
out.append(ast.NameConstant(value=None))
continue
if node["type"] == "return":
out.append(
ast.Return(value=parse_nodes(node["value"])[0])
)
continue
if node["type"] == "assign":
out.append(
ast.Assign(
targets=[
ast.Name(id=node["name"], ctx=ast.Store())
],
value=parse_nodes(node["value"])[0],
)
)
continue
if node["type"] == "name":
out.append(
ast.Name(id=node["name"], ctx=ctx_klass()),
)
continue
if node["type"] == "expr":
out.append(
ast.Expr(
value=parse_nodes(node["value"])[0]
)
)
continue
if node["type"] == "function":
body_nodes = parse_nodes(node["body"])
out.append(
ast.FunctionDef(
name=node["name"],
args=ast.arguments(
args=[
ast.arg(
arg=x["name"],
annotation=None,
) for x in node["args"]
],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]
),
body=body_nodes,
decorator_list=[],
)
)
continue
if node["type"] == "if":
test_nodes = parse_nodes(node["test"])
body_nodes = parse_nodes(node["body"])
else_nodes = parse_nodes(node["else"])
out.append(
ast.If(
test=test_nodes[0],
body=body_nodes,
orelse=else_nodes,
)
)
continue
if node["type"] == "for":
target_expr = parse_nodes(node["target"], ctx_klass=ast.Store)
body_expr = parse_nodes(node["body"])
iteration_nodes = node["iteration"]
# Apply range constructor
if iteration_nodes[0]["type"] == "tuple":
iteration_expr = [
ast.Call(
func=ast.Name(id='get_for_range', ctx=ast.Load()),
args=parse_nodes(iteration_nodes[0]["value"]),
keywords=[],
)
]
else:
iteration_expr = parse_nodes(iteration_nodes)
out.append(
ast.For(
target=target_expr[0],
iter=iteration_expr[0],
body=body_expr,
orelse=[]
)
)
continue
if node["type"] == "while":
test_nodes = parse_nodes(node["test"])
body_nodes = parse_nodes(node["body"])
out.append(
ast.While(
test=test_nodes[0],
body=body_nodes,
orelse=[],
)
)
if node["type"] == "else":
body_nodes = parse_nodes(node["body"])
out = out + body_nodes
continue
if node["type"] == "call":
if node["name"] == "#":
out.append(
ast.Call(
func=ast.Name(id='len', ctx=ast.Load()),
args=parse_nodes(node["args"]),
keywords=[],
)
)
continue
if node["name"] == "[":
value_node = node["args"][0]
value_expression = parse_nodes([value_node])[0]
out.append(
ast.Subscript(
value=value_expression,
slice=ast.Index(
value=parse_nodes(node["args"][1])[0]
),
ctx=ast.Load(),
)
)
continue
if node["name"] == "=":
name_arg = node["args"][0]
value_arg = node["args"][1]
target_expr = parse_nodes([name_arg], ctx_klass=ast.Store)
value_expr = parse_nodes(value_arg)
out.append(
ast.Assign(
targets=target_expr,
value=value_expr[0],
)
)
continue
if node["name"] in ["-", "%", "+", "..", "*", "/"]:
ops = node["name"]
arg_left = parse_nodes([node["args"][0]])
arg_right = parse_nodes(node["args"][1])
ops_ref = {
"-": ast.Sub,
"%": ast.Mod,
"+": ast.Add,
"..": ast.Add,
"*": ast.Mult,
"/": ast.Div,
}
out.append(
ast.BinOp(
left=arg_left[0],
op=ops_ref[ops](),
right=arg_right[0],
)
)
continue
if node["name"] in ["and", "or"]:
ops = node["name"]
arg_left = parse_nodes([node["args"][0]])
arg_right = parse_nodes(node["args"][1])
ops_ref = {
"and": ast.And,
"or": ast.Or,
}
out.append(
ast.BoolOp(
op=ops_ref[ops](),
values=[
arg_left[0],
arg_right[0],
]
)
)
continue
if node["name"] in [">", "<", "~=", "==", "<=", ">="]:
ops = node["name"]
arg_left = parse_nodes([node["args"][0]])
arg_right = parse_nodes(node["args"][1])
ops_ref = {
">": ast.Gt,
">=": ast.GtE,
"<": ast.Lt,
"<=": ast.LtE,
"~=": ast.NotEq,
"==": ast.Eq,
}
out.append(
ast.Compare(
left=arg_left[0],
ops=[ops_ref[ops]()],
comparators=arg_right,
)
)
continue
if node["name"] == "not":
out.append(
ast.UnaryOp(
op=ast.Not(),
operand=parse_nodes(node["args"])[0]
)
)
continue
out.append(
ast.Call(
func=ast.Name(id=node["name"], ctx=ast.Load()),
args=parse_nodes(node["args"], ctx_klass=ast.Load),
keywords=[]
)
)
continue
return out
def test_num_subscript(self):
value = ast.Name("foo", ast.Load)
slice = ast.Index(ast.Num(123))
subscript = ast.Subscript(value, slice, ast.Load)
assert name_to_str(subscript) == "foo[123]"
def from_phpast(node):
if node is None:
return py.Pass(**pos(node))
if isinstance(node, str):
return py.Str(node, **pos(node))
if isinstance(node, (int, float)):
return py.Num(node, **pos(node))
if isinstance(node, php.Array):
if node.nodes:
if node.nodes[0].key is not None:
keys = []
values = []
for elem in node.nodes:
keys.append(from_phpast(elem.key))
values.append(from_phpast(elem.value))
return py.Dict(keys, values, **pos(node))
else:
return py.List([from_phpast(x.value) for x in node.nodes],
py.Load(**pos(node)), **pos(node))
else:
return py.List([], py.Load(**pos(node)), **pos(node))
if isinstance(node, php.InlineHTML):
args = [py.Str(node.data, **pos(node))]
return py.Call(
py.Name('inline_html', py.Load(**pos(node)), **pos(node)), args,
[], None, None, **pos(node))
if isinstance(node, php.Echo):
return py.Call(py.Name('echo', py.Load(**pos(node)), **pos(node)),
list(map(from_phpast, node.nodes)), [], None, None,
**pos(node))
if isinstance(node, php.Print):
return py.Print(None, [from_phpast(node.node)], True, **pos(node))
if isinstance(node, php.Exit):
args = []
if node.expr is not None:
args.append(from_phpast(node.expr))
return py.Raise(
py.Call(py.Name('Exit', py.Load(**pos(node)), **pos(node)), args,
[], None, None, **pos(node)), None, None, **pos(node))
if isinstance(node, php.Return):
if node.node is None:
return py.Return(None, **pos(node))
else:
return py.Return(from_phpast(node.node), **pos(node))
if isinstance(node, php.Break):
assert node.node is None, 'level on break not supported'
return py.Break(**pos(node))
if isinstance(node, php.Continue):
assert node.node is None, 'level on continue not supported'
return py.Continue(**pos(node))
if isinstance(node, php.Silence):
return from_phpast(node.expr)
if isinstance(node, php.Block):
return from_phpast(php.If(1, node, [], None, lineno=node.lineno))
if isinstance(node, php.Unset):
return py.Delete(list(map(from_phpast, node.nodes)), **pos(node))
if isinstance(node, php.IsSet) and len(node.nodes) == 1:
if isinstance(node.nodes[0], php.ArrayOffset):
return py.Compare(from_phpast(node.nodes[0].expr),
[py.In(**pos(node))],
[from_phpast(node.nodes[0].node)], **pos(node))
if isinstance(node.nodes[0], php.ObjectProperty):
return py.Call(
py.Name('hasattr', py.Load(**pos(node)), **pos(node)), [
from_phpast(node.nodes[0].node),
from_phpast(node.nodes[0].name)
], [], None, None, **pos(node))
if isinstance(node.nodes[0], php.Variable):
return py.Compare(py.Str(
node.nodes[0].name[1:], **pos(node)), [py.In(**pos(node))], [
py.Call(py.Name('vars', py.Load(**pos(node)), **pos(node)),
[], [], None, None, **pos(node))
], **pos(node))
return py.Compare(from_phpast(node.nodes[0]), [py.IsNot(**pos(node))],
[py.Name('None', py.Load(**pos(node)), **pos(node))],
**pos(node))
if isinstance(node, php.Empty):
return from_phpast(
php.UnaryOp('!',
php.BinaryOp('&&',
php.IsSet([node.expr],
lineno=node.lineno),
node.expr,
lineno=node.lineno),
lineno=node.lineno))
if isinstance(node, php.Assignment):
if (isinstance(node.node, php.ArrayOffset) and node.node.expr is None):
return py.Call(
py.Attribute(from_phpast(node.node.node), 'append',
py.Load(**pos(node)), **pos(node)),
[from_phpast(node.expr)], [], None, None, **pos(node))
if (isinstance(node.node, php.ObjectProperty)
and isinstance(node.node.name, php.BinaryOp)):
return to_stmt(
py.Call(py.Name('setattr', py.Load(**pos(node)), **pos(node)),
[
from_phpast(node.node.node),
from_phpast(node.node.name),
from_phpast(node.expr)
], [], None, None, **pos(node)))
return py.Assign([store(from_phpast(node.node))],
from_phpast(node.expr), **pos(node))
if isinstance(node, php.ListAssignment):
return py.Assign([
py.Tuple(list(map(store, list(map(from_phpast, node.nodes)))),
py.Store(**pos(node)), **pos(node))
], from_phpast(node.expr), **pos(node))
if isinstance(node, php.AssignOp):
return from_phpast(
php.Assignment(node.left,
php.BinaryOp(node.op[:-1],
node.left,
node.right,
lineno=node.lineno),
False,
lineno=node.lineno))
if isinstance(node, (php.PreIncDecOp, php.PostIncDecOp)):
return from_phpast(
php.Assignment(node.expr,
php.BinaryOp(node.op[0],
node.expr,
1,
lineno=node.lineno),
False,
lineno=node.lineno))
if isinstance(node, php.ArrayOffset):
return py.Subscript(from_phpast(node.node),
py.Index(from_phpast(node.expr), **pos(node)),
py.Load(**pos(node)), **pos(node))
if isinstance(node, php.ObjectProperty):
if isinstance(node.name, (php.Variable, php.BinaryOp)):
return py.Call(
py.Name('getattr', py.Load(**pos(node)), **pos(node)),
[from_phpast(node.node),
from_phpast(node.name)], [], None, None, **pos(node))
return py.Attribute(from_phpast(node.node), node.name,
py.Load(**pos(node)), **pos(node))
if isinstance(node, php.Constant):
name = node.name
if name.lower() == 'true': name = 'True'
if name.lower() == 'false': name = 'False'
if name.lower() == 'null': name = 'None'
return py.Name(name, py.Load(**pos(node)), **pos(node))
if isinstance(node, php.Variable):
name = node.name[1:]
if name == 'this': name = 'self'
return py.Name(name, py.Load(**pos(node)), **pos(node))
if isinstance(node, php.Global):
return py.Global([var.name[1:] for var in node.nodes], **pos(node))
if isinstance(node, php.Include):
once = py.Name('True' if node.once else 'False', py.Load(**pos(node)),
**pos(node))
return py.Call(py.Name('include', py.Load(**pos(node)), **pos(node)),
[from_phpast(node.expr), once], [], None, None,
**pos(node))
if isinstance(node, php.Require):
once = py.Name('True' if node.once else 'False', py.Load(**pos(node)),
**pos(node))
return py.Call(py.Name('require', py.Load(**pos(node)), **pos(node)),
[from_phpast(node.expr), once], [], None, None,
**pos(node))
if isinstance(node, php.UnaryOp):
op = unary_ops.get(node.op)
assert op is not None, "unknown unary operator: '%s'" % node.op
op = op(**pos(node))
return py.UnaryOp(op, from_phpast(node.expr), **pos(node))
if isinstance(node, php.BinaryOp):
if node.op == '.':
pattern, pieces = build_format(node.left, node.right)
if pieces:
return py.BinOp(
py.Str(pattern, **pos(node)), py.Mod(**pos(node)),
py.Tuple(list(map(from_phpast, pieces)),
py.Load(**pos(node)), **pos(node)), **pos(node))
else:
return py.Str(pattern % (), **pos(node))
if node.op in bool_ops:
op = bool_ops[node.op](**pos(node))
return py.BoolOp(op,
[from_phpast(node.left),
from_phpast(node.right)], **pos(node))
if node.op in cmp_ops:
op = cmp_ops[node.op](**pos(node))
return py.Compare(from_phpast(node.left), [op],
[from_phpast(node.right)], **pos(node))
op = binary_ops.get(node.op)
if node.op == 'instanceof':
return py.Call(
func=py.Name(id='isinstance', ctx=py.Load(**pos(node))),
args=[from_phpast(node.left),
from_phpast(node.right)],
keywords=[],
starargs=None,
kwargs=None)
assert op is not None, "unknown binary operator: '%s'" % node.op
op = op(**pos(node))
return py.BinOp(from_phpast(node.left), op, from_phpast(node.right),
**pos(node))
if isinstance(node, php.TernaryOp):
return py.IfExp(from_phpast(node.expr), from_phpast(node.iftrue),
from_phpast(node.iffalse), **pos(node))
if isinstance(node, php.Cast):
return py.Call(
py.Name(casts.get(node.type, node.type), py.Load(**pos(node)),
**pos(node)), [from_phpast(node.expr)], [], None, None,
**pos(node))
if isinstance(node, php.If):
orelse = []
if node.else_:
for else_ in map(from_phpast, deblock(node.else_.node)):
orelse.append(to_stmt(else_))
for elseif in reversed(node.elseifs):
orelse = [
py.If(
from_phpast(elseif.expr),
list(
map(to_stmt,
list(map(from_phpast, deblock(elseif.node))))),
orelse, **pos(node))
]
return py.If(
from_phpast(node.expr),
list(map(to_stmt, list(map(from_phpast, deblock(node.node))))),
orelse, **pos(node))
if isinstance(node, php.For):
assert node.test is None or len(node.test) == 1, \
'only a single test is supported in for-loops'
return from_phpast(
php.Block((node.start or []) + [
php.While(node.test[0] if node.test else 1,
php.Block(deblock(node.node) + (node.count or []),
lineno=node.lineno),
lineno=node.lineno)
],
lineno=node.lineno))
if isinstance(node, php.Foreach):
if node.keyvar is None:
target = py.Name(node.valvar.name[1:], py.Store(**pos(node)),
**pos(node))
else:
target = py.Tuple([
py.Name(node.keyvar.name[1:], py.Store(**pos(node))),
py.Name(node.valvar.name[1:], py.Store(**pos(node)))
], py.Store(**pos(node)), **pos(node))
return py.For(
target, from_phpast(node.expr),
list(map(to_stmt, list(map(from_phpast, deblock(node.node))))), [],
**pos(node))
if isinstance(node, php.While):
return py.While(
from_phpast(node.expr),
list(map(to_stmt, list(map(from_phpast, deblock(node.node))))), [],
**pos(node))
if isinstance(node, php.DoWhile):
condition = php.If(php.UnaryOp('!', node.expr, lineno=node.lineno),
php.Break(None, lineno=node.lineno), [],
None,
lineno=node.lineno)
return from_phpast(
php.While(1,
php.Block(deblock(node.node) + [condition],
lineno=node.lineno),
lineno=node.lineno))
if isinstance(node, php.Try):
return py.TryExcept(
list(map(to_stmt, list(map(from_phpast, node.nodes)))), [
py.ExceptHandler(
py.Name(catch.class_, py.Load(**pos(node)), **pos(node)),
store(from_phpast(catch.var)),
list(map(to_stmt, list(map(from_phpast, catch.nodes)))),
**pos(node)) for catch in node.catches
], [], **pos(node))
if isinstance(node, php.Throw):
return py.Raise(from_phpast(node.node), None, None, **pos(node))
if isinstance(node, php.Function):
args = []
defaults = []
for param in node.params:
args.append(
py.Name(param.name[1:], py.Param(**pos(node)), **pos(node)))
if param.default is not None:
defaults.append(from_phpast(param.default))
body = list(map(to_stmt, list(map(from_phpast, node.nodes))))
if not body: body = [py.Pass(**pos(node))]
return py.FunctionDef(node.name,
py.arguments(args, None, None, defaults), body,
[], **pos(node))
if isinstance(node, php.Method):
args = []
defaults = []
decorator_list = []
if 'static' in node.modifiers:
decorator_list.append(
py.Name('classmethod', py.Load(**pos(node)), **pos(node)))
args.append(py.Name('cls', py.Param(**pos(node)), **pos(node)))
else:
args.append(py.Name('self', py.Param(**pos(node)), **pos(node)))
for param in node.params:
args.append(
py.Name(param.name[1:], py.Param(**pos(node)), **pos(node)))
if param.default is not None:
defaults.append(from_phpast(param.default))
body = list(map(to_stmt, list(map(from_phpast, node.nodes))))
if not body: body = [py.Pass(**pos(node))]
return py.FunctionDef(node.name,
py.arguments(args, None, None, defaults), body,
decorator_list, **pos(node))
if isinstance(node, php.Class):
name = node.name
bases = []
extends = node.extends or 'object'
bases.append(py.Name(extends, py.Load(**pos(node)), **pos(node)))
body = list(map(to_stmt, list(map(from_phpast, node.nodes))))
for stmt in body:
if (isinstance(stmt, py.FunctionDef)
and stmt.name in (name, '__construct')):
stmt.name = '__init__'
if not body: body = [py.Pass(**pos(node))]
return py.ClassDef(name, bases, body, [], **pos(node))
if isinstance(node, (php.ClassConstants, php.ClassVariables)):
assert len(node.nodes) == 1, \
'only one class-level assignment supported per line'
if isinstance(node.nodes[0], php.ClassConstant):
name = php.Constant(node.nodes[0].name, lineno=node.lineno)
else:
name = php.Variable(node.nodes[0].name, lineno=node.lineno)
initial = node.nodes[0].initial
if initial is None:
initial = php.Constant('None', lineno=node.lineno)
return py.Assign([store(from_phpast(name))], from_phpast(initial),
**pos(node))
if isinstance(node, (php.FunctionCall, php.New)):
if isinstance(node.name, str):
name = py.Name(node.name, py.Load(**pos(node)), **pos(node))
else:
name = py.Subscript(
py.Call(py.Name('vars', py.Load(**pos(node)), **pos(node)), [],
[], None, None, **pos(node)),
py.Index(from_phpast(node.name), **pos(node)),
py.Load(**pos(node)), **pos(node))
args, kwargs = build_args(node.params)
return py.Call(name, args, kwargs, None, None, **pos(node))
if isinstance(node, php.MethodCall):
args, kwargs = build_args(node.params)
return py.Call(
py.Attribute(from_phpast(node.node), node.name,
py.Load(**pos(node)), **pos(node)), args, kwargs,
None, None, **pos(node))
if isinstance(node, php.StaticMethodCall):
class_ = node.class_
if class_ == 'self': class_ = 'cls'
args, kwargs = build_args(node.params)
return py.Call(
py.Attribute(py.Name(class_, py.Load(**pos(node)),
**pos(node)), node.name, py.Load(**pos(node)),
**pos(node)), args, kwargs, None, None, **pos(node))
if isinstance(node, php.StaticProperty):
class_ = node.node
name = node.name
if isinstance(name, php.Variable):
name = name.name[1:]
return py.Attribute(py.Name(class_, py.Load(**pos(node)), **pos(node)),
name, py.Load(**pos(node)), **pos(node))
return py.Call(py.Name('XXX', py.Load(**pos(node)),
**pos(node)), [py.Str(str(node), **pos(node))], [],
None, None, **pos(node))
def test_unsupported_subscript(self):
# Just don't crash...
value = ast.Name("foo", ast.Load)
slice = ast.Index(ast.Name("bar", ast.Load))
subscript = ast.Subscript(value, slice, ast.Load)
assert isinstance(name_to_str(subscript), str)
def _bind_arguments(self, f_ast, call_expr, new_stmts):
args_def = f_ast.args
# Scope a variable name as unique to the function, and update any references
# to it in the function
def unique_and_rename(name):
unique_name = f'{name}{SEP}{f_ast.name}'
renamer = Rename(name, unique_name)
for stmt in f_ast.body:
renamer.visit(stmt)
return unique_name
args = call_expr.args[:]
# Rename all variables declared in the function that aren't arguments
assgn_finder = FindAssignments()
assgn_finder.visit(f_ast)
arg_names = set([arg.arg for arg in args_def.args])
for name in assgn_finder.names:
if name not in arg_names:
unique_and_rename(name)
# If function is called with f(*args)
if len(call_expr.args) > 0 and \
isinstance(call_expr.args[-1], ast.Starred):
star_arg = call_expr.args.pop().value
# Get the length of the star_arg runtime list
star_arg_obj = eval(a2s(star_arg), self.globls, self.globls)
# Generate an indexing expression for each element of the list
call_star_args = [
ast.Subscript(value=star_arg,
slice=ast.Index(value=ast.Num(i)))
for i in range(len(star_arg_obj))
]
else:
star_arg = None
# If function is called with f(**kwargs)
star_kwarg = [arg for arg in call_expr.keywords if arg.arg is None]
star_kwarg = star_kwarg[0].value if len(star_kwarg) > 0 else None
if star_kwarg is not None:
star_kwarg_dict = eval(a2s(star_kwarg), self.globls, self.globls)
call_star_kwarg = {
key: ast.Subscript(value=star_kwarg,
slice=ast.Index(value=ast.Str(key)))
for key in star_kwarg_dict.keys()
}
# Function's anonymous arguments, e.g. f(1, 2) becomes [1, 2]
call_anon_args = call_expr.args[:]
# Function's keyword arguments, e.g. f(x=1, y=2) becomes {'x': 1, 'y': 2}
call_kwargs = {
arg.arg: arg.value
for arg in call_expr.keywords if arg.arg is not None
}
# Match up defaults with variable names.
#
# Python convention is that if function has N arguments and K < N defaults, then
# the defaults correspond to arguments N - K .. N.
nodefault = len(args_def.args) - len(args_def.defaults)
anon_defaults = {
arg.arg: default
for arg, default in zip(args_def.args[nodefault:],
args_def.defaults)
}
# All keyword-only arguments must have defaults.
#
# kwonlyargs occur if a function definition has args AFTER a *args, e.g.
# the var "y" in `def foo(x, *args, y=1)`
kw_defaults = {
arg.arg: default
for arg, default in zip(args_def.kwonlyargs, args_def.kw_defaults)
}
# For each non-keyword-only argument, match it up with the corresponding
# syntax from the call expression
for arg in args_def.args:
k = arg.arg
# First, match with anonymous arguments
if len(call_anon_args) > 0:
v = call_anon_args.pop(0)
# Then use *args if it exists
elif star_arg is not None and len(call_star_args) > 0:
v = call_star_args.pop(0)
# Then use keyword arguments
elif k in call_kwargs:
v = call_kwargs.pop(k)
# Then use **kwargs if it exists
elif star_kwarg is not None and k in call_star_kwarg:
v = call_star_kwarg.pop(k)
# Otherwise use the default value
else:
v = anon_defaults.pop(k)
# Add a binding from function argument to call argument
uniq_k = unique_and_rename(k)
stmt = ast.Assign(targets=[make_name(uniq_k)], value=v)
new_stmts.append(stmt)
# Perform equivalent procedure as above, but for keyword-only arguments
for arg in args_def.kwonlyargs:
k = arg.arg
if k in call_kwargs:
v = call_kwargs.pop(k)
elif star_kwarg is not None and k in call_star_kwarg:
v = call_star_kwarg.pop(k)
else:
v = kw_defaults.pop(k)
uniq_k = unique_and_rename(k)
stmt = ast.Assign(targets=[make_name(uniq_k)], value=v)
new_stmts.append(stmt)
# If function definition uses *args, then assign it to the remaining anonymous
# arguments from the call_expr
if args_def.vararg is not None:
k = unique_and_rename(args_def.vararg.arg)
v = call_anon_args[:]
if star_arg is not None:
v += call_star_args
new_stmts.append(
ast.Assign(targets=[make_name(k)], value=ast.List(elts=v)))
# Similarly for **kwargs in the function definition
if args_def.kwarg is not None:
k = unique_and_rename(args_def.kwarg.arg)
items = call_kwargs.items()
if star_kwarg is not None:
items = itertools.chain(items, call_star_kwarg.items())
kwkeys, kwvalues = unzip(items)
new_stmts.append(
ast.Assign(targets=[make_name(k)],
value=ast.Dict([ast.Str(s) for s in kwkeys],
kwvalues)))
def mk_subscript_assign(obj_expr, index_expr, value_expr):
subscript_expr = ast.Subscript(value=obj_expr,
slice=ast.Index(value=index_expr),
ctx=ast.Store())
return ast.Assign(targets=[subscript_expr], value=value_expr)
def build(self) -> typing.Optional[ast.ClassDef]:
base_class: typing.Union[ast.Attribute, ast.Name]
# Create the base class
if not self.resource.base or self.resource.base.name == "object":
base_class = ast.Attribute(value=ast.Name(id="marshmallow"),
attr="Schema")
else:
if self.resource.base.name in FIELD_TYPES:
return None
base_class = ast.Name(id=self.resource.base.name + "Schema")
if self.resource.package_name != self.resource.base.package_name:
self.generator.import_resource(
self.resource.package_name,
self.resource.base.package_name,
self.resource.base.name + "Schema",
)
# Define the base class
class_node = ast.ClassDef(
name=self.resource.name + "Schema",
bases=[base_class],
keywords=[],
decorator_list=[],
body=[],
)
doc_string = (
f"Marshmallow schema for :class:`commercetools.types.{self.resource.name}`."
)
class_node.body.append(
ast.Expr(value=ast.Str(s=doc_string, kind=None)))
# Add the field definitions
for prop in self.resource.properties:
node = self._create_schema_property(prop)
if node:
class_node.body.append(node)
# Add the Meta class
class_node.body.append(
ast.ClassDef(
name="Meta",
bases=[],
keywords=[],
body=[
ast.Assign(
targets=[ast.Name(id="unknown")],
value=ast.Name(id="marshmallow.EXCLUDE"),
)
],
decorator_list=[],
))
# Create the post_load() method
post_load_node = self._create_marshmallow_hook("post_load")
if self.contains_regex_field:
post_load_node.body.append(
self._create_regex_call("_regex", "postprocess"))
post_load_node.body.append(
ast.Return(value=ast.Call(
func=ast.Name(id=f"types.{self.resource.name}"),
args=[],
keywords=[ast.keyword(arg=None, value=ast.Name(id="data"))],
)))
class_node.body.append(post_load_node)
# Create the pre_load() method
if self.contains_regex_field:
node = self._create_marshmallow_hook("pre_load")
node.body.append(self._create_regex_call("_regex", "preprocess"))
node.body.append(ast.Return(value=ast.Name(id="data")))
class_node.body.append(node)
node = self._create_marshmallow_hook("pre_dump")
node.body.append(self._create_regex_call("_regex", "preprocess"))
node.body.append(ast.Return(value=ast.Name(id="data")))
class_node.body.append(node)
node = self._create_marshmallow_hook("post_dump")
node.body.append(self._create_regex_call("_regex", "postprocess"))
node.body.append(ast.Return(value=ast.Name(id="data")))
class_node.body.append(node)
d_field = self.resource.get_discriminator_field()
if d_field:
post_load_node.body.insert(
0,
ast.Delete(targets=[
ast.Subscript(
value=ast.Name(id="data"),
slice=ast.Index(value=ast.Str(s=d_field.attribute_name,
kind=None)),
)
]),
)
return class_node
MethodIntr(),
"keys":
MethodIntr(),
"pop":
MethodIntr(),
"popitem":
MethodIntr(),
"setdefault":
MethodIntr(lambda self, node: len(node.args) == 3 and self.combine(
node.args[0],
node.args[1],
unary_op=lambda x: cxxtypes.DictType(x, self.result[node.args[2]]),
register=True,
aliasing_type=True),
return_alias=lambda args: {
ast.Subscript(args[0], ast.Index(args[1]), ast.Load())
}.union({args[2]} if len(args) == 3 else set())),
"update":
MethodIntr(update_effects),
"values":
MethodIntr(),
"viewitems":
MethodIntr(),
"viewkeys":
MethodIntr(),
"viewvalues":
MethodIntr(),
},
"file": {
# Member variables
"closed": AttributeIntr(return_type=NamedType("bool")),
def make_fv(key: str):
return ast.Subscript(value=node.right,
slice=ast.Index(value=ast.Str(s=key)))
def visit_Name(self, node):
if node.id in input_vars:
return ast.Subscript(node, ast.Index(index_var), node.ctx)
return node
def as_ast(dct):
"""See https://docs.python.org/2/library/ast.html"""
if dct['ast_type'] == "Module":
return ast.Module(dct["body"])
elif dct['ast_type'] == "Interactive":
return ast.Interactive(dct["body"])
elif dct['ast_type'] == "Expression":
return ast.Expression(dct["body"])
elif dct['ast_type'] == "Suite":
return ast.Suite(dct["body"])
elif dct['ast_type'] == "FunctionDef":
return ast.FunctionDef(dct["name"], dct["args"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "ClassDef":
return ast.ClassDef(dct["name"], dct["bases"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "Return":
return ast.Return(dct["value"])
elif dct['ast_type'] == "Delete":
return ast.Delete(dct["targets"])
elif dct['ast_type'] == "Assign":
return ast.Assign(dct["targets"], dct["value"])
elif dct['ast_type'] == "AugAssign":
return ast.AugAssign(dct["target"], dct["op"], dct["value"])
elif dct['ast_type'] == "Print":
return ast.Print(dct["dest"], dct["values"], dct["nl"])
elif dct['ast_type'] == "For":
return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "While":
return ast.While(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "If":
return ast.If(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "With":
return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"])
elif dct['ast_type'] == "Raise":
return ast.Raise(dct["type"], dct["inst"], dct["tback"])
elif dct['ast_type'] == "TryExcept":
return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"])
elif dct['ast_type'] == "TryFinally":
return ast.TryFinally(dct["body"], dct["finalbody"])
elif dct['ast_type'] == "Assert":
return ast.Assert(dct["test"], dct["msg"])
elif dct['ast_type'] == "Import":
return ast.Import(dct["names"])
elif dct['ast_type'] == "ImportFrom":
return ast.ImportFrom(dct["module"], dct["names"], dct["level"])
elif dct['ast_type'] == "Exec":
return ast.Exec(dct["body"], dct["globals"], dct["locals"])
elif dct['ast_type'] == "Global":
return ast.Global(dct["names"])
elif dct['ast_type'] == "Expr":
return ast.Expr(dct["value"])
elif dct['ast_type'] == "Pass":
return ast.Pass()
elif dct['ast_type'] == "Break":
return ast.Break()
elif dct['ast_type'] == "Continue":
return ast.Continue()
elif dct['ast_type'] == "BoolOp":
return ast.BoolOp(dct["op"], dct["values"])
elif dct['ast_type'] == "BinOp":
return ast.BinOp(dct["left"], dct["op"], dct["right"])
elif dct['ast_type'] == "UnaryOp":
return ast.UnaryOp(dct["op"], dct["operand"])
elif dct['ast_type'] == "Lambda":
return ast.Lambda(dct["args"], dct["body"])
elif dct['ast_type'] == "IfExp":
return ast.IfExp(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "Dict":
return ast.Dict(dct["keys"], dct["values"])
elif dct['ast_type'] == "Set":
return ast.Set(dct["elts"])
elif dct['ast_type'] == "ListComp":
return ast.ListComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "SetComp":
return ast.SetComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "DictComp":
return ast.DictComp(dct["key"], dct["value"], dct["generators"])
elif dct['ast_type'] == "GeneratorExp":
return ast.GeneratorExp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "Yield":
return ast.Yield(dct["value"])
elif dct['ast_type'] == "Compare":
return ast.Compare(dct["left"], dct["ops"], dct["comparators"])
elif dct['ast_type'] == "Call":
return ast.Call(dct["func"], dct["args"], dct["keywords"],
dct["starargs"], dct["kwargs"])
elif dct['ast_type'] == "Repr":
return ast.Repr(dct["value"])
elif dct['ast_type'] == "Num":
return ast.Num(dct["n"])
elif dct['ast_type'] == "Str":
# Converting to ASCII
return ast.Str(dct["s"].encode('ascii', 'ignore'))
elif dct['ast_type'] == "Attribute":
return ast.Attribute(dct["value"], dct["attr"], dct["ctx"])
elif dct['ast_type'] == "Subscript":
return ast.Subscript(dct["value"], dct["slice"], dct["ctx"])
elif dct['ast_type'] == "Name":
return ast.Name(dct["id"], dct["ctx"])
elif dct['ast_type'] == "List":
return ast.List(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Tuple":
return ast.Tuple(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Load":
return ast.Load()
elif dct['ast_type'] == "Store":
return ast.Store()
elif dct['ast_type'] == "Del":
return ast.Del()
elif dct['ast_type'] == "AugLoad":
return ast.AugLoad()
elif dct['ast_type'] == "AugStore":
return ast.AugStore()
elif dct['ast_type'] == "Param":
return ast.Param()
elif dct['ast_type'] == "Ellipsis":
return ast.Ellipsis()
elif dct['ast_type'] == "Slice":
return ast.Slice(dct["lower"], dct["upper"], dct["step"])
elif dct['ast_type'] == "ExtSlice":
return ast.ExtSlice(dct["dims"])
elif dct['ast_type'] == "Index":
return ast.Index(dct["value"])
elif dct['ast_type'] == "And":
return ast.And()
elif dct['ast_type'] == "Or":
return ast.Or()
elif dct['ast_type'] == "Add":
return ast.Add()
elif dct['ast_type'] == "Sub":
return ast.Sub()
elif dct['ast_type'] == "Mult":
return ast.Mult()
elif dct['ast_type'] == "Div":
return ast.Div()
elif dct['ast_type'] == "Mod":
return ast.Mod()
elif dct['ast_type'] == "Pow":
return ast.Pow()
elif dct['ast_type'] == "LShift":
return ast.LShift()
elif dct['ast_type'] == "RShift":
return ast.RShift()
elif dct['ast_type'] == "BitOr":
return ast.BitOr()
elif dct['ast_type'] == "BitXor":
return ast.BitXor()
elif dct['ast_type'] == "BitAnd":
return ast.BitAnd()
elif dct['ast_type'] == "FloorDiv":
return ast.FloorDiv()
elif dct['ast_type'] == "Invert":
return ast.Invert()
elif dct['ast_type'] == "Not":
return ast.Not()
elif dct['ast_type'] == "UAdd":
return ast.UAdd()
elif dct['ast_type'] == "USub":
return ast.USub()
elif dct['ast_type'] == "Eq":
return ast.Eq()
elif dct['ast_type'] == "NotEq":
return ast.NotEq()
elif dct['ast_type'] == "Lt":
return ast.Lt()
elif dct['ast_type'] == "LtE":
return ast.LtE()
elif dct['ast_type'] == "Gt":
return ast.Gt()
elif dct['ast_type'] == "GtE":
return ast.GtE()
elif dct['ast_type'] == "Is":
return ast.Is()
elif dct['ast_type'] == "IsNot":
return ast.IsNot()
elif dct['ast_type'] == "In":
return ast.In()
elif dct['ast_type'] == "NotIn":
return ast.NotIn()
elif dct['ast_type'] == "comprehension":
return ast.comprehension(dct["target"], dct["iter"], dct["ifs"])
elif dct['ast_type'] == "ExceptHandler":
return ast.ExceptHandler(dct["type"], dct["name"], dct["body"])
elif dct['ast_type'] == "arguments":
return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"],
dct["defaults"])
elif dct['ast_type'] == "keyword":
return ast.keyword(dct["arg"], dct["value"])
elif dct['ast_type'] == "alias":
return ast.alias(dct["name"], dct["asname"])
else:
return dct
def translate_to_tptv1(parsed_model, data_batch, hypers):
parsed_model = u.replace_hypers(parsed_model, hypers)
input_dependents = get_input_dependent_vars(parsed_model)
idx_var_name = "input_idx"
idx_var = ast.Name(idx_var_name, ast.Load())
input_number = len(data_batch['instances'])
range_expr = ast.Num(input_number)
input_vars = set()
output_vars = set()
var_decls = []
input_stmts = []
general_stmts = []
output_stmts = []
for stmt in parsed_model.body:
if isinstance(stmt, ast.Assign) and is_input_declaration(stmt.value):
input_vars.add(get_var_name(stmt.targets[0]))
var_decls.append(stmt)
elif isinstance(stmt, ast.Assign) and is_output_declaration(stmt.value):
output_vars.add(get_var_name(stmt.targets[0]))
var_decls.append(stmt)
elif isinstance(stmt, ast.Assign) and is_var_declaration(stmt.value):
var_decls.append(stmt)
elif ast_uses_varset(stmt, input_dependents):
input_stmts.append(add_input_indices(stmt, input_dependents, idx_var))
elif ast_uses_varset(stmt, output_vars):
output_stmts.append(stmt)
else:
general_stmts.append(stmt)
input_init = []
output_observation = []
for input_idx, instance in enumerate(data_batch['instances']):
for var_name, val in instance.iteritems():
if var_name in input_vars:
input_init.extend(generate_io_stmt(input_idx, var_name, val, "set_to_constant"))
elif var_name in output_vars:
output_observation.extend(generate_io_stmt(input_idx, var_name, val, "observe_value"))
extended_var_decls = []
for var_decl in var_decls:
# If input-dependent, extend dimension by one
if get_var_name(var_decl.targets[0]) in input_dependents:
new_decl = copy.deepcopy(var_decl)
if isinstance(new_decl.value, ast.Subscript):
new_decl.value = extend_subscript_for_input(new_decl.value,
ast.Num(input_number))
else:
new_decl.value = ast.Subscript(new_decl.value,
ast.Index(ast.Num(input_number)),
ast.Load())
extended_var_decls.append(new_decl)
else:
extended_var_decls.append(var_decl)
input_loop = ast.For(ast.Name(idx_var_name, ast.Store()),
ast.Call(ast.Name("range", ast.Load()),
[range_expr],
[], None, None),
input_stmts,
[])
parsed_model.body = general_stmts + extended_var_decls + input_init + [input_loop] + output_stmts + output_observation
ast.fix_missing_locations(parsed_model)
return parsed_model
def resolve_field_type(
field_type: Union[AvroPrimitives, DefinedType, Record, Enum, Array,
Map, Fixed],
required_imports: List[Union[ast.Import, ast.ImportFrom]]) -> ast.AST:
"""Resolve the requisite type annotation for a supplied field type."""
if isinstance(field_type, Primitives):
# http://avro.apache.org/docs/1.10.0/spec.html#schema_primitive
return ast.Name(id=PRIMITIVE_TYPE_MAPPINGS[field_type])
if isinstance(field_type, str):
# DefinedType case
return ast.Str(s=field_type)
if isinstance(field_type, list):
# http://avro.apache.org/docs/1.10.0/spec.html#Unions
if len(field_type) == 2 and Primitives.NULL in field_type:
# special case of 'optional'
_, nested_type = field_type
required_imports.append(from_typing('Optional'))
return ast.Subscript(value=ast.Name(id='Optional'),
slice=ast.Index(value=resolve_field_type(
nested_type,
required_imports=required_imports,
)))
required_imports.append(from_typing('Union'))
return ast.Subscript(value=ast.Name(id='Union'),
slice=ast.Index(value=ast.Tuple(elts=[
resolve_field_type(
nested_field,
required_imports=required_imports,
) for nested_field in field_type
])))
if isinstance(field_type, Map):
# http://avro.apache.org/docs/1.10.0/spec.html#Maps
required_imports.append(from_typing('Dict'))
return ast.Subscript(value=ast.Name(id='Dict'),
slice=ast.Index(value=ast.Tuple(elts=[
ast.Name(id='str'),
resolve_field_type(
field_type.values,
required_imports=required_imports,
),
])))
if isinstance(field_type, Array):
# http://avro.apache.org/docs/1.10.0/spec.html#Arrays
required_imports.append(from_typing('List'))
return ast.Subscript(value=ast.Name(id='List'),
slice=ast.Index(value=resolve_field_type(
field_type.items,
required_imports=required_imports,
)))
if isinstance(field_type, Fixed):
# http://avro.apache.org/docs/1.10.0/spec.html#Fixed
return ast.Name(id='bytes')
if isinstance(field_type, LogicalType):
import_module, type_ = LOGICAL_TYPE_MAPPINGS[field_type.logical_type]
required_imports.append(
ast.Import(names=[ast.alias(name=import_module, asname=None)]))
return ast.Name(id=type_)
if isinstance(field_type, (Record, Enum)):
if field_type.namespace:
return ast.Str(s="{0.namespace}.{0.name}".format(
field_type)) # this is mutated/imported at a later date
# TODO - is this case legal/reachable...?
return ast.Str(s=field_type.name)
raise ValueError(f"Didn't know how to handle field type `{field_type}`")
def subscript(name, value, ctx):
return ast.Subscript(
value=value,
slice=ast.Index(value=ast.Str(s=name)),
ctx=ctx,
)