def visit_While(self, node):
if not coloring.is_green(node):
return node
# testVar = not bool(...test)
testVar = gensym.gensym("test")
testStatement = ast.Assign(
targets=[ast.Name(id=testVar, ctx=ast.Store())],
value=ast.UnaryOp(op=ast.Not(),
operand=ast.Call(func=ast.Name(id='bool',
ctx=ast.Load()),
args=[node.test],
keywords=[])))
ast.copy_location(testStatement, node)
# TODO testStatement needs to be colored appropriately
# based on whether test contains green calls
# "if <testVar>: break"
breakNode = ast.Break()
ast.copy_location(breakNode, node)
breakNode.color = True
loopIf = ast.If(test=ast.Name(id=testVar, ctx=ast.Load()),
body=[breakNode],
orelse=[])
ast.copy_location(loopIf, node)
loopIf.color = True
# build the new body
newbody = []
newbody.append(testStatement)
newbody.append(loopIf)
newbody.extend(node.body)
node.body = self._transform_sequence(newbody)
node.test = ast.copy_location(ast.NameConstant(value=True), node)
return node
def _generate_python_binary_cli_ast_node(self) -> ast.Expr:
"""Generate an AST node for a python_binary Pants target"""
binary_name, source_module_path = self._parse_entry_point()
node = ast.Expr(
value=ast.Call(
func=ast.Name(id="python_binary"),
args=[],
keywords=[
ast.keyword(arg="name", value=ast.Str(binary_name)),
ast.keyword(
arg="dependencies", value=ast.List(elts=[ast.Str(":lib")])
),
ast.keyword(
arg="sources",
value=ast.List(elts=[ast.Str(source_module_path)]),
),
ast.keyword(
arg="zip_safe", value=ast.NameConstant(self.config.zip_safe)
),
self._tags_keyword,
],
)
)
return node
def noneSpecialFunction(cv):
"""If the old type is 'None' (which won't show up in the original), move up in the AST to get the metadata"""
if (not isinstance(cv, AddVector)) and cv.oldSubtree == None:
cvCopy = cv.deepcopy()
if cv.path[0] == ('value', 'Return'):
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
cv.newSubtree = ast.Return(cv.newSubtree)
cv.path = cv.path[1:]
elif cv.path[0] == ('value', 'Name Constant'):
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
cv.newSubtree = ast.NameConstant(cv.newSubtree)
cv.path = cv.path[1:]
elif cv.path[0] in [('lower', 'Slice'), ('upper', 'Slice'), ('step', 'Slice')]:
tmpNew = cv.newSubtree
cvCopy = cv.deepcopy()
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
cv.newSubtree = deepcopy(cv.oldSubtree) # use the same slice
if cv.path[0][0] == 'lower':
cv.newSubtree.lower = tmpNew
elif cv.path[0][0] == 'upper':
cv.newSubtree.upper = tmpNew
else:
cv.newSubtree.step = tmpNew
cv.path = cv.path[1:] # get rid of None and the val
else:
log("Individualize\tmapEdit\tMissing option in None special case 1: " + str(cv.path[0]), "bug")
elif cv.oldSubtree == "None":
cv.path = cv.path[1:] # get rid of None and the id
cvCopy = cv.deepcopy()
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
if cv.path[0] == ('value', 'Return'):
cv.newSubtree = ast.Return(ast.Name(cv.newSubtree, ast.Load()))
else:
log("Individualize\tmapEdit\tMissing option in None special case 2: " + str(cv.path[0]), "bug")
cv.path = cv.path[1:]
return cv
def basicTypeSpecialFunction(cv):
"""If you're in a number or string (which has no metadata), move up to the AST to make the special functions work."""
if isinstance(cv, SwapVector) or isinstance(cv, MoveVector):
return cv
if (cv.path[0] in [('n', 'Number'), ('s', 'String'), ('id', 'Name'), ('arg', 'Argument'),
('value', 'Name Constant'), ('s', 'Bytes'), ('name', 'Alias')]):
cvCopy = cv.deepcopy()
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
if cv.path[0] == ('n', 'Number'):
cv.newSubtree = ast.Num(cv.newSubtree)
elif cv.path[0] == ('s', 'String'):
cv.newSubtree = ast.Str(cv.newSubtree)
elif cv.path[0] == ('id', 'Name'):
cv.newSubtree = ast.Name(cv.newSubtree, cv.oldSubtree.ctx)
elif cv.path[0] == ('arg', 'Argument'):
cv.newSubtree = ast.arg(cv.newSubtree, cv.oldSubtree.annotation)
elif cv.path[0] == ('value', 'Name Constant'):
cv.newSubtree = ast.NameConstant(cv.newSubtree)
elif cv.path[0] == ('s', 'Bytes'):
cv.newSubtree = ast.Bytes(cv.newSubtree)
elif cv.path[0] == ('name', 'Alias'):
cv.newSubtree = ast.alias(cv.newSubtree, cv.oldSubtree.asname)
cv.path = cv.path[1:]
return cv
def ex_literal(val):
"""An int, float, long, bool, string, or None literal with the given
value.
"""
if sys.version_info[:2] < (3, 4):
if val is None:
return ast.Name('None', ast.Load())
elif isinstance(val, six.integer_types):
return ast.Num(val)
elif isinstance(val, bool):
return ast.Name(bytes(val), ast.Load())
elif isinstance(val, six.string_types):
return ast.Str(val)
raise TypeError(u'no literal for {0}'.format(type(val)))
elif sys.version_info[:2] < (3, 6):
if val in [None, True, False]:
return ast.NameConstant(val)
elif isinstance(val, six.integer_types):
return ast.Num(val)
elif isinstance(val, six.string_types):
return ast.Str(val)
raise TypeError(u'no literal for {0}'.format(type(val)))
else:
return ast.Constant(val)
def atom_rewrite(loc, name, token, value, number, strs, namedc, ellipsis, dict,
is_dict, is_gen, is_list, comp, yield_expr):
if name:
if not token:
return ast.Name(name.value, ast.Load(), **loc @ name)
return ex_ast.AssignExpr(ast.Name(name.value, ast.Store(),
**loc @ name),
value=value,
**loc @ token)
if number:
return ast.Num(eval(number.value), **loc @ number)
if strs:
return str_maker(*strs)
if ellipsis:
return ast.Ellipsis()
if namedc:
return ast.NameConstant(eval(namedc.value), **loc @ namedc)
if is_dict:
return dict or ex_ast.ExDict([], [], ast.Load(), **loc @ is_dict)
if is_gen:
if yield_expr:
return yield_expr
return comp(is_tuple=True) if comp else ast.Tuple([], ast.Load(), **
loc @ is_gen)
if is_list:
return comp(is_list=True) if comp else ast.List([], ast.Load(), **
loc @ is_list)
raise TypeError
def visit_BinOp(self, node):
if node.op.__class__ in self.operators:
sympy_class = self.operators[node.op.__class__]
right = self.visit(node.right)
left = self.visit(node.left)
rev = False
if isinstance(node.op, ast.Sub):
right = ast.Call(func=ast.Name(id='Mul', ctx=ast.Load()),
args=[
ast.UnaryOp(op=ast.USub(),
operand=ast.Num(1)), right
],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
elif isinstance(node.op, ast.Div):
if isinstance(node.left, ast.UnaryOp):
left, right = right, left
rev = True
left = ast.Call(func=ast.Name(id='Pow', ctx=ast.Load()),
args=[
left,
ast.UnaryOp(op=ast.USub(),
operand=ast.Num(1))
],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
else:
right = ast.Call(func=ast.Name(id='Pow', ctx=ast.Load()),
args=[
right,
ast.UnaryOp(op=ast.USub(),
operand=ast.Num(1))
],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
if rev: # undo reversal
left, right = right, left
new_node = ast.Call(func=ast.Name(id=sympy_class, ctx=ast.Load()),
args=[left, right],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
if sympy_class in ('Add', 'Mul'):
# Denest Add or Mul as appropriate
new_node.args = self.flatten(new_node.args, sympy_class)
return new_node
return node
def parse_nodes(nodes, ctx_klass=ast.Load):
out = []
while len(nodes) > 0:
node = nodes.pop(0)
if node["type"] == "name" and node["name"] == "_G":
out.append(
ast.Call(
func=ast.Name(id='globals', ctx=ast.Load()),
args=[],
keywords=[],
)
)
continue
if node["type"] == "tuple":
expressions = parse_nodes(node["value"], ctx_klass=ctx_klass)
out.append(
ast.Tuple(
elts=expressions,
ctx=ctx_klass(),
)
)
continue
if node["type"] == "table":
argument_nodes = []
keyword_nodes = []
for x in node["value"]:
if not (x["type"] == "call" and x["name"] == "="):
argument_nodes.append(x)
continue
keyword_nodes.append(x)
key_nodes = [x["args"][0] for x in keyword_nodes]
# Convert name references to strings
key_nodes = [
{"type": "string", "value": x["name"]}
if x["type"] == "name" else x
for x in key_nodes
]
value_nodes = [x["args"][1] for x in keyword_nodes]
value_nodes = [x[0] for x in value_nodes]
value_nodes = parse_nodes(value_nodes)
keywords = []
for x in (zip(key_nodes, value_nodes)):
name_node, value_node = x
name = name_node["value"]
# Apply __ to make sure its casted in Table
if name_node["type"] == "number":
name = "__{0}".format(name)
keywords.append(
ast.keyword(arg=name, value=value_node)
)
out.append(
ast.Call(
func=ast.Name(id='Table', ctx=ast.Load()),
args=parse_nodes(argument_nodes),
keywords=keywords,
)
)
continue
if node["type"] == "string":
out.append(ast.Str(s=node["value"]))
continue
if node["type"] == "boolean":
value = node["value"]
value = True if value == "true" else value
value = False if value == "false" else value
out.append(ast.NameConstant(value=value))
continue
if node["type"] == "number":
value = node["value"]
value = float(value) if "." in value else int(value)
out.append(ast.Num(n=value))
continue
if node["type"] == "nil":
out.append(ast.NameConstant(value=None))
continue
if node["type"] == "return":
out.append(
ast.Return(value=parse_nodes(node["value"])[0])
)
continue
if node["type"] == "assign":
out.append(
ast.Assign(
targets=[
ast.Name(id=node["name"], ctx=ast.Store())
],
value=parse_nodes(node["value"])[0],
)
)
continue
if node["type"] == "name":
out.append(
ast.Name(id=node["name"], ctx=ctx_klass()),
)
continue
if node["type"] == "expr":
out.append(
ast.Expr(
value=parse_nodes(node["value"])[0]
)
)
continue
if node["type"] == "function":
body_nodes = parse_nodes(node["body"])
out.append(
ast.FunctionDef(
name=node["name"],
args=ast.arguments(
args=[
ast.arg(
arg=x["name"],
annotation=None,
) for x in node["args"]
],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]
),
body=body_nodes,
decorator_list=[],
)
)
continue
if node["type"] == "if":
test_nodes = parse_nodes(node["test"])
body_nodes = parse_nodes(node["body"])
else_nodes = parse_nodes(node["else"])
out.append(
ast.If(
test=test_nodes[0],
body=body_nodes,
orelse=else_nodes,
)
)
continue
if node["type"] == "for":
target_expr = parse_nodes(node["target"], ctx_klass=ast.Store)
body_expr = parse_nodes(node["body"])
iteration_nodes = node["iteration"]
# Apply range constructor
if iteration_nodes[0]["type"] == "tuple":
iteration_expr = [
ast.Call(
func=ast.Name(id='get_for_range', ctx=ast.Load()),
args=parse_nodes(iteration_nodes[0]["value"]),
keywords=[],
)
]
else:
iteration_expr = parse_nodes(iteration_nodes)
out.append(
ast.For(
target=target_expr[0],
iter=iteration_expr[0],
body=body_expr,
orelse=[]
)
)
continue
if node["type"] == "while":
test_nodes = parse_nodes(node["test"])
body_nodes = parse_nodes(node["body"])
out.append(
ast.While(
test=test_nodes[0],
body=body_nodes,
orelse=[],
)
)
if node["type"] == "else":
body_nodes = parse_nodes(node["body"])
out = out + body_nodes
continue
if node["type"] == "call":
if node["name"] == "#":
out.append(
ast.Call(
func=ast.Name(id='len', ctx=ast.Load()),
args=parse_nodes(node["args"]),
keywords=[],
)
)
continue
if node["name"] == "[":
value_node = node["args"][0]
value_expression = parse_nodes([value_node])[0]
out.append(
ast.Subscript(
value=value_expression,
slice=ast.Index(
value=parse_nodes(node["args"][1])[0]
),
ctx=ast.Load(),
)
)
continue
if node["name"] == "=":
name_arg = node["args"][0]
value_arg = node["args"][1]
target_expr = parse_nodes([name_arg], ctx_klass=ast.Store)
value_expr = parse_nodes(value_arg)
out.append(
ast.Assign(
targets=target_expr,
value=value_expr[0],
)
)
continue
if node["name"] in ["-", "%", "+", "..", "*", "/"]:
ops = node["name"]
arg_left = parse_nodes([node["args"][0]])
arg_right = parse_nodes(node["args"][1])
ops_ref = {
"-": ast.Sub,
"%": ast.Mod,
"+": ast.Add,
"..": ast.Add,
"*": ast.Mult,
"/": ast.Div,
}
out.append(
ast.BinOp(
left=arg_left[0],
op=ops_ref[ops](),
right=arg_right[0],
)
)
continue
if node["name"] in ["and", "or"]:
ops = node["name"]
arg_left = parse_nodes([node["args"][0]])
arg_right = parse_nodes(node["args"][1])
ops_ref = {
"and": ast.And,
"or": ast.Or,
}
out.append(
ast.BoolOp(
op=ops_ref[ops](),
values=[
arg_left[0],
arg_right[0],
]
)
)
continue
if node["name"] in [">", "<", "~=", "==", "<=", ">="]:
ops = node["name"]
arg_left = parse_nodes([node["args"][0]])
arg_right = parse_nodes(node["args"][1])
ops_ref = {
">": ast.Gt,
">=": ast.GtE,
"<": ast.Lt,
"<=": ast.LtE,
"~=": ast.NotEq,
"==": ast.Eq,
}
out.append(
ast.Compare(
left=arg_left[0],
ops=[ops_ref[ops]()],
comparators=arg_right,
)
)
continue
if node["name"] == "not":
out.append(
ast.UnaryOp(
op=ast.Not(),
operand=parse_nodes(node["args"])[0]
)
)
continue
out.append(
ast.Call(
func=ast.Name(id=node["name"], ctx=ast.Load()),
args=parse_nodes(node["args"], ctx_klass=ast.Load),
keywords=[]
)
)
continue
return out
def TE(self, expression, ctx):
# default cost
cost = 1
if not self.axiom_mode:
try:
lineno = expression.lineno + self.function_line_offset
cost = self.line_durations[lineno]
except:
print('Current line duration is unknown.')
if isinstance(expression, ast.Name):
if not expression.id in ctx.keys():
node = IdentifierNode(self.compute_id(), expression.id)
elif isinstance(ctx[expression.id], TemporaryNode):
tmp_node = ctx[expression.id]
node = TemporaryNode(self.compute_id(), tmp_node.name)
else:
node = ctx[expression.id]
elif isinstance(expression, ast.Num):
node = NumNode(self.compute_id(), expression)
elif isinstance(expression, ast.Str):
node = StrNode(self.compute_id(), expression)
elif isinstance(expression, ast.NameConstant):
node = NameConstantNode(self.compute_id(), expression)
elif isinstance(expression, ast.List):
elements = list(map(lambda e: self.TE(e, ctx), expression.elts))
node = ListNode(self.compute_id(), elements)
elif isinstance(expression, ast.Set):
elements = list(map(lambda e: self.TE(e, ctx), expression.elts))
node = SetNode(self.compute_id(), elements)
elif isinstance(expression, ast.Tuple):
elements = list(map(lambda e: self.TE(e, ctx), expression.elts))
node = TupleNode(self.compute_id(), elements)
elif isinstance(expression, ast.Dict):
keys = list(map(lambda k: self.TE(k, ctx), expression.keys))
values = list(map(lambda v: self.TE(v, ctx), expression.values))
node = DictNode(self.compute_id(), [keys] + [values])
elif isinstance(expression, ast.BinOp):
op = expression.op
args = [
self.TE(expression.left, ctx),
self.TE(expression.right, ctx)
]
node = BinOpNode(self.compute_id(), op, args)
node.cost = cost
elif isinstance(expression, ast.BoolOp):
op = expression.op
args = [self.TE(operand, ctx) for operand in expression.values]
node = BoolOpNode(self.compute_id(), op, args)
node.cost = cost
elif isinstance(expression, ast.Compare):
head = self.TE(expression.left, ctx)
ops = expression.ops
tail = [
self.TE(operand, ctx) for operand in expression.comparators
]
node = CompareNode(self.compute_id(), ops, [head] + tail)
node.cost = cost
elif isinstance(expression, ast.UnaryOp):
node = UnaryOpNode(self.compute_id(), expression.op,
[self.TE(expression.operand, ctx)])
self.nodes.append(node)
node.cost = cost
elif isinstance(expression, ast.Call):
args = list(map(lambda a: self.TE(a, ctx), expression.args))
# adding children to nodes created in axiom mode
if self.axiom_mode and isinstance(
expression.func,
ast.Name) and expression.func.id in ctx.keys():
name = expression.func
ctx[name.id].children = args
node = ctx[name.id]
# creating nparray node
elif isinstance(expression.func, ast.Attribute) and isinstance(expression.func.value, ast.Name) and \
expression.func.value.id == 'np' and expression.func.attr == 'array':
node = NPArrayNode(self.compute_id(), args)
else:
func = self.TE(expression.func, ctx)
node = FunctionCall(self.compute_id(), [func] + args)
node.cost = cost
elif isinstance(expression, ast.IfExp):
cond = self.TE(expression.test, ctx)
t = self.TE(expression.body, ctx)
f = self.TE(expression.orelse, ctx)
node = IfExpNode(self.compute_id(), [cond, t, f])
node.cost = cost
elif isinstance(expression, ast.Lambda):
args = list(map(lambda a: self.TE(a, ctx), expression.args.args))
# lambda function has new ctx
new_ctx = ctx.copy()
for arg in args:
new_ctx[arg.name] = arg
body = self.TE(expression.body, new_ctx)
node = LambdaNode(self.compute_id(), args + [body])
elif isinstance(expression, ast.arg):
node = VariableNode(self.compute_id(), expression.arg)
elif isinstance(expression, ast.Attribute):
value = self.TE(expression.value, ctx)
node = AttributeNode(self.compute_id(), expression.attr, [value])
elif isinstance(expression, ast.Subscript):
value = self.TE(expression.value, ctx)
slice = self.TE(expression.slice, ctx)
node = SubscriptNode(self.compute_id(), [value, slice])
elif isinstance(expression, ast.Index):
value = self.TE(expression.value, ctx)
node = IndexNode(self.compute_id(), [value])
elif isinstance(expression, ast.Slice):
none_expression = ast.NameConstant(None)
lower = self.TE(expression.lower, ctx) if expression.lower != None else \
NameConstantNode(self.compute_id(), none_expression)
upper = self.TE(expression.upper, ctx) if expression.upper != None else \
NameConstantNode(self.compute_id(), none_expression)
step = self.TE(expression.step, ctx) if expression.step != None else \
NameConstantNode(self.compute_id(), none_expression)
node = SliceNode(self.compute_id(), [lower, upper, step])
elif isinstance(expression, ast.ExtSlice):
dims = [self.TE(dim, ctx) for dim in expression.dims]
node = ExtSliceNode(self.compute_id(), dims)
elif isinstance(expression, ast.comprehension):
if isinstance(expression.target, ast.Name):
target = VariableNode(self.compute_id(), expression.target.id)
ctx[target.name] = target
else:
target_vars = [
VariableNode(self.compute_id(), arg.id)
for arg in expression.target
]
for var in target_vars:
ctx[var.name] = var
target = TupleNode(self.compute_id(), target_vars)
iter = self.TE(expression.iter, ctx)
ifs = [self.TE(test, ctx) for test in expression.ifs]
node = ComprehensionNode(self.compute_id(), [target, iter] + ifs)
elif isinstance(expression, ast.ListComp):
generators = [self.TE(gen, ctx) for gen in expression.generators]
elt = self.TE(expression.elt, ctx)
node = ListCompNode(self.compute_id(), [elt] + generators)
elif isinstance(expression, ast.SetComp):
generators = [self.TE(gen, ctx) for gen in expression.generators]
elt = self.TE(expression.elt, ctx)
node = SetCompNode(self.compute_id(), [elt] + generators)
elif isinstance(expression, ast.GeneratorExp):
generators = [self.TE(gen, ctx) for gen in expression.generators]
elt = self.TE(expression.elt, ctx)
node = GeneratorExpNode(self.compute_id(), [elt] + generators)
elif isinstance(expression, ast.DictComp):
generators = [self.TE(gen, ctx) for gen in expression.generators]
key = self.TE(expression.key, ctx)
value = self.TE(expression.value, ctx)
node = ListCompNode(self.compute_id(), [key, value] + generators)
else:
raise Exception('Currently unsuported type: ' +
str(type(expression)))
self.nodes.append(node)
return node
def _create_init_method(self, class_node) -> ast.FunctionDef:
"""Create the __init__ method
def __init__(self, id: typing.Optional[int]=None) -> None:
self.id = None
"""
# Create the arguments for the __init__ method
init_args = []
for prop in self.properties:
attribute_name = prop.attribute_name
if not attribute_name:
continue
init_args.append(
ast.arg(
arg=attribute_name,
annotation=self._get_annotation_for_property(prop),
))
# Generate an __init__ function. This isn't really needed for
# attrs but it results in some nicer code completion for editors
init_func = ast.FunctionDef(
name="__init__",
args=ast.arguments(
args=[ast.arg(arg="self", annotation=None)],
vararg=None,
kwonlyargs=init_args,
kw_defaults=[
ast.NameConstant(value=None)
for _ in range(0, len(init_args))
],
kwarg=None,
defaults=[],
),
body=[],
decorator_list=[],
returns=ast.NameConstant(value=None),
)
# Create assignments (self.x = x)
for prop in self.resource.properties:
attribute_name = prop.attribute_name
if not attribute_name:
continue
init_func.body.append(
ast.Assign(
targets=[
ast.Attribute(value=ast.Name(id="self"),
attr=attribute_name)
],
value=ast.Name(id=attribute_name),
))
# Create the keyword arguments for the super() call. It should only
# contain the arguments which are part of the superclass. Special
# handling is done for discriminator values which are enums.
init_values = []
super_attributes = []
if self.resource.base and self.resource.base.properties:
super_attributes = [
p.attribute_name
for p in self.resource.base.get_all_properties()
]
for name in self.attribute_names:
if (self.discriminator_attr
and name == self.discriminator_attr.attribute_name):
if self.discriminator_attr.type.enum:
init_values.append(
ast.keyword(
arg=name,
value=ast.Attribute(
value=ast.Name(
id=f"{self.discriminator_attr.type.name}"),
attr=enum_attr(
self.resource.discriminator_value),
),
))
if (self.resource.package_name !=
self.discriminator_attr.type.package_name):
self.generator.add_import_statement(
self.resource.package_name,
f"commercetools.types.{self.discriminator_attr.type.package_name}",
self.discriminator_attr.type.name,
)
else:
init_values.append(
ast.keyword(arg=name,
value=ast.Str(
s=self.resource.discriminator_value)))
elif name in super_attributes:
init_values.append(
ast.keyword(arg=name, value=ast.Name(id=name)))
# Add the super().__init__(args) call. This call is only required when
# we have a superclass. Only the args
if class_node.bases:
init_func.body.append(
ast.Expr(value=ast.Call(
func=ast.Attribute(
value=ast.Call(
func=ast.Name(id="super"), args=[], keywords=[]),
attr="__init__",
),
args=[],
keywords=init_values,
)))
return init_func
right=ast.Name(id="b", ctx=ast.Load()))
],
ctx=ast.Load())
# a, b = b, a + b
assign1 = ast.Assign(targets=left_target, value=right_value)
# -------------------------------------------------------------------------------------
'''
while True: # First iteration:
yield a # yield 0 to start with and then
a, b = b, a + b # a will now be 1, and b will also be 1, (0 + 1)
'''
while_body = [yield_a, assign1]
whileLoop = ast.While(test=ast.NameConstant(value=True),
body=while_body,
orelse=[])
'''
def fib():
a = 0
b = 1
while True: # First iteration:
yield a # yield 0 to start with and then
a, b = b, a + b # a will now be 1, and b will also be 1, (0 + 1)
'''
fib_def = ast.FunctionDef(name="fib",
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
def visit_Call(self, node):
new_node = self.generic_visit(node)
if isinstance(node.func, ast.Name) and node.func.id in self.functions:
new_node.keywords.append(ast.keyword(arg='evaluate', value=ast.NameConstant(value=False, ctx=ast.Load())))
return new_node
def visit_Module(self, node):
setDoneNode = ast.Expr(ast.Call(ast.Attribute(ast.Name("finishedExecSig", ast.Load()), "set_result", ast.Load()), [ast.NameConstant(None)], []))
setExceptionNode = ast.Expr(ast.Call(ast.Attribute(ast.Name("finishedExecSig", ast.Load()), "set_exception", ast.Load()), [ast.Name("e", ast.Load())], []))
mainBody = node.body + [setDoneNode]
tryExceptNode = ast.ExceptHandler(ast.Name("Exception", ast.Load()), "e", [setExceptionNode])
tryNode = ast.Try(mainBody, [tryExceptNode], [], [])
newNode = ast.Module([tryNode])
ast.copy_location(newNode, node)
ast.fix_missing_locations(newNode)
return newNode
def test_nameconstant(self):
self.expr(ast.NameConstant(4), "singleton must be True, False, or None")
def visit_For(self, loop):
"""Rewrite for loops as while loops
for i in iterable:
<body>
becomes
p = iter(iterable)
while True:
try:
i = next(p)
except StopIteration:
break
<body>
>>> self = SyntaxRewriter(buffer='foo')
>>> code = '''
...
... x
... for i in range(2):
... for j in range(2):
... k = i + j
... print(k)
... y
...
... '''
>>> tree = ast.parse(code)
>>> loop = tree.body[1]
"""
loop = self.generic_visit(loop)
# p = iter(iterable)
var = __random_string__()
assign_iter = ast.Assign(targets=[ast.Name(id=var, ctx=ast.Store())],
value=ast.Call(func=ast.Name(id='iter',
ctx=ast.Load()),
args=[loop.iter],
keywords=[]))
# i = next(iter(iterable))
assign_next = ast.Assign(targets=[loop.target],
value=ast.Call(
func=ast.Name(id='next', ctx=ast.Load()),
args=[ast.Name(id=var, ctx=ast.Load())],
keywords=[]))
# try:
# p = iter(iterable)
# except:
# break
try_node = ast.Try(body=[assign_next],
handlers=[
ast.ExceptHandler(type=ast.Name(
id='StopIteration', ctx=ast.Load()),
name=None,
body=[ast.Break()])
],
orelse=[],
finalbody=[])
# while True:
# try:
# p = iter(iterable)
# except:
# break
while_node = ast.While(test=ast.NameConstant(value=True),
body=[try_node] + loop.body,
orelse=[])
content = f'`for {astor.to_source(loop.target).strip()} in {astor.to_source(loop.iter).strip()} ...`'
exprs = [
Annotator.make_annotation(buffer=self.buffer,
content=content,
cell_type='2',
lineno=loop.lineno), assign_iter,
while_node
]
return exprs
def at_dampe_time(self, node):
if self.world.ensure_tod_access:
# tod has DAMPE or (tod == NONE and (find a path from a provider))
# parsing is better than constructing this expression by hand
return ast.parse("(tod & TimeOfDay.DAMPE) if tod else state.search.can_reach(spot.parent_region, age=age, tod=TimeOfDay.DAMPE)", mode='eval').body
return ast.NameConstant(True)
def at_day(self, node):
if self.world.ensure_tod_access:
# tod has DAY or (tod == NONE and (ss or find a path from a provider))
# parsing is better than constructing this expression by hand
return ast.parse("(tod & TimeOfDay.DAY) if tod else (state.has_all_of(('Ocarina', 'Suns Song')) or state.search.can_reach(spot.parent_region, age=age, tod=TimeOfDay.DAY))", mode='eval').body
return ast.NameConstant(True)
def global_value_to_node(self,
value,
parent_node,
qualname,
recurse=False,
detect_callables=False):
# if recurse is false, we don't allow recursion into lists
# this should not happen anyway; the globals dict should only contain
# single "level" lists
if not recurse and isinstance(value, (list, tuple)):
# bail after more than one level of lists
return None
if isinstance(value, list):
elts = [
self.global_value_to_node(v,
parent_node,
qualname + f'[{i}]',
detect_callables=detect_callables)
for i, v in enumerate(value)
]
if any(e is None for e in elts):
return None
newnode = ast.List(elts=elts, ctx=parent_node.ctx)
elif isinstance(value, tuple):
elts = [
self.global_value_to_node(v,
parent_node,
qualname + f'[{i}]',
detect_callables=detect_callables)
for i, v in enumerate(value)
]
if any(e is None for e in elts):
return None
newnode = ast.Tuple(elts=elts, ctx=parent_node.ctx)
elif isinstance(value, symbolic.symbol):
# Symbols resolve to the symbol name
newnode = ast.Name(id=value.name, ctx=ast.Load())
elif (dtypes.isconstant(value) or isinstance(value, SDFG)
or hasattr(value, '__sdfg__')):
# Could be a constant, an SDFG, or SDFG-convertible object
if isinstance(value, SDFG) or hasattr(value, '__sdfg__'):
self.closure.closure_sdfgs[qualname] = value
else:
self.closure.closure_constants[qualname] = value
# Compatibility check since Python changed their AST nodes
if sys.version_info >= (3, 8):
newnode = ast.Constant(value=value, kind='')
else:
if value is None:
newnode = ast.NameConstant(value=None)
elif isinstance(value, str):
newnode = ast.Str(s=value)
else:
newnode = ast.Num(n=value)
newnode.oldnode = copy.deepcopy(parent_node)
elif detect_callables and hasattr(value, '__call__') and hasattr(
value.__call__, '__sdfg__'):
return self.global_value_to_node(value.__call__, parent_node,
qualname, recurse,
detect_callables)
elif isinstance(value, numpy.ndarray):
# Arrays need to be stored as a new name and fed as an argument
if id(value) in self.closure.array_mapping:
arrname = self.closure.array_mapping[id(value)]
else:
arrname = self._qualname_to_array_name(qualname)
desc = data.create_datadescriptor(value)
self.closure.closure_arrays[arrname] = (
qualname, desc, lambda: eval(qualname, self.globals),
False)
self.closure.array_mapping[id(value)] = arrname
newnode = ast.Name(id=arrname, ctx=ast.Load())
elif detect_callables and callable(value):
# Try parsing the function as a dace function/method
newnode = None
try:
from dace.frontend.python import parser # Avoid import loops
parent_object = None
if hasattr(value, '__self__'):
parent_object = value.__self__
# If it is a callable object
if (not inspect.isfunction(value)
and not inspect.ismethod(value)
and not inspect.isbuiltin(value)
and hasattr(value, '__call__')):
parent_object = value
value = value.__call__
# Replacements take precedence over auto-parsing
try:
if has_replacement(value, parent_object, parent_node):
return None
except Exception:
pass
# Store the handle to the original callable, in case parsing fails
cbqualname = astutils.rname(parent_node)
cbname = self._qualname_to_array_name(cbqualname, prefix='')
self.closure.callbacks[cbname] = (cbqualname, value, False)
# From this point on, any failure will result in a callback
newnode = ast.Name(id=cbname, ctx=ast.Load())
# Decorated or functions with missing source code
sast, _, _, _ = astutils.function_to_ast(value)
if len(sast.body[0].decorator_list) > 0:
return newnode
parsed = parser.DaceProgram(value, [], {}, False,
dtypes.DeviceType.CPU)
# If method, add the first argument (which disappears due to
# being a bound method) and the method's object
if parent_object is not None:
parsed.methodobj = parent_object
parsed.objname = inspect.getfullargspec(value).args[0]
res = self.global_value_to_node(parsed, parent_node, qualname,
recurse, detect_callables)
# Keep callback in callbacks in case of parsing failure
# del self.closure.callbacks[cbname]
return res
except Exception: # Parsing failed (almost any exception can occur)
return newnode
else:
return None
if parent_node is not None:
return ast.copy_location(newnode, parent_node)
else:
return newnode
def _comparison_rhs_is_none(node):
"Determine if the node is a comparison with `None` on the RHS."
return ((len(node.comparators) == 1)
and (compare_ast(node.comparators[0], ast.NameConstant(None))))
def compile_func(gen: 'Generator',
func: Callable,
strategy: Strategy,
with_hooks: bool = False) -> Callable:
"""
The compilation basically assigns functionality to each of the operator calls as
governed by the semantics (strategy). Memoization is done with the keys as the `func`,
the class of the `strategy` and the `with_hooks` argument.
Args:
gen (Generator): The generator object containing the function to compile
func (Callable): The function to compile
strategy (Strategy): The strategy governing the behavior of the operators
with_hooks (bool): Whether support for hooks is required
Returns:
The compiled function
"""
if isinstance(strategy, PartialReplayStrategy):
strategy = strategy.backup_strategy
if with_hooks:
cache = CompilationCache.WITH_HOOKS[strategy.__class__]
else:
cache = CompilationCache.WITHOUT_HOOKS[strategy.__class__]
if func in cache:
return cache[func]
cache[func] = None
source_code, start_lineno = inspect.getsourcelines(func)
source_code = ''.join(source_code)
f_ast = astutils.parse(textwrap.dedent(source_code))
# This matches up line numbers with original file and is thus super useful for debugging
ast.increment_lineno(f_ast, start_lineno - 1)
# Remove the ``@generator`` decorator to avoid recursive compilation
f_ast.decorator_list = [
d for d in f_ast.decorator_list
if (not isinstance(d, ast.Name) or d.id != 'generator') and (
not isinstance(d, ast.Attribute) or d.attr != 'generator') and (
not (isinstance(d, ast.Call) and isinstance(d.func, ast.Name))
or d.func.id != 'generator')
]
# Get all the external dependencies of this function.
# We rely on a modified closure function adopted from the ``inspect`` library.
closure_vars = getclosurevars_recursive(func, f_ast)
g = {**closure_vars.nonlocals.copy(), **closure_vars.globals.copy()}
known_ops: Set[str] = strategy.get_known_ops()
known_methods: Set[str] = strategy.get_known_methods()
op_info_constructor = OpInfoConstructor()
delayed_compilations: List[Tuple[Generator, str]] = []
ops = {}
handlers = {}
op_infos = {}
op_idx: int = 0
composition_cnt: int = 0
for n in astutils.preorder_traversal(f_ast):
if isinstance(n, ast.Call) and isinstance(
n.func, ast.Name) and n.func.id in known_ops:
# Rename the function call, and assign a new function to be called during execution.
# This new function is determined by the semantics (strategy) being used for compilation.
# Also determine if there any eligible hooks for this operator call.
op_idx += 1
handler_idx = len(handlers)
op_info: OpInfo = op_info_constructor.get(n, gen.name, gen.group)
n.keywords.append(
ast.keyword(arg='model',
value=ast.Name(_GEN_MODEL_VAR, ctx=ast.Load())))
n.keywords.append(
ast.keyword(arg='op_info',
value=ast.Name(f"_op_info_{op_idx}",
ctx=ast.Load())))
op_infos[f"_op_info_{op_idx}"] = op_info
n.keywords.append(
ast.keyword(arg='handler',
value=ast.Name(f"_handler_{handler_idx}",
ctx=ast.Load())))
handler = strategy.get_op_handler(op_info)
handlers[f"_handler_{handler_idx}"] = handler
if not with_hooks:
n.func = astutils.parse(
f"{_GEN_STRATEGY_VAR}.generic_op").value
else:
n.keywords.append(
ast.keyword(arg=_GEN_HOOK_VAR,
value=ast.Name(_GEN_HOOK_VAR, ctx=ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_STRATEGY_VAR,
value=ast.Name(_GEN_STRATEGY_VAR,
ctx=ast.Load())))
n.func.id = _GEN_HOOK_WRAPPER
ops[_GEN_HOOK_WRAPPER] = hook_wrapper
ast.fix_missing_locations(n)
elif isinstance(n, ast.Call) and isinstance(
n.func, ast.Name) and n.func.id in known_methods:
# Similar in spirit to the known_ops case, just much less fancy stuff to do.
# Only need to get the right handler which we will achieve by simply making this
# a method call instead of a regular call.
n.func = ast.Attribute(value=ast.Name(_GEN_STRATEGY_VAR,
ctx=ast.Load()),
attr=n.func.id,
ctx=ast.Load())
ast.fix_missing_locations(n)
elif isinstance(n, ast.Call):
# Try to check if it is a call to a Generator
# TODO : Can we be more sophisticated in our static analysis here
try:
function = eval(astunparse.unparse(n.func), g)
if isinstance(function, Generator):
call_id = f"{_GEN_COMPOSITION_ID}_{composition_cnt}"
composition_cnt += 1
n.func.id = call_id
n.keywords.append(
ast.keyword(arg=_GEN_EXEC_ENV_VAR,
value=ast.Name(_GEN_EXEC_ENV_VAR,
ctx=ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_STRATEGY_VAR,
value=ast.Name(_GEN_STRATEGY_VAR,
ctx=ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_MODEL_VAR,
value=ast.Name(_GEN_MODEL_VAR,
ctx=ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_HOOK_VAR,
value=ast.Name(_GEN_HOOK_VAR,
ctx=ast.Load())))
ast.fix_missing_locations(n)
# We delay compilation to handle mutually recursive generators
delayed_compilations.append((function, call_id))
except:
pass
# Add the execution environment argument to the function
f_ast.args.kwonlyargs.append(
ast.arg(arg=_GEN_EXEC_ENV_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the strategy argument to the function
f_ast.args.kwonlyargs.append(
ast.arg(arg=_GEN_STRATEGY_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the strategy argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_MODEL_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the hook argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_HOOK_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
ast.fix_missing_locations(f_ast)
# New name so it doesn't clash with original
func_name = f"{_GEN_COMPILED_TARGET_ID}_{len(cache)}"
g.update({k: v for k, v in ops.items()})
g.update({k: v for k, v in handlers.items()})
g.update({k: v for k, v in op_infos.items()})
module = ast.Module()
module.body = [f_ast]
# Passing ``g`` to exec allows us to execute all the new functions
# we assigned to every operator call in the previous AST walk
exec(compile(module, filename=inspect.getabsfile(func), mode="exec"), g)
result = g[func.__name__]
if inspect.ismethod(func):
result = result.__get__(func.__self__, func.__self__.__class__)
# Restore the correct namespace so that tracebacks contain actual function names
g[gen.name] = gen
g[func_name] = result
cache[func] = result
# Handle the delayed compilations now that we have populated the cache
for gen, call_id in delayed_compilations:
compiled_func = compile_func(gen, gen.func, strategy, with_hooks)
if gen.caching and isinstance(strategy, DfsStrategy):
# Add instructions for using cached result if any
g[call_id] = cache_wrapper(compiled_func)
else:
g[call_id] = compiled_func
return result
def to_ast(self, input_val):
return ast.NameConstant(input_val)
def test_not_pp(self):
self.assertEqual(
unaryops[ast.Not].pprint(ast.NameConstant(value=True)), "not True")
self.assertEqual(
unaryops[ast.Not].pprint(ast.NameConstant(value=False)),
"not False")
def visit_Delete(self, node):
"""
This converter replaces bare deletes with conditional global pops.
It is roughly equivalent to transforming:
.. code:: python
del foobar
into:
.. code:: python
if 'foobar' in globals():
globals().pop('foobar')
else:
del foobar
This thus makes deletions in retain mode work more-or-less as intended.
"""
return ast.If(
test=ast.NameConstant(
value=True, # if True; aka unconditional, will be optimized out
lineno=node.lineno,
col_offset=node.col_offset
),
body=[
ast.If(
# if 'x' in globals():
test=ast.Compare(
# 'x'
left=ast.Str(
s=target.id,
lineno=node.lineno,
col_offset=node.col_offset
),
ops=[
# in
ast.In(
lineno=node.lineno,
col_offset=node.col_offset
)
],
comparators=[
# globals()
self.globals_call(node)
],
lineno=node.lineno,
col_offset=node.col_offset
),
body=[
ast.Expr(
# globals().pop('x')
value=ast.Call(
# globals().pop
func=ast.Attribute(
value=self.globals_call(node),
attr='pop',
ctx=ast.Load(),
lineno=node.lineno,
col_offset=node.col_offset
),
args=[
# 'x'
ast.Str(
s=target.id,
lineno=node.lineno,
col_offset=node.col_offset
)
],
keywords=[],
lineno=node.lineno,
col_offset=node.col_offset
),
lineno=node.lineno,
col_offset=node.col_offset
)
],
# else:
orelse=[
# del x
ast.Delete(
targets=[target],
lineno=node.lineno,
col_offset=node.col_offset
)
],
lineno=node.lineno,
col_offset=node.col_offset
)
if isinstance(target, ast.Name) else
ast.Delete(
targets=[target],
lineno=node.lineno,
col_offset=node.col_offset
)
# for each target to be deleted, e.g. `del {x}, {y}, {z}`
for target in node.targets
],
orelse=[],
lineno=node.lineno,
col_offset=node.col_offset
)
def generate(self, element, GC: GenerationContext):
from anoky.generation.util import expr_wrap
acode = element.code
if isinstance(acode, Form):
head = acode.first
headcode = head.code
if isinstance(
headcode,
Identifier) and headcode.full_name in GC.special_forms:
head.color = colors.SPECIAL_FORM
hcname = headcode.full_name
special_form = GC.special_forms[hcname]
generated_code = special_form.generate(element, GC)
return generated_code
else:
# function call
# #(func arg1 arg2 arg3 ...)
func_element = head
#func_element_code = headcode
with GC.let(domain=ExDom):
func_code = GC.generate(func_element)
arg_elements = acode[1:]
args = []
keywords = []
for arg_element in arg_elements:
arg_element_code = arg_element.code
if isinstance(arg_element_code, Form):
if is_form(arg_element_code, '='):
# keyword argument
kw_name = arg_element_code[1].code.full_name
with GC.let(domain=ExDom):
value_code = GC.generate(arg_element_code[2])
keywords.append(ast.keyword(kw_name, value_code))
elif is_form(arg_element_code,
'*') and len(arg_element_code) == 2:
# stared argument - expand as list
with GC.let(domain=ExDom):
arg_code = GC.generate(arg_element_code[1])
args.append(ast.Starred(arg_code, ast.Load()))
elif is_form(arg_element_code,
'**') and len(arg_element_code) == 2:
# double starred argument - expand as kwlist
assert len(arg_element_code) == 2
# verify no other dblstars already?
with GC.let(domain=ExDom):
arg_code = GC.generate(arg_element_code[1])
keywords.append(ast.keyword(None, arg_code))
else:
# positional argument
with GC.let(domain=ExDom):
arg_code = GC.generate(arg_element)
args.append(arg_code)
else:
# arg_element_code not a Form
# then generate as expression
with GC.let(domain=ExDom):
arg_code = GC.generate(arg_element)
args.append(arg_code)
return expr_wrap(ast.Call(func_code, args, keywords), GC)
if isinstance(acode, Seq):
seq_codes = []
with GC.let(domain=ExDom):
for e in acode:
seq_codes.append(GC.generate(e))
if GC.domain == LVDom:
return ast.Tuple(seq_codes, ast.Store())
elif GC.domain in [ExDom, SDom]:
return expr_wrap(ast.Tuple(seq_codes, ast.Load()), GC)
else:
raise CodeGenerationError(
acode.range,
"Unexpected seq in domain `%s`." % str(GC.domain))
if isinstance(acode, Literal):
element.color = colors.LITERAL
if acode.type is str:
return expr_wrap(ast.Str(acode.value), GC)
elif acode.type in [int, float]:
return expr_wrap(ast.Num(acode.value), GC)
else:
assert False
if isinstance(acode, Identifier):
if acode.full_name == "True":
return expr_wrap(ast.NameConstant(True), GC)
elif acode.full_name == "False":
return expr_wrap(ast.NameConstant(False), GC)
elif acode.full_name == "None":
return expr_wrap(ast.NameConstant(None), GC)
elif acode.full_name in GC.special_forms:
element.color = colors.SPECIAL_FORM
raise CodeGenerationError(
acode.range,
"Refering to special form `%s` by name requires the use of `the`."
% acode.full_name)
# elif acode.full_name in GC.macros:
# raise CodeGenerationError(acode.range,
# "Refering to macro `%s` by name requires the use of `the`." % acode.full_name)
# elif acode.full_name in GC.id_macros:
# raise CodeGenerationError(acode.range,
# "Refering to identifier macro `%s` by name requires the use of `the`." % acode.full_name)
elif GC.domain == LVDom:
return ast.Name(acode.full_name, ast.Store())
elif GC.domain == DelDom:
return ast.Name(acode.full_name, ast.Del())
else:
return expr_wrap(ast.Name(acode.full_name, ast.Load()), GC)
def _get_property_field(self, prop):
if prop.type is None:
return ast.Call(
func=ast.Name(id=FIELD_TYPES["object"]),
args=[],
keywords=[
ast.keyword(arg="allow_none",
value=ast.NameConstant(True, kind=None))
],
)
elif prop.type.enum:
self.generator.add_import_statement(self.resource.package_name,
"marshmallow_enum")
return ast.Call(
func=ast.Name(id="marshmallow_enum.EnumField"),
args=[
ast.Attribute(value=ast.Name(id="types"),
attr=prop.type.name)
],
keywords=[
ast.keyword(arg="by_value",
value=ast.NameConstant(True, kind=None))
],
)
elif prop.type.discriminator:
return self._create_discriminator_field(prop.type)
elif prop.type.name.startswith("/"):
return ast.Call(func=ast.Name(id=prop.type.name + "Field"),
args=[],
keywords=[])
elif prop.type.name in FIELD_TYPES:
node = ast.Call(
func=ast.Name(id=FIELD_TYPES[prop.type.name]),
args=[],
keywords=[
ast.keyword(arg="allow_none",
value=ast.NameConstant(True, kind=None))
],
)
if prop.type.name == "array":
assert prop.items, f"The array property {prop.name} has no items"
assert prop.items_types
# TODO: We for now assume that the items are all subclasses of
# the first item. We shouldn't do that :-)
if prop.items_types[0].discriminator:
node.args.append(
self._create_discriminator_field(prop.items_types[0]))
else:
node.args.append(
self._create_nested_field(prop.items_types[0]))
return node
# Dict Field
elif "asMap" in prop.type.annotations:
if self.resource.package_name != prop.type.package_name:
self.generator.import_resource(
self.resource.package_name,
prop.type.package_name,
prop.type.name + "Field",
)
return ast.Call(
func=ast.Name(id=prop.type.name + "Field"),
args=[],
keywords=[
ast.keyword(arg="allow_none",
value=ast.NameConstant(True, kind=None))
],
)
elif prop.type.base and prop.type.base.name == "string":
return ast.Call(func=ast.Name(id="marshmallow.fields.String"),
args=[],
keywords=[])
else:
return self._create_nested_field(prop.type)
def visit_Assert(self, assert_):
"""Return the AST statements to replace the ast.Assert instance.
This rewrites the test of an assertion to provide
intermediate values and replace it with an if statement which
raises an assertion error with a detailed explanation in case
the expression is false.
"""
if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1:
from _pytest.warning_types import PytestAssertRewriteWarning
import warnings
warnings.warn_explicit(
PytestAssertRewriteWarning(
"assertion is always true, perhaps remove parentheses?"
),
category=None,
filename=self.module_path,
lineno=assert_.lineno,
)
self.statements = []
self.variables = []
self.variable_counter = itertools.count()
if self.enable_assertion_pass_hook:
self.format_variables = []
self.stack = []
self.expl_stmts = []
self.push_format_context()
# Rewrite assert into a bunch of statements.
top_condition, explanation = self.visit(assert_.test)
# If in a test module, check if directly asserting None, in order to warn [Issue #3191]
if self.module_path is not None:
self.statements.append(
self.warn_about_none_ast(
top_condition, module_path=self.module_path, lineno=assert_.lineno
)
)
if self.enable_assertion_pass_hook: # Experimental pytest_assertion_pass hook
negation = ast.UnaryOp(ast.Not(), top_condition)
msg = self.pop_format_context(ast.Str(explanation))
# Failed
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
gluestr = "\n>assert "
else:
assertmsg = ast.Str("")
gluestr = "assert "
err_explanation = ast.BinOp(ast.Str(gluestr), ast.Add(), msg)
err_msg = ast.BinOp(assertmsg, ast.Add(), err_explanation)
err_name = ast.Name("AssertionError", ast.Load())
fmt = self.helper("_format_explanation", err_msg)
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
statements_fail = []
statements_fail.extend(self.expl_stmts)
statements_fail.append(raise_)
# Passed
fmt_pass = self.helper("_format_explanation", msg)
orig = self._assert_expr_to_lineno()[assert_.lineno]
hook_call_pass = ast.Expr(
self.helper(
"_call_assertion_pass",
ast.Num(assert_.lineno),
ast.Str(orig),
fmt_pass,
)
)
# If any hooks implement assert_pass hook
hook_impl_test = ast.If(
self.helper("_check_if_assertion_pass_impl"),
self.expl_stmts + [hook_call_pass],
[],
)
statements_pass = [hook_impl_test]
# Test for assertion condition
main_test = ast.If(negation, statements_fail, statements_pass)
self.statements.append(main_test)
if self.format_variables:
variables = [
ast.Name(name, ast.Store()) for name in self.format_variables
]
clear_format = ast.Assign(variables, ast.NameConstant(None))
self.statements.append(clear_format)
else: # Original assertion rewriting
# Create failure message.
body = self.expl_stmts
negation = ast.UnaryOp(ast.Not(), top_condition)
self.statements.append(ast.If(negation, body, []))
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
explanation = "\n>assert " + explanation
else:
assertmsg = ast.Str("")
explanation = "assert " + explanation
template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
msg = self.pop_format_context(template)
fmt = self.helper("_format_explanation", msg)
err_name = ast.Name("AssertionError", ast.Load())
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
body.append(raise_)
# Clear temporary variables by setting them to None.
if self.variables:
variables = [ast.Name(name, ast.Store()) for name in self.variables]
clear = ast.Assign(variables, ast.NameConstant(None))
self.statements.append(clear)
# Fix line numbers.
for stmt in self.statements:
set_location(stmt, assert_.lineno, assert_.col_offset)
return self.statements
def gen_none_node(self):
if IS_PY34_OR_GREATER:
return ast.NameConstant(value=None)
else:
return ast.Name(id='None', ctx=ast.Load())
def parse_annotation_mutable_layers(code, lineno):
"""Parse the string of mutable layers in annotation.
Return a list of AST Expr nodes
code: annotation string (excluding '@')
"""
module = ast.parse(code)
assert type(module) is ast.Module, 'internal error #1'
assert len(
module.body
) == 1, 'Annotation mutable_layers contains more than one expression'
assert type(module.body[0]) is ast.Expr, 'Annotation is not expression'
call = module.body[0].value
nodes = []
mutable_id = 'mutable_block_' + str(lineno)
mutable_layer_cnt = 0
for arg in call.args:
fields = {
'layer_choice': False,
'fixed_inputs': False,
'optional_inputs': False,
'optional_input_size': False,
'layer_output': False
}
for k, value in zip(arg.keys, arg.values):
if k.id == 'layer_choice':
assert not fields[
'layer_choice'], 'Duplicated field: layer_choice'
assert type(
value
) is ast.List, 'Value of layer_choice should be a list'
call_funcs_keys = []
call_funcs_values = []
call_kwargs_values = []
for call in value.elts:
assert type(
call
) is ast.Call, 'Element in layer_choice should be function call'
call_name = astor.to_source(call).strip()
call_funcs_keys.append(ast.Str(s=call_name))
call_funcs_values.append(call.func)
assert not call.args, 'Number of args without keyword should be zero'
kw_args = []
kw_values = []
for kw in call.keywords:
kw_args.append(kw.arg)
kw_values.append(kw.value)
call_kwargs_values.append(
ast.Dict(keys=kw_args, values=kw_values))
call_funcs = ast.Dict(keys=call_funcs_keys,
values=call_funcs_values)
call_kwargs = ast.Dict(keys=call_funcs_keys,
values=call_kwargs_values)
fields['layer_choice'] = True
elif k.id == 'fixed_inputs':
assert not fields[
'fixed_inputs'], 'Duplicated field: fixed_inputs'
assert type(
value
) is ast.List, 'Value of fixed_inputs should be a list'
fixed_inputs = value
fields['fixed_inputs'] = True
elif k.id == 'optional_inputs':
assert not fields[
'optional_inputs'], 'Duplicated field: optional_inputs'
assert type(
value
) is ast.List, 'Value of optional_inputs should be a list'
var_names = [
ast.Str(s=astor.to_source(var).strip())
for var in value.elts
]
optional_inputs = ast.Dict(keys=var_names, values=value.elts)
fields['optional_inputs'] = True
elif k.id == 'optional_input_size':
assert not fields[
'optional_input_size'], 'Duplicated field: optional_input_size'
assert type(
value
) is ast.Num, 'Value of optional_input_size should be a number'
optional_input_size = value
fields['optional_input_size'] = True
elif k.id == 'layer_output':
assert not fields[
'layer_output'], 'Duplicated field: layer_output'
assert type(
value
) is ast.Name, 'Value of layer_output should be ast.Name type'
layer_output = value
fields['layer_output'] = True
else:
raise AssertionError('Unexpected field in mutable layer')
# make call for this mutable layer
assert fields['layer_choice'], 'layer_choice must exist'
assert fields['layer_output'], 'layer_output must exist'
mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt)
mutable_layer_cnt += 1
target_call_attr = ast.Attribute(value=ast.Name(id='nni',
ctx=ast.Load()),
attr='mutable_layer',
ctx=ast.Load())
target_call_args = [
ast.Str(s=mutable_id),
ast.Str(s=mutable_layer_id), call_funcs, call_kwargs
]
if fields['fixed_inputs']:
target_call_args.append(fixed_inputs)
else:
target_call_args.append(ast.NameConstant(value=None))
if fields['optional_inputs']:
target_call_args.append(optional_inputs)
assert fields[
'optional_input_size'], 'optional_input_size must exist when optional_inputs exists'
target_call_args.append(optional_input_size)
else:
target_call_args.append(ast.NameConstant(value=None))
target_call = ast.Call(func=target_call_attr,
args=target_call_args,
keywords=[])
node = ast.Assign(targets=[layer_output], value=target_call)
nodes.append(node)
return nodes
def getDispatch(self):
caller = ""
callee = ""
commNodeList = []
commProtocolList = []
currentCommFuncDict = {}
newIfList = []
hasIfStmt = False
for elem in calleeArr:
if elem[1] == self.className:
if currentCommFuncDict == {}:
callee = elem[1]
caller = elem[2]
currentCommFuncDict[caller] = [elem[0]]
else:
if elem[1] == callee and elem[2] == caller:
currentCommFuncDict.get(caller).append(elem[0])
else:
caller = elem[2]
if caller in currentCommFuncDict.keys():
currentCommFuncDict.get(caller).append(elem[0])
else:
currentCommFuncDict[caller] = [elem[0]]
currentCommFuncKeys = list(currentCommFuncDict.keys())
# 같은 통신 방법을 사용하는 클래스를 묶기 위한 과정
for f_index in range(0, len(currentCommFuncDict.keys()) - 1):
f_commu = commuTable.get(classArr.get(currentCommFuncKeys[f_index])).get(classArr.get(self.className))
for s_index in range(f_index + 1, len(currentCommFuncDict.keys())):
s_commu = commuTable.get(classArr.get(currentCommFuncKeys[s_index])).get(classArr.get(self.className))
# 현재 클래스와 첫번째 클래스, 두번째 클래스가 사용하는 통신 방법이 같은 경우
if f_commu == s_commu:
if currentCommFuncKeys[s_index] not in currentCommFuncDict.get(currentCommFuncKeys[f_index]):
sFuncList = currentCommFuncDict.get(currentCommFuncKeys[s_index])
for func in sFuncList:
currentCommFuncDict.get(currentCommFuncKeys[f_index]).append(func)
del currentCommFuncDict[currentCommFuncKeys[s_index]]
for currentCaller in currentCommFuncDict.keys():
funNum = -1
for locProcTup in allLocProcList:
if locProcTup[0] == self.className and locProcTup[1] in currentCommFuncDict.get(currentCaller):
hasIfStmt = True
funNum = locProcTup[2]
ifSource = "if funid == " + str(funNum) + ":\n"
ifSource += "\t" + locProcTup[1] + "()\n"
newIfList.append(ast.parse(ifSource))
commProtocol = commuTable.get(classArr.get(currentCaller)).get(classArr.get(self.className))
if commProtocol not in commProtocolList:
commNodeList.append(self.getCommuNode(self.className, currentCaller, newIfList))
commProtocolList.append(commProtocol)
newIfList = []
if hasIfStmt == False:
return []
n = 0
newFunctionList = []
for commNode in commNodeList:
newFunction = ast.FunctionDef()
newFunction.body = []
commProtocolIndex = commNodeList.index(commNode)
for comm in commNode:
newFunction.body.insert(n, comm)
n = n + 1
if classArr.get(self.className) == 'Arduino':
newFunction.name = 'dispatch_' + commProtocolList[commProtocolIndex]
newFunction.returns = ast.NameConstant(value = None)
self.dispatchName.append('dispatch_' + commProtocolList[commProtocolIndex])
else:
newFunction.name = 'dispatch_' + commProtocolList[commProtocolIndex]
newFunction.args =[]
newFunction.decorator_list = []
self.dispatch_flag = True
newFunctionList.append(newFunction)
if len(newFunctionList) > 1:
if "threading" not in replaceAST.importList:
replaceAST.importList.append("threading")
return newFunctionList
def test_NameConstant(self):
self.verify(ast.NameConstant(None), 'None')
self.verify(ast.NameConstant(True), 'True')
self.verify(ast.NameConstant(False), 'False')
