def _trace_assignment(self, target, default_lineno=None):
lineno = getattr(target, 'lineno', default_lineno)
# name, value, line number
if isinstance(target, Name):
arg_name = target.id
elif arg and isinstance(target, arg):
arg_name = target.arg
elif isinstance(target, Starred):
arg_name = target.value.id
elif isinstance(target, Attribute):
args = [
Str(s='{}.{}'.format(target.value.id, target.attr)),
Attribute(value=Name(id=target.value.id, ctx=Load()),
attr=target.attr,
ctx=Load()),
Num(n=lineno)
]
return self._create_context_call('assign', args)
elif isinstance(target, str):
arg_name = target
else:
message = 'Cannot trace assignment to a {}.'.format(type(target))
raise TypeError(message)
args = [Str(s=arg_name), Name(id=arg_name, ctx=Load()), Num(n=lineno)]
return self._create_context_call('assign', args)
def _trace_assignment(self, target, default_lineno=None):
lineno = getattr(target, 'lineno', default_lineno)
#name, value, line number
if isinstance(target, Name):
args = [Str(s=target.id),
Name(id=target.id, ctx=Load()),
Num(n=lineno)]
elif arg and isinstance(target, arg):
args=[Str(s=target.arg),
Name(id=target.arg, ctx=Load()),
Num(n=lineno)]
elif isinstance(target, Subscript):
return self._trace_assignment(target.value)
elif isinstance(target, Attribute):
names = self._get_attribute_names(target)
if names is None:
raise TypeError('Unknown target: %s' % target)
args = [Str(s='.'.join(names)),
Attribute(value=target.value,
attr=target.attr,
ctx=Load()),
Num(n=lineno)]
else:
raise TypeError('Unknown target: %s' % target)
return self._create_context_call('assign', args)
def _trace_assignment(self, target, default_lineno=None):
lineno = getattr(target, 'lineno', default_lineno)
#name, value, line number
if isinstance(target, Name):
args = [
Str(s=target.id),
Name(id=target.id, ctx=Load()),
Num(n=lineno)
]
elif arg and isinstance(target, arg):
args = [
Str(s=target.arg),
Name(id=target.arg, ctx=Load()),
Num(n=lineno)
]
elif isinstance(target, Subscript):
args = [
Str(s=self._get_subscript_repr(target)),
Subscript(value=target.value, slice=target.slice, ctx=Load()),
Num(n=lineno)
]
else:
raise TypeError('Unknown target: %s' % target)
return self._create_context_call('assign', args)
def visit_Lambda(self, node):
""" Instrument a lambda expression by displaying the parameter values.
We create calls to trace assignment to each argument, then wrap them
all in a tuple together with the original expression, and pull the
original expression out of the tuple.
"""
new_node = self.generic_visit(node)
line_numbers = set()
self._find_line_numbers(new_node, line_numbers)
# trace lambda argument values
calls = [getattr(self._trace_assignment(target, node.lineno),
'value',
None)
for target in new_node.args.args
if getattr(target, 'id', 'self') != 'self' or
getattr(target, 'arg', 'self') != 'self']
args = [Num(n=min(line_numbers)),
Num(n=max(line_numbers))]
calls.insert(0, self._create_context_call('start_block', args).value)
calls.append(new_node.body)
new_node.body = Subscript(value=Tuple(elts=calls,
ctx=Load()),
slice=Index(value=Num(n=-1)),
ctx=Load())
return new_node
def visit_While(self, node):
new_node = self.generic_visit(node)
line_numbers = set()
self._find_line_numbers(new_node, line_numbers)
args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))]
new_node.body.insert(0, self._create_context_call('start_block', args))
return new_node
def visit_FunctionDef(self, node):
""" Instrument a function definition by creating a new report builder
for this stack frame and putting it in a local variable. The local
variable has the same name as the global variable so all calls can
use the same CONTEXT_NAME symbol, but it means that I had to use this:
x = globals()['x'].start_frame()
Kind of ugly, but I think it was worth it to handle recursive calls.
"""
if node.name == '__repr__':
return node
new_node = self.generic_visit(node)
line_numbers = set()
self._find_line_numbers(new_node, line_numbers)
first_line_number = min(line_numbers)
last_line_number = max(line_numbers)
args = [Num(n=first_line_number),
Num(n=last_line_number)]
try_body = new_node.body
globals_call = Call(func=Name(id='globals', ctx=Load()),
args=[],
keywords=[],
starargs=None,
kwargs=None)
global_context = Subscript(value=globals_call,
slice=Index(value=Str(s=CONTEXT_NAME)),
ctx=Load())
start_frame_call = Call(func=Attribute(value=global_context,
attr='start_frame',
ctx=Load()),
args=args,
keywords=[],
starargs=None,
kwargs=None)
context_assign = Assign(targets=[Name(id=CONTEXT_NAME, ctx=Store())],
value=start_frame_call)
new_node.body = [context_assign]
# trace function parameter values
for target in new_node.args.args:
if isinstance(target, Name) and target.id == 'self':
continue
if arg and isinstance(target, arg) and target.arg == 'self':
continue
new_node.body.append(self._trace_assignment(target, node.lineno))
handler_body = [self._create_context_call('exception'),
Raise()]
new_node.body.append(
TryExcept(body=try_body,
handlers=[ExceptHandler(body=handler_body)],
orelse=[],
finalbody=[]))
self._set_statement_line_numbers(try_body, first_line_number)
self._set_statement_line_numbers(handler_body, last_line_number)
return new_node
def zero_Multiplier(node): # 0 -> int(n * 0)
return Call(func=Name(id='int', ctx=Load()),
args=[
BinOp(left=Num(n=random.random()),
right=Num(n=0),
op=Mult())
],
keywords=[],
starargs=None,
kwargs=None)
def test_ext_slice():
result = match_ast(pattern, parse('x[0, 1, 2]'))
actual = deslicify(result['slice'])
check = match_ast(
Tuple(
elts=[Num(n=set(['n1'])),
Num(n=set(['n2'])),
Num(n=set(['n3']))]), actual)
assert check == {'n1': 0, 'n2': 1, 'n3': 2}
def visit_For(self, node):
new_node = self.generic_visit(node)
line_numbers = set()
self._find_line_numbers(new_node, line_numbers)
args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))]
new_body = [self._create_context_call('start_block', args)]
new_body.extend(self._trace_assignment_list(new_node.target))
new_body.extend(new_node.body)
new_node.body = new_body
return new_node
def visit_While(self, node):
new_node = self.generic_visit(node)
# Collect line numbers from all except else block.
line_numbers = set()
find_line_numbers(new_node.test, line_numbers)
for statement in new_node.body:
find_line_numbers(statement, line_numbers)
line_numbers.add(new_node.lineno)
args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))]
new_node.body.insert(0, self._create_context_call('start_block', args))
return new_node
def compile_factory(fff: FlatFunctionFactory):
arguments = [*fff.arguments.keys()]
funname = fff.funname
unpacking = []
for i, (arg_group_name, arg_group) in enumerate(fff.arguments.items()):
for pos, sym in enumerate(arg_group):
rhs = Subscript(value=Name(id=arg_group_name, ctx=Load()),
slice=Index(Num(pos)),
ctx=Load())
val = Assign(targets=[Name(id=sym, ctx=Store())], value=rhs)
unpacking.append(val)
body = []
for (k, neq) in fff.preamble.items():
val = parse_string(neq).value
line = Assign(targets=[Name(id=k, ctx=Store())], value=val)
body.append(line)
for n, (k, neq) in enumerate(fff.content.items()):
# should the result of parse_string always of type Expr ?
val = parse_string(neq).value
line = Assign(targets=[Name(id=k, ctx=Store())], value=val)
body.append(line)
#
for n, (lhs, neq) in enumerate(fff.content.items()):
line = Assign(targets=[
Subscript(value=Name(id='out', ctx=Load()),
slice=Index(Num(n)),
ctx=Store())
],
value=Name(id=lhs, ctx=Load()))
body.append(line)
f = FunctionDef(name=funname,
args=ast_arguments(args=[arg(arg=a) for a in arguments] +
[arg(arg='out')],
vararg=None,
kwarg=None,
kwonlyargs=[],
kw_defaults=[],
defaults=[]),
body=unpacking + body,
decorator_list=[])
mod = Module(body=[f])
mmod = ast.fix_missing_locations(mod)
return mmod
def visit_Assign(self, node):
existing_node = self.generic_visit(node)
try:
targets = existing_node.targets
except AttributeError:
targets = [existing_node.target]
if any(map(self._is_untraceable_attribute, targets)):
return existing_node
if existing_node.value is None:
return existing_node
line_numbers = set()
find_line_numbers(existing_node, line_numbers)
first_line_number = min(line_numbers)
last_line_number = max(line_numbers)
if len(targets) == 1 and isinstance(targets[0], Name):
existing_node.value = self._create_bare_context_call(
'assign', [
Str(s=targets[0].id), existing_node.value,
Num(n=first_line_number)
])
return existing_node
new_nodes = []
format_string = self._wrap_assignment_targets(targets)
if (len(targets) == 1 and isinstance(targets[0], Tuple)):
existing_node.value = Call(func=Name(id='tuple', ctx=Load()),
args=[existing_node.value],
keywords=[],
starargs=None,
kwargs=None)
existing_node.value = self._create_bare_context_call(
'set_assignment_value', [existing_node.value])
new_nodes.append(self._create_context_call('start_assignment'))
try_body = [existing_node]
if format_string is not None:
try_body.append(
self._create_context_call(
'report_assignment',
[Str(s=format_string),
Num(n=existing_node.lineno)]))
end_assignment = self._create_context_call('end_assignment')
finally_body = [end_assignment]
new_nodes.append(
Try(body=try_body,
finalbody=finally_body,
handlers=[],
orelse=[],
lineno=first_line_number))
self._set_statement_line_numbers(try_body, first_line_number)
self._set_statement_line_numbers(finally_body, last_line_number)
return new_nodes
def visit_Lambda(self, node):
new_node = self.generic_visit(node)
line_numbers = set()
self._find_line_numbers(new_node, line_numbers)
arg_names = (getattr(old_arg, 'id', getattr(old_arg, 'arg', None))
for old_arg in new_node.args.args)
new_args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))]
new_args.extend(Name(id=name, ctx=Load()) for name in arg_names)
new_args.append(new_node.body)
new_node.body = self._create_bare_context_call('report_lambda',
new_args)
return new_node
def visit_With(self, node):
new_node = self.generic_visit(node)
# Collect line numbers from all except else block.
line_numbers = set()
for item in new_node.items:
find_line_numbers(item, line_numbers)
for statement in new_node.body:
find_line_numbers(statement, line_numbers)
line_numbers.add(new_node.lineno)
args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))]
new_body = [self._create_context_call('start_block', args)]
new_body.extend(new_node.body)
new_node.body = new_body
return new_node
class ImportBombast(RenameBombast):
# import sys -> sys = __import__('sys', globals(), locals(), [], 0)
one = Transformation(lambda n: Assign(
targets=[Name(id=n.names[0].name, ctx=Store())],
value=Call(func=Name(id='__import__', ctx=Load()),
args=[
Str(s=n.names[0].name),
Call(func=Name(id='globals', ctx=Load()),
args=[],
keywords=[],
starargs=None,
kwargs=None),
Call(func=Name(id='locals', ctx=Load()),
args=[],
keywords=[],
starargs=None,
kwargs=None),
List(elts=[], ctx=Load()),
Num(n=0)
],
keywords=[],
starargs=None,
kwargs=None)))
def transform(self):
num_imports = len(self.node.names)
if num_imports == 1:
return self.one.transform(self.node)
else:
return self.node
def visit_Match(self, node):
match_node = self.generic_visit(node)
line_numbers = set()
find_line_numbers(match_node, line_numbers)
first_line_number = min(line_numbers)
new_nodes = [self._create_context_call('start_assignment')]
try_body = [match_node]
finally_body = [self._create_context_call('end_assignment')]
new_nodes.append(
Try(body=try_body,
finalbody=finally_body,
handlers=[],
orelse=[],
lineno=first_line_number))
match_node.subject = self._create_bare_context_call(
'set_assignment_value', [match_node.subject])
for case in match_node.cases:
try:
format_string = self._wrap_assignment_targets((case.pattern, ))
except ValueError:
# Not a capture pattern, nothing to report.
format_string = None
if format_string is not None:
case.body.insert(
0,
self._create_context_call(
'report_assignment',
[Str(s=format_string),
Num(n=case.pattern.lineno)]))
def visit_AugAssign(self, node):
read_target = deepcopy(node.target)
existing_node = self.generic_visit(node)
line_numbers = set()
find_line_numbers(existing_node, line_numbers)
first_line_number = min(line_numbers)
last_line_number = max(line_numbers)
new_nodes = []
try_body = [existing_node]
new_nodes.append(self._create_context_call('start_assignment'))
format_string = self._wrap_assignment_target(existing_node.target)
if format_string is not None:
if ':' in format_string:
existing_node.value = self._create_bare_context_call(
'set_assignment_value', [existing_node.value])
operator_char = OPERATOR_CHARS.get(type(existing_node.op), '?')
format_string += ' {}= {{}} '.format(operator_char)
else:
self._wrap_assignment_target(read_target, index_to_get=-1)
read_target.ctx = Load()
set_assignment_value = self._create_context_call(
'set_assignment_value', [read_target])
try_body.append(set_assignment_value)
format_string += ' = {}'
try_body.append(
self._create_context_call(
'report_assignment',
[Str(s=format_string),
Num(n=existing_node.lineno)]))
self._create_end_assignment(new_nodes, try_body, first_line_number,
last_line_number)
return new_nodes
def visit_Assign(self, node):
existing_node = self.generic_visit(node)
if any(map(self._is_untraceable_attribute, existing_node.targets)):
return existing_node
line_numbers = set()
self._find_line_numbers(existing_node, line_numbers)
first_line_number = min(line_numbers)
last_line_number = max(line_numbers)
new_nodes = []
format_string = self._wrap_assignment_targets(
existing_node.targets)
existing_node.value = self._create_bare_context_call(
'set_assignment_value',
[existing_node.value])
new_nodes.append(self._create_context_call('start_assignment'))
try_body = [existing_node]
if format_string is not None:
try_body.append(self._create_context_call(
'report_assignment',
[Str(s=format_string), Num(n=existing_node.lineno)]))
end_assignment = self._create_context_call('end_assignment')
finally_body = [end_assignment]
new_nodes.append(TryFinally(body=try_body,
finalbody=finally_body,
handlers=[],
orelse=[],
lineno=first_line_number))
self._set_statement_line_numbers(try_body, first_line_number)
self._set_statement_line_numbers(finally_body, last_line_number)
return new_nodes
def set_value(value, kind=None):
"""
Creates a Constant or a Str depending on Python version in use
:param value: AST node
:type value: ```Any```
:param kind: AST node
:type kind: ```Optional[Any]```
:return: Probably a string, but could be any constant value
:rtype: ```Union[Constant, Str, NameConstant]```
"""
if (
value is not None
and isinstance(value, str)
and len(value) > 2
and value[0] + value[-1] in frozenset(('""', "''"))
):
value = value[1:-1]
return (
Constant(kind=kind, value=value, constant_value=None, string=None)
if PY_GTE_3_8
else (
Str(s=value, constant_value=None, string=None)
if isinstance(value, str)
else Num(n=value, constant_value=None, string=None)
if not isinstance(value, bool) and isinstance(value, (int, float, complex))
else NameConstant(value=value, constant_value=None, string=None)
)
)
def visit_YieldFrom(self, node):
existing_node = self.generic_visit(node)
value = existing_node.value
existing_node.value = self._create_bare_context_call(
'yield_from',
[value, Num(n=existing_node.lineno)])
return existing_node
def visit_Call(self, node):
existing_node = self.generic_visit(node)
value_node = existing_node.func
if isinstance(value_node, Name) and value_node.id == 'print':
return self._trace_print_function(existing_node)
names = self._get_attribute_names(value_node)
if names is None:
return existing_node
args = [
Str(s='.'.join(names[:-1])),
Call(func=Name(id='repr', ctx=Load()),
args=[existing_node.func.value],
keywords=[],
starargs=None,
kwargs=None), existing_node,
Call(func=Name(id='repr', ctx=Load()),
args=[existing_node.func.value],
keywords=[],
starargs=None,
kwargs=None),
Num(n=existing_node.lineno)
]
new_node = self._create_bare_context_call('record_call', args)
return new_node
def visit_AugAssign(self, node):
read_target = deepcopy(node.target)
existing_node = self.generic_visit(node)
line_numbers = set()
self._find_line_numbers(existing_node, line_numbers)
first_line_number = min(line_numbers)
last_line_number = max(line_numbers)
new_nodes = []
format_string = (self._wrap_assignment_target(existing_node.target) +
' = {!r}')
new_nodes.append(self._create_context_call('start_assignment'))
self._wrap_assignment_target(read_target, index_to_get=-1)
read_target.ctx = Load()
set_assignment_value = self._create_context_call(
'set_assignment_value',
[read_target])
report_assignment = self._create_context_call(
'report_assignment',
[Str(s=format_string), Num(n=existing_node.lineno)])
end_assignment = self._create_context_call('end_assignment')
try_body = [existing_node, set_assignment_value, report_assignment]
finally_body = [end_assignment]
new_nodes.append(TryFinally(body=try_body,
finalbody=finally_body,
handlers=[],
orelse=[],
lineno=first_line_number))
self._set_statement_line_numbers(try_body, first_line_number)
self._set_statement_line_numbers(finally_body, last_line_number)
return new_nodes
def convert_output(ret_type):
"""Use the function return type to produce auxiliary variables to store outputs.
Returns ([assignments of output vars], [extra arguments to pass to op call],
expression collecting output)"""
if isinstance(ret_type, relay.TensorType):
output_var_name = self.generate_var_name('_out')
output_var = Name(output_var_name, Load())
shape = ast.Tuple(
[Num(dim) for dim in ret_type.concrete_shape], Load())
# create a new TensorValue of the right shape and dtype
assign_output = Assign(
[Name(output_var_name, Store())],
self.create_call('TensorValue', [
self.create_call('numpy.empty',
[shape, Str(ret_type.dtype)])
]))
# we pass the data field as an argument
extra_arg = ast.Attribute(output_var, 'data', Load())
return ([assign_output], [extra_arg], output_var)
assert isinstance(ret_type, relay.TupleType)
assignments = []
extra_args = []
fields = []
for t in ret_type.fields:
inner_assignments, inner_args, inner_output = convert_output(t)
assignments += inner_assignments
extra_args += inner_args
fields.append(inner_output)
return (assignments, extra_args,
self.create_call('TupleValue', fields))
def collect_var_assignments(pat, val):
"""This helper function ensures that the pattern is used to
properly assign all subfields of the given AST for use
in the clause body
E.g., for PatternConstructor(A, PatternVar(v), PatternWildcard(),
PatternConstructor(B, PatternVar(w)))
we would want to have
v = a.fields[0]
w = a.fields[2].fields[0]
"""
if isinstance(pat, relay.PatternWildcard):
return []
if isinstance(pat, relay.PatternVar):
return [Assign([self.include_var(pat.var, assign=True)], val)]
# constructor pattern: assign each field of the value
# based on subpatterns
assignments = []
for i in range(len(pat.patterns)):
# we want the assignments for val.fields[i]
field = ast.Subscript(
ast.Attribute(val, "fields", Load()), ast.Index(Num(i)), Load()
)
assignments += collect_var_assignments(pat.patterns[i], field)
return assignments
def convert_output(ret_type):
"""Use the function return type to produce auxiliary variables to store outputs.
Returns ([assignments of output vars], [extra arguments to pass to op call],
expression collecting output)"""
if isinstance(ret_type, relay.TensorType):
output_var_name = self.generate_var_name("_out")
output_var = Name(output_var_name, Load())
shape = ast.Tuple(
[Num(dim) for dim in ret_type.concrete_shape], Load())
# create a new NDArray of the right shape and dtype
assign_output = Assign(
[Name(output_var_name, Store())],
self.create_call("nd.array", [
self.create_call("numpy.empty",
[shape, Str(ret_type.dtype)])
]),
)
return ([assign_output], [output_var], output_var)
assert isinstance(ret_type, relay.TupleType)
assignments = []
extra_args = []
fields = []
for t in ret_type.fields:
inner_assignments, inner_args, inner_output = convert_output(t)
assignments += inner_assignments
extra_args += inner_args
fields.append(inner_output)
fields = [ast.List(fields, Load())]
return (assignments, extra_args,
self.create_call("_container.tuple_object", fields))
def visit_Call(self, node):
existing_node = self.generic_visit(node)
func_node = existing_node.func
if self._is_untraceable_attribute(func_node):
return existing_node
comparisons = [] # [(name, node)]
names = self._get_attribute_names(func_node)
if names is not None:
comparisons.append(
('.'.join(names[:-1]), existing_node.func.value))
for arg_node in existing_node.args:
if isinstance(arg_node, Name):
comparisons.append((arg_node.id, arg_node))
if not comparisons:
return existing_node
args = [
List(elts=[], ctx=Load()),
List(elts=[], ctx=Load()), existing_node,
List(elts=[], ctx=Load()),
Num(n=existing_node.lineno)
]
for name, node in comparisons:
args[0].elts.append(Str(s=name)) # name
args[1].elts.append( # repr() before
self._create_bare_context_call('get_repr', [node]))
args[3].elts.append( # repr() after
self._create_bare_context_call('get_repr', [node]))
new_node = self._create_bare_context_call('record_call', args)
return new_node
def visit_Call(self, node):
"""Wrap require statements with lambdas and unpack any argument packages."""
func = node.func
if isinstance(
func,
Name) and func.id == requireStatement: # Require statement
# Soft reqs have 2 arguments, including the probability, which is given as the
# first argument by the token translator; so we allow an extra argument here and
# validate it later on (in case the user wrongly gives 2 arguments to require).
if not (1 <= len(node.args) <= 2):
raise self.parseError(node,
'require takes exactly one argument')
if len(node.keywords) != 0:
raise self.parseError(node,
'require takes no keyword arguments')
cond = node.args[-1]
if isinstance(cond, Starred):
raise self.parseError(
node, 'argument unpacking cannot be used with require')
req = self.visit(cond)
line = self.lineMap[node.lineno]
reqID = Num(len(self.requirements)) # save ID number
self.requirements.append(
req) # save condition for later inspection when pruning
closure = Lambda(noArgs, req) # enclose requirement in a lambda
lineNum = Num(line) # save line number for error messages
copy_location(closure, req)
copy_location(lineNum, req)
newArgs = [reqID, closure, lineNum]
if len(node.args) == 2: # get probability for soft requirements
prob = node.args[0]
if not isinstance(prob, Num):
raise self.parseError(
node, 'malformed requirement '
'(should be a single expression)')
newArgs.append(prob)
return copy_location(Call(func, newArgs, []), node)
else: # Ordinary function call
newArgs = []
# Translate arguments, unpacking any argument packages
for arg in node.args:
if isinstance(arg, BinOp) and isinstance(arg.op, packageNode):
newArgs.extend(self.unpack(arg, 2, node))
else:
newArgs.append(self.visit(arg))
newKeywords = [self.visit(kwarg) for kwarg in node.keywords]
return copy_location(Call(func, newArgs, newKeywords), node)
def visit_Yield(self, node):
existing_node = self.generic_visit(node)
value = existing_node.value
if value is None:
value = Name(id='None', ctx=Load())
return Yield(value=self._create_bare_context_call(
'yield_value', [value, Num(n=existing_node.lineno)]))
def test_selector_standalone():
from ast import Expr, Num # use python's builtin ast library
Expr._priority = 0
Num._priority = 1
node = Expr(value=Num(n=1))
sel = Selector(Num)
sel.visit(node)
assert isinstance(sel.out[0], Num)
def visit_For(self, node):
new_node = self.generic_visit(node)
# Collect line numbers from all except else block.
line_numbers = set()
self._find_line_numbers(new_node.target, line_numbers)
self._find_line_numbers(new_node.iter, line_numbers)
for statement in new_node.body:
self._find_line_numbers(statement, line_numbers)
line_numbers.add(new_node.lineno)
args = [Num(n=min(line_numbers)),
Num(n=max(line_numbers))]
new_body = [self._create_context_call('start_block', args)]
new_body.extend(self._trace_assignment_list(new_node.target))
new_body.extend(new_node.body)
new_node.body = new_body
return new_node
def __init__(self, number):
int.__init__(self, number)
Num.__init__(self, n=number)
NesType.__init__(self)
