def _trace_assignment(self, target, default_lineno=None): lineno = getattr(target, 'lineno', default_lineno) # name, value, line number if isinstance(target, Name): arg_name = target.id elif arg and isinstance(target, arg): arg_name = target.arg elif isinstance(target, Starred): arg_name = target.value.id elif isinstance(target, Attribute): args = [ Str(s='{}.{}'.format(target.value.id, target.attr)), Attribute(value=Name(id=target.value.id, ctx=Load()), attr=target.attr, ctx=Load()), Num(n=lineno) ] return self._create_context_call('assign', args) elif isinstance(target, str): arg_name = target else: message = 'Cannot trace assignment to a {}.'.format(type(target)) raise TypeError(message) args = [Str(s=arg_name), Name(id=arg_name, ctx=Load()), Num(n=lineno)] return self._create_context_call('assign', args)
def _trace_assignment(self, target, default_lineno=None): lineno = getattr(target, 'lineno', default_lineno) #name, value, line number if isinstance(target, Name): args = [Str(s=target.id), Name(id=target.id, ctx=Load()), Num(n=lineno)] elif arg and isinstance(target, arg): args=[Str(s=target.arg), Name(id=target.arg, ctx=Load()), Num(n=lineno)] elif isinstance(target, Subscript): return self._trace_assignment(target.value) elif isinstance(target, Attribute): names = self._get_attribute_names(target) if names is None: raise TypeError('Unknown target: %s' % target) args = [Str(s='.'.join(names)), Attribute(value=target.value, attr=target.attr, ctx=Load()), Num(n=lineno)] else: raise TypeError('Unknown target: %s' % target) return self._create_context_call('assign', args)
def _trace_assignment(self, target, default_lineno=None): lineno = getattr(target, 'lineno', default_lineno) #name, value, line number if isinstance(target, Name): args = [ Str(s=target.id), Name(id=target.id, ctx=Load()), Num(n=lineno) ] elif arg and isinstance(target, arg): args = [ Str(s=target.arg), Name(id=target.arg, ctx=Load()), Num(n=lineno) ] elif isinstance(target, Subscript): args = [ Str(s=self._get_subscript_repr(target)), Subscript(value=target.value, slice=target.slice, ctx=Load()), Num(n=lineno) ] else: raise TypeError('Unknown target: %s' % target) return self._create_context_call('assign', args)
def visit_Lambda(self, node): """ Instrument a lambda expression by displaying the parameter values. We create calls to trace assignment to each argument, then wrap them all in a tuple together with the original expression, and pull the original expression out of the tuple. """ new_node = self.generic_visit(node) line_numbers = set() self._find_line_numbers(new_node, line_numbers) # trace lambda argument values calls = [getattr(self._trace_assignment(target, node.lineno), 'value', None) for target in new_node.args.args if getattr(target, 'id', 'self') != 'self' or getattr(target, 'arg', 'self') != 'self'] args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] calls.insert(0, self._create_context_call('start_block', args).value) calls.append(new_node.body) new_node.body = Subscript(value=Tuple(elts=calls, ctx=Load()), slice=Index(value=Num(n=-1)), ctx=Load()) return new_node
def visit_While(self, node): new_node = self.generic_visit(node) line_numbers = set() self._find_line_numbers(new_node, line_numbers) args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] new_node.body.insert(0, self._create_context_call('start_block', args)) return new_node
def visit_FunctionDef(self, node): """ Instrument a function definition by creating a new report builder for this stack frame and putting it in a local variable. The local variable has the same name as the global variable so all calls can use the same CONTEXT_NAME symbol, but it means that I had to use this: x = globals()['x'].start_frame() Kind of ugly, but I think it was worth it to handle recursive calls. """ if node.name == '__repr__': return node new_node = self.generic_visit(node) line_numbers = set() self._find_line_numbers(new_node, line_numbers) first_line_number = min(line_numbers) last_line_number = max(line_numbers) args = [Num(n=first_line_number), Num(n=last_line_number)] try_body = new_node.body globals_call = Call(func=Name(id='globals', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None) global_context = Subscript(value=globals_call, slice=Index(value=Str(s=CONTEXT_NAME)), ctx=Load()) start_frame_call = Call(func=Attribute(value=global_context, attr='start_frame', ctx=Load()), args=args, keywords=[], starargs=None, kwargs=None) context_assign = Assign(targets=[Name(id=CONTEXT_NAME, ctx=Store())], value=start_frame_call) new_node.body = [context_assign] # trace function parameter values for target in new_node.args.args: if isinstance(target, Name) and target.id == 'self': continue if arg and isinstance(target, arg) and target.arg == 'self': continue new_node.body.append(self._trace_assignment(target, node.lineno)) handler_body = [self._create_context_call('exception'), Raise()] new_node.body.append( TryExcept(body=try_body, handlers=[ExceptHandler(body=handler_body)], orelse=[], finalbody=[])) self._set_statement_line_numbers(try_body, first_line_number) self._set_statement_line_numbers(handler_body, last_line_number) return new_node
def zero_Multiplier(node): # 0 -> int(n * 0) return Call(func=Name(id='int', ctx=Load()), args=[ BinOp(left=Num(n=random.random()), right=Num(n=0), op=Mult()) ], keywords=[], starargs=None, kwargs=None)
def test_ext_slice(): result = match_ast(pattern, parse('x[0, 1, 2]')) actual = deslicify(result['slice']) check = match_ast( Tuple( elts=[Num(n=set(['n1'])), Num(n=set(['n2'])), Num(n=set(['n3']))]), actual) assert check == {'n1': 0, 'n2': 1, 'n3': 2}
def visit_For(self, node): new_node = self.generic_visit(node) line_numbers = set() self._find_line_numbers(new_node, line_numbers) args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] new_body = [self._create_context_call('start_block', args)] new_body.extend(self._trace_assignment_list(new_node.target)) new_body.extend(new_node.body) new_node.body = new_body return new_node
def visit_While(self, node): new_node = self.generic_visit(node) # Collect line numbers from all except else block. line_numbers = set() find_line_numbers(new_node.test, line_numbers) for statement in new_node.body: find_line_numbers(statement, line_numbers) line_numbers.add(new_node.lineno) args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] new_node.body.insert(0, self._create_context_call('start_block', args)) return new_node
def compile_factory(fff: FlatFunctionFactory): arguments = [*fff.arguments.keys()] funname = fff.funname unpacking = [] for i, (arg_group_name, arg_group) in enumerate(fff.arguments.items()): for pos, sym in enumerate(arg_group): rhs = Subscript(value=Name(id=arg_group_name, ctx=Load()), slice=Index(Num(pos)), ctx=Load()) val = Assign(targets=[Name(id=sym, ctx=Store())], value=rhs) unpacking.append(val) body = [] for (k, neq) in fff.preamble.items(): val = parse_string(neq).value line = Assign(targets=[Name(id=k, ctx=Store())], value=val) body.append(line) for n, (k, neq) in enumerate(fff.content.items()): # should the result of parse_string always of type Expr ? val = parse_string(neq).value line = Assign(targets=[Name(id=k, ctx=Store())], value=val) body.append(line) # for n, (lhs, neq) in enumerate(fff.content.items()): line = Assign(targets=[ Subscript(value=Name(id='out', ctx=Load()), slice=Index(Num(n)), ctx=Store()) ], value=Name(id=lhs, ctx=Load())) body.append(line) f = FunctionDef(name=funname, args=ast_arguments(args=[arg(arg=a) for a in arguments] + [arg(arg='out')], vararg=None, kwarg=None, kwonlyargs=[], kw_defaults=[], defaults=[]), body=unpacking + body, decorator_list=[]) mod = Module(body=[f]) mmod = ast.fix_missing_locations(mod) return mmod
def visit_Assign(self, node): existing_node = self.generic_visit(node) try: targets = existing_node.targets except AttributeError: targets = [existing_node.target] if any(map(self._is_untraceable_attribute, targets)): return existing_node if existing_node.value is None: return existing_node line_numbers = set() find_line_numbers(existing_node, line_numbers) first_line_number = min(line_numbers) last_line_number = max(line_numbers) if len(targets) == 1 and isinstance(targets[0], Name): existing_node.value = self._create_bare_context_call( 'assign', [ Str(s=targets[0].id), existing_node.value, Num(n=first_line_number) ]) return existing_node new_nodes = [] format_string = self._wrap_assignment_targets(targets) if (len(targets) == 1 and isinstance(targets[0], Tuple)): existing_node.value = Call(func=Name(id='tuple', ctx=Load()), args=[existing_node.value], keywords=[], starargs=None, kwargs=None) existing_node.value = self._create_bare_context_call( 'set_assignment_value', [existing_node.value]) new_nodes.append(self._create_context_call('start_assignment')) try_body = [existing_node] if format_string is not None: try_body.append( self._create_context_call( 'report_assignment', [Str(s=format_string), Num(n=existing_node.lineno)])) end_assignment = self._create_context_call('end_assignment') finally_body = [end_assignment] new_nodes.append( Try(body=try_body, finalbody=finally_body, handlers=[], orelse=[], lineno=first_line_number)) self._set_statement_line_numbers(try_body, first_line_number) self._set_statement_line_numbers(finally_body, last_line_number) return new_nodes
def visit_Lambda(self, node): new_node = self.generic_visit(node) line_numbers = set() self._find_line_numbers(new_node, line_numbers) arg_names = (getattr(old_arg, 'id', getattr(old_arg, 'arg', None)) for old_arg in new_node.args.args) new_args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] new_args.extend(Name(id=name, ctx=Load()) for name in arg_names) new_args.append(new_node.body) new_node.body = self._create_bare_context_call('report_lambda', new_args) return new_node
def visit_With(self, node): new_node = self.generic_visit(node) # Collect line numbers from all except else block. line_numbers = set() for item in new_node.items: find_line_numbers(item, line_numbers) for statement in new_node.body: find_line_numbers(statement, line_numbers) line_numbers.add(new_node.lineno) args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] new_body = [self._create_context_call('start_block', args)] new_body.extend(new_node.body) new_node.body = new_body return new_node
class ImportBombast(RenameBombast): # import sys -> sys = __import__('sys', globals(), locals(), [], 0) one = Transformation(lambda n: Assign( targets=[Name(id=n.names[0].name, ctx=Store())], value=Call(func=Name(id='__import__', ctx=Load()), args=[ Str(s=n.names[0].name), Call(func=Name(id='globals', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='locals', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), List(elts=[], ctx=Load()), Num(n=0) ], keywords=[], starargs=None, kwargs=None))) def transform(self): num_imports = len(self.node.names) if num_imports == 1: return self.one.transform(self.node) else: return self.node
def visit_Match(self, node): match_node = self.generic_visit(node) line_numbers = set() find_line_numbers(match_node, line_numbers) first_line_number = min(line_numbers) new_nodes = [self._create_context_call('start_assignment')] try_body = [match_node] finally_body = [self._create_context_call('end_assignment')] new_nodes.append( Try(body=try_body, finalbody=finally_body, handlers=[], orelse=[], lineno=first_line_number)) match_node.subject = self._create_bare_context_call( 'set_assignment_value', [match_node.subject]) for case in match_node.cases: try: format_string = self._wrap_assignment_targets((case.pattern, )) except ValueError: # Not a capture pattern, nothing to report. format_string = None if format_string is not None: case.body.insert( 0, self._create_context_call( 'report_assignment', [Str(s=format_string), Num(n=case.pattern.lineno)]))
def visit_AugAssign(self, node): read_target = deepcopy(node.target) existing_node = self.generic_visit(node) line_numbers = set() find_line_numbers(existing_node, line_numbers) first_line_number = min(line_numbers) last_line_number = max(line_numbers) new_nodes = [] try_body = [existing_node] new_nodes.append(self._create_context_call('start_assignment')) format_string = self._wrap_assignment_target(existing_node.target) if format_string is not None: if ':' in format_string: existing_node.value = self._create_bare_context_call( 'set_assignment_value', [existing_node.value]) operator_char = OPERATOR_CHARS.get(type(existing_node.op), '?') format_string += ' {}= {{}} '.format(operator_char) else: self._wrap_assignment_target(read_target, index_to_get=-1) read_target.ctx = Load() set_assignment_value = self._create_context_call( 'set_assignment_value', [read_target]) try_body.append(set_assignment_value) format_string += ' = {}' try_body.append( self._create_context_call( 'report_assignment', [Str(s=format_string), Num(n=existing_node.lineno)])) self._create_end_assignment(new_nodes, try_body, first_line_number, last_line_number) return new_nodes
def visit_Assign(self, node): existing_node = self.generic_visit(node) if any(map(self._is_untraceable_attribute, existing_node.targets)): return existing_node line_numbers = set() self._find_line_numbers(existing_node, line_numbers) first_line_number = min(line_numbers) last_line_number = max(line_numbers) new_nodes = [] format_string = self._wrap_assignment_targets( existing_node.targets) existing_node.value = self._create_bare_context_call( 'set_assignment_value', [existing_node.value]) new_nodes.append(self._create_context_call('start_assignment')) try_body = [existing_node] if format_string is not None: try_body.append(self._create_context_call( 'report_assignment', [Str(s=format_string), Num(n=existing_node.lineno)])) end_assignment = self._create_context_call('end_assignment') finally_body = [end_assignment] new_nodes.append(TryFinally(body=try_body, finalbody=finally_body, handlers=[], orelse=[], lineno=first_line_number)) self._set_statement_line_numbers(try_body, first_line_number) self._set_statement_line_numbers(finally_body, last_line_number) return new_nodes
def set_value(value, kind=None): """ Creates a Constant or a Str depending on Python version in use :param value: AST node :type value: ```Any``` :param kind: AST node :type kind: ```Optional[Any]``` :return: Probably a string, but could be any constant value :rtype: ```Union[Constant, Str, NameConstant]``` """ if ( value is not None and isinstance(value, str) and len(value) > 2 and value[0] + value[-1] in frozenset(('""', "''")) ): value = value[1:-1] return ( Constant(kind=kind, value=value, constant_value=None, string=None) if PY_GTE_3_8 else ( Str(s=value, constant_value=None, string=None) if isinstance(value, str) else Num(n=value, constant_value=None, string=None) if not isinstance(value, bool) and isinstance(value, (int, float, complex)) else NameConstant(value=value, constant_value=None, string=None) ) )
def visit_YieldFrom(self, node): existing_node = self.generic_visit(node) value = existing_node.value existing_node.value = self._create_bare_context_call( 'yield_from', [value, Num(n=existing_node.lineno)]) return existing_node
def visit_Call(self, node): existing_node = self.generic_visit(node) value_node = existing_node.func if isinstance(value_node, Name) and value_node.id == 'print': return self._trace_print_function(existing_node) names = self._get_attribute_names(value_node) if names is None: return existing_node args = [ Str(s='.'.join(names[:-1])), Call(func=Name(id='repr', ctx=Load()), args=[existing_node.func.value], keywords=[], starargs=None, kwargs=None), existing_node, Call(func=Name(id='repr', ctx=Load()), args=[existing_node.func.value], keywords=[], starargs=None, kwargs=None), Num(n=existing_node.lineno) ] new_node = self._create_bare_context_call('record_call', args) return new_node
def visit_AugAssign(self, node): read_target = deepcopy(node.target) existing_node = self.generic_visit(node) line_numbers = set() self._find_line_numbers(existing_node, line_numbers) first_line_number = min(line_numbers) last_line_number = max(line_numbers) new_nodes = [] format_string = (self._wrap_assignment_target(existing_node.target) + ' = {!r}') new_nodes.append(self._create_context_call('start_assignment')) self._wrap_assignment_target(read_target, index_to_get=-1) read_target.ctx = Load() set_assignment_value = self._create_context_call( 'set_assignment_value', [read_target]) report_assignment = self._create_context_call( 'report_assignment', [Str(s=format_string), Num(n=existing_node.lineno)]) end_assignment = self._create_context_call('end_assignment') try_body = [existing_node, set_assignment_value, report_assignment] finally_body = [end_assignment] new_nodes.append(TryFinally(body=try_body, finalbody=finally_body, handlers=[], orelse=[], lineno=first_line_number)) self._set_statement_line_numbers(try_body, first_line_number) self._set_statement_line_numbers(finally_body, last_line_number) return new_nodes
def convert_output(ret_type): """Use the function return type to produce auxiliary variables to store outputs. Returns ([assignments of output vars], [extra arguments to pass to op call], expression collecting output)""" if isinstance(ret_type, relay.TensorType): output_var_name = self.generate_var_name('_out') output_var = Name(output_var_name, Load()) shape = ast.Tuple( [Num(dim) for dim in ret_type.concrete_shape], Load()) # create a new TensorValue of the right shape and dtype assign_output = Assign( [Name(output_var_name, Store())], self.create_call('TensorValue', [ self.create_call('numpy.empty', [shape, Str(ret_type.dtype)]) ])) # we pass the data field as an argument extra_arg = ast.Attribute(output_var, 'data', Load()) return ([assign_output], [extra_arg], output_var) assert isinstance(ret_type, relay.TupleType) assignments = [] extra_args = [] fields = [] for t in ret_type.fields: inner_assignments, inner_args, inner_output = convert_output(t) assignments += inner_assignments extra_args += inner_args fields.append(inner_output) return (assignments, extra_args, self.create_call('TupleValue', fields))
def collect_var_assignments(pat, val): """This helper function ensures that the pattern is used to properly assign all subfields of the given AST for use in the clause body E.g., for PatternConstructor(A, PatternVar(v), PatternWildcard(), PatternConstructor(B, PatternVar(w))) we would want to have v = a.fields[0] w = a.fields[2].fields[0] """ if isinstance(pat, relay.PatternWildcard): return [] if isinstance(pat, relay.PatternVar): return [Assign([self.include_var(pat.var, assign=True)], val)] # constructor pattern: assign each field of the value # based on subpatterns assignments = [] for i in range(len(pat.patterns)): # we want the assignments for val.fields[i] field = ast.Subscript( ast.Attribute(val, "fields", Load()), ast.Index(Num(i)), Load() ) assignments += collect_var_assignments(pat.patterns[i], field) return assignments
def convert_output(ret_type): """Use the function return type to produce auxiliary variables to store outputs. Returns ([assignments of output vars], [extra arguments to pass to op call], expression collecting output)""" if isinstance(ret_type, relay.TensorType): output_var_name = self.generate_var_name("_out") output_var = Name(output_var_name, Load()) shape = ast.Tuple( [Num(dim) for dim in ret_type.concrete_shape], Load()) # create a new NDArray of the right shape and dtype assign_output = Assign( [Name(output_var_name, Store())], self.create_call("nd.array", [ self.create_call("numpy.empty", [shape, Str(ret_type.dtype)]) ]), ) return ([assign_output], [output_var], output_var) assert isinstance(ret_type, relay.TupleType) assignments = [] extra_args = [] fields = [] for t in ret_type.fields: inner_assignments, inner_args, inner_output = convert_output(t) assignments += inner_assignments extra_args += inner_args fields.append(inner_output) fields = [ast.List(fields, Load())] return (assignments, extra_args, self.create_call("_container.tuple_object", fields))
def visit_Call(self, node): existing_node = self.generic_visit(node) func_node = existing_node.func if self._is_untraceable_attribute(func_node): return existing_node comparisons = [] # [(name, node)] names = self._get_attribute_names(func_node) if names is not None: comparisons.append( ('.'.join(names[:-1]), existing_node.func.value)) for arg_node in existing_node.args: if isinstance(arg_node, Name): comparisons.append((arg_node.id, arg_node)) if not comparisons: return existing_node args = [ List(elts=[], ctx=Load()), List(elts=[], ctx=Load()), existing_node, List(elts=[], ctx=Load()), Num(n=existing_node.lineno) ] for name, node in comparisons: args[0].elts.append(Str(s=name)) # name args[1].elts.append( # repr() before self._create_bare_context_call('get_repr', [node])) args[3].elts.append( # repr() after self._create_bare_context_call('get_repr', [node])) new_node = self._create_bare_context_call('record_call', args) return new_node
def visit_Call(self, node): """Wrap require statements with lambdas and unpack any argument packages.""" func = node.func if isinstance( func, Name) and func.id == requireStatement: # Require statement # Soft reqs have 2 arguments, including the probability, which is given as the # first argument by the token translator; so we allow an extra argument here and # validate it later on (in case the user wrongly gives 2 arguments to require). if not (1 <= len(node.args) <= 2): raise self.parseError(node, 'require takes exactly one argument') if len(node.keywords) != 0: raise self.parseError(node, 'require takes no keyword arguments') cond = node.args[-1] if isinstance(cond, Starred): raise self.parseError( node, 'argument unpacking cannot be used with require') req = self.visit(cond) line = self.lineMap[node.lineno] reqID = Num(len(self.requirements)) # save ID number self.requirements.append( req) # save condition for later inspection when pruning closure = Lambda(noArgs, req) # enclose requirement in a lambda lineNum = Num(line) # save line number for error messages copy_location(closure, req) copy_location(lineNum, req) newArgs = [reqID, closure, lineNum] if len(node.args) == 2: # get probability for soft requirements prob = node.args[0] if not isinstance(prob, Num): raise self.parseError( node, 'malformed requirement ' '(should be a single expression)') newArgs.append(prob) return copy_location(Call(func, newArgs, []), node) else: # Ordinary function call newArgs = [] # Translate arguments, unpacking any argument packages for arg in node.args: if isinstance(arg, BinOp) and isinstance(arg.op, packageNode): newArgs.extend(self.unpack(arg, 2, node)) else: newArgs.append(self.visit(arg)) newKeywords = [self.visit(kwarg) for kwarg in node.keywords] return copy_location(Call(func, newArgs, newKeywords), node)
def visit_Yield(self, node): existing_node = self.generic_visit(node) value = existing_node.value if value is None: value = Name(id='None', ctx=Load()) return Yield(value=self._create_bare_context_call( 'yield_value', [value, Num(n=existing_node.lineno)]))
def test_selector_standalone(): from ast import Expr, Num # use python's builtin ast library Expr._priority = 0 Num._priority = 1 node = Expr(value=Num(n=1)) sel = Selector(Num) sel.visit(node) assert isinstance(sel.out[0], Num)
def visit_For(self, node): new_node = self.generic_visit(node) # Collect line numbers from all except else block. line_numbers = set() self._find_line_numbers(new_node.target, line_numbers) self._find_line_numbers(new_node.iter, line_numbers) for statement in new_node.body: self._find_line_numbers(statement, line_numbers) line_numbers.add(new_node.lineno) args = [Num(n=min(line_numbers)), Num(n=max(line_numbers))] new_body = [self._create_context_call('start_block', args)] new_body.extend(self._trace_assignment_list(new_node.target)) new_body.extend(new_node.body) new_node.body = new_body return new_node
def __init__(self, number): int.__init__(self, number) Num.__init__(self, n=number) NesType.__init__(self)