import astprogramming class Number(astprogramming.ASTNode): attribute_names = ('value',) def __init__(self, value, **kwargs): super().__init__(value=value, **kwargs) astprogramming.operators.Pythonizer.set_method(Number, lambda self, node: ast.Num(node.value)) astprogramming.operators.PseudocodeGenerator.set_method(Number, lambda self, node: str(node.value)) class Add(astprogramming.ASTNode): child_names = ('left', 'right') def __init__(self, left, right, **kwargs): super().__init__(left=left, right=right, **kwargs) astprogramming.operators.Pythonizer.set_method(Add, lambda self, node: ast.BinOp(op=ast.Add(), left=self.call(node.left), right=self.call(node.right))) astprogramming.operators.PseudocodeGenerator.set_method(Add, lambda self, node: '{} + {}'.format(self.call(node.left), self.call(node.right))) class Variable(astprogramming.ASTNode): attribute_names = ('name',) def __init__(self, name, **kwargs): super().__init__(name=name, **kwargs) astprogramming.operators.Pythonizer.set_method(Variable, lambda self, node: ast.Name(id=node.name)) astprogramming.operators.PseudocodeGenerator.set_method(Variable, lambda self, node: node.name) class Assignment(astprogramming.ASTNode): child_names = ('lhs', 'value') def __init__(self, lhs, value, **kwargs): super().__init__(lhs=lhs, value=value, **kwargs)
def visit_Assert(self, assert_): """Return the AST statements to replace the ast.Assert instance. This rewrites the test of an assertion to provide intermediate values and replace it with an if statement which raises an assertion error with a detailed explanation in case the expression is false. """ if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1: from _pytest.warning_types import PytestAssertRewriteWarning import warnings warnings.warn_explicit( PytestAssertRewriteWarning( "assertion is always true, perhaps remove parentheses?"), category=None, filename=fspath(self.module_path), lineno=assert_.lineno, ) self.statements = [] # type: List[ast.stmt] self.variables = [] # type: List[str] self.variable_counter = itertools.count() if self.enable_assertion_pass_hook: self.format_variables = [] # type: List[str] self.stack = [] # type: List[Dict[str, ast.expr]] self.expl_stmts = [] # type: List[ast.stmt] self.push_format_context() # Rewrite assert into a bunch of statements. top_condition, explanation = self.visit(assert_.test) negation = ast.UnaryOp(ast.Not(), top_condition) if self.enable_assertion_pass_hook: # Experimental pytest_assertion_pass hook msg = self.pop_format_context(ast.Str(explanation)) # Failed if assert_.msg: assertmsg = self.helper("_format_assertmsg", assert_.msg) gluestr = "\n>assert " else: assertmsg = ast.Str("") gluestr = "assert " err_explanation = ast.BinOp(ast.Str(gluestr), ast.Add(), msg) err_msg = ast.BinOp(assertmsg, ast.Add(), err_explanation) err_name = ast.Name("AssertionError", ast.Load()) fmt = self.helper("_format_explanation", err_msg) exc = ast.Call(err_name, [fmt], []) raise_ = ast.Raise(exc, None) statements_fail = [] statements_fail.extend(self.expl_stmts) statements_fail.append(raise_) # Passed fmt_pass = self.helper("_format_explanation", msg) orig = self._assert_expr_to_lineno()[assert_.lineno] hook_call_pass = ast.Expr( self.helper( "_call_assertion_pass", ast.Num(assert_.lineno), ast.Str(orig), fmt_pass, )) # If any hooks implement assert_pass hook hook_impl_test = ast.If( self.helper("_check_if_assertion_pass_impl"), self.expl_stmts + [hook_call_pass], [], ) statements_pass = [hook_impl_test] # Test for assertion condition main_test = ast.If(negation, statements_fail, statements_pass) self.statements.append(main_test) if self.format_variables: variables = [ ast.Name(name, ast.Store()) for name in self.format_variables ] clear_format = ast.Assign(variables, ast.NameConstant(None)) self.statements.append(clear_format) else: # Original assertion rewriting # Create failure message. body = self.expl_stmts self.statements.append(ast.If(negation, body, [])) if assert_.msg: assertmsg = self.helper("_format_assertmsg", assert_.msg) explanation = "\n>assert " + explanation else: assertmsg = ast.Str("") explanation = "assert " + explanation template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation)) msg = self.pop_format_context(template) fmt = self.helper("_format_explanation", msg) err_name = ast.Name("AssertionError", ast.Load()) exc = ast.Call(err_name, [fmt], []) raise_ = ast.Raise(exc, None) body.append(raise_) # Clear temporary variables by setting them to None. if self.variables: variables = [ ast.Name(name, ast.Store()) for name in self.variables ] clear = ast.Assign(variables, ast.NameConstant(None)) self.statements.append(clear) # Fix line numbers. for stmt in self.statements: set_location(stmt, assert_.lineno, assert_.col_offset) return self.statements
def _(node): return ast.BinOp(left=translate(node.left), op=ast.Add(), right=translate(node.right))
def addition(a, b): #print(f"adding {a} and {b}") return ast.BinOp(left=derive(a), right=derive(b), op=ast.Add()) return f"({derive(a)} + {derive(b)})"
def visit_Mult(self, node: ast.Mult) -> typing.Any: return ast.Add()
def __add__(self, other): return _make_binop(ast.Add(), self._expr, other)
def mutate_Sub(self, node): if self.should_mutate(node): return ast.Add() raise MutationResign()
def to_python_ast(self): # print(self.text) if self.text == 'define': fn_node = self.children[0] fn_name = fn_node.text param_nodes = fn_node.children ast_args = ast.arguments(args=[ ast.arg(arg=x.text, annotation=None) for x in param_nodes ], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]) return ast.FunctionDef( name=fn_name, args=ast_args, body=[ast.Return(value=self.children[1].to_python_ast())], decorator_list=[], returns=None, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == 'if': tst = self.children[0].to_python_ast() body = self.children[1].to_python_ast() if len(self.children) == 3: orelse = self.children[2].to_python_ast() else: orelse = ast.NameConstant(value=None) return ast.IfExp(test=tst, body=body, orelse=orelse, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == 'true': return ast.NameConstant(value=True, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == 'false': return ast.NameConstant(value=False, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text.isdigit(): return ast.Num(n=int(self.text), lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '=': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.Compare(left=left, ops=[ast.Eq()], comparators=[right], lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '+': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.BinOp(left=left, op=ast.Add(), right=right, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '-': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.BinOp(left=left, op=ast.Sub(), right=right, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '*': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.BinOp(left=left, op=ast.Mult(), right=right, lineno=self.centroid[1], col_offset=self.centroid[0]) else: if len(self.children) == 0: # Local var return ast.Name(id=self.text, ctx=ast.Load(), lineno=self.centroid[1], col_offset=self.centroid[0]) else: # Function call if self.children[0].text == '': # Empty parameters if child is blank. args = [] else: args = [x.to_python_ast() for x in self.children] return ast.Call(func=ast.Name(id=self.text, ctx=ast.Load()), args=args, keywords=[], lineno=self.centroid[1], col_offset=self.centroid[0])
def visit_BinOp(self, node): return ast.BinOp(left=node.left, op=ast.Add(), right=node.right)
def visit_Name(self, node: ast.Name): if node.id == self.vname: result = ast.BinOp(left=ast.Name(self.vname, ctx=node.ctx), op=ast.Add(), right=ast.Num(self.const)) return result return node
def test_empty_init(self): # Jython 2.5.0 did not allow empty constructors for many ast node types # but CPython ast nodes do allow this. For the moment, I don't see a # reason to allow construction of the super types (like ast.AST and # ast.stmt) as well as the op types that are implemented as enums in # Jython (like boolop), but I've left them in but commented out for # now. We may need them in the future since CPython allows this, but # it may fall under implementation detail. #ast.AST() ast.Add() ast.And() ast.Assert() ast.Assign() ast.Attribute() ast.AugAssign() ast.AugLoad() ast.AugStore() ast.BinOp() ast.BitAnd() ast.BitOr() ast.BitXor() ast.BoolOp() ast.Break() ast.Call() ast.ClassDef() ast.Compare() ast.Continue() ast.Del() ast.Delete() ast.Dict() ast.Div() ast.Ellipsis() ast.Eq() ast.Exec() ast.Expr() ast.Expression() ast.ExtSlice() ast.FloorDiv() ast.For() ast.FunctionDef() ast.GeneratorExp() ast.Global() ast.Gt() ast.GtE() ast.If() ast.IfExp() ast.Import() ast.ImportFrom() ast.In() ast.Index() ast.Interactive() ast.Invert() ast.Is() ast.IsNot() ast.LShift() ast.Lambda() ast.List() ast.ListComp() ast.Load() ast.Lt() ast.LtE() ast.Mod() ast.Module() ast.Mult() ast.Name() ast.Not() ast.NotEq() ast.NotIn() ast.Num() ast.Or() ast.Param() ast.Pass() ast.Pow() ast.Print() ast.RShift() ast.Raise() ast.Repr() ast.Return() ast.Slice() ast.Store() ast.Str() ast.Sub() ast.Subscript() ast.Suite() ast.TryExcept() ast.TryFinally() ast.Tuple() ast.UAdd() ast.USub() ast.UnaryOp() ast.While() ast.With() ast.Yield() ast.alias() ast.arguments() #ast.boolop() #ast.cmpop() ast.comprehension() #ast.excepthandler() #ast.expr() #ast.expr_context() ast.keyword()
def visit_Mult(self, node): return ast.copy_location(ast.Add(), node)
def test_binop(self): py_ast = ast.BinOp(ast.Num(1), ast.Add(), ast.Num(2)) c_ast = Add(Constant(1), Constant(2)) self._check(py_ast, c_ast)
def test_AddAssign(self): py_ast = ast.AugAssign(ast.Name('i', ast.Load()), ast.Add(), ast.Num(3)) c_ast = AddAssign(SymbolRef('i'), Constant(3)) self._check(py_ast, c_ast)
def check(self): return not search_ast( self.tree, ast.BinOp(left=ast.Str(), op=ast.Add(), right=ast.Str()), )
compiled = compile( # this feels in missing line, col. no type junk ast.fix_missing_locations(tree), # req. but doesn't matter what gets put filename='<ast>', # series of commands (exec) or value of an expression (eval), also # hook for interactive that prints an expression result (single) mode='eval' ) print(eval(compiled)) # 3 def ast_compile(_ast): return compile( ast.fix_missing_locations(_ast), filename='<ast>', mode='eval', ) mantree = ast.Expression( ast.BinOp( op=ast.Add(), left=ast.Num(1), right=ast.Num(2), ) ) mancomp = ast_compile(mantree) print(eval(mancomp))
def add_node(x, y): return ast.BinOp(left=x, op=ast.Add(), right=y)
def strip_mining(self, xf): assert xf.in_dim == self.analyze.dims assert xf.name == "sm" strip_size = xf.strip_size dim_strip = xf.dim_strip + 1 # 1 based indexing assert self.analyze.representation[dim_strip].loop #for k in self.analyze.indvars: # print(ast.dump(self.analyze.indvars[k])) # add new induction variable dim_strip_indvar = self.analyze.indvars[dim_strip] new_dim_indvar = copy.deepcopy(dim_strip_indvar) new_dim_indvar.arg = new_dim_indvar.arg + str(dim_strip + 1) self.analyze.indvars = shift(self.analyze.indvars, dim_strip) self.analyze.indvars[dim_strip + 1] = new_dim_indvar # change induction variable order new_indvar_id = self.analyze.indvars[dim_strip + 1].arg addarg = CallAddArg(new_indvar_id, dim_strip) for d in range(1, self.analyze.dims + 1): rep = self.analyze.representation[d] for r in rep.rcall: rep.rcall[r] = addarg.visit(rep.rcall[r]) for t in rep.tcall: rep.tcall[t] = addarg.visit(rep.tcall[t]) # shift dimension (first labels, then objects) for d in range(1, self.analyze.dims + 1): if d > dim_strip: rep = self.analyze.representation[d] #print("before ", rep.dim) rep.dim += 1 #print("after ", rep.dim) #print("before ", rep.alp) new_alp = ['e'] for a in rep.alp[1:]: if a[0] != 's': new_alp.append(a[0] + str(rep.dim) + a[2:]) else: new_alp.append(a) rep.alp = new_alp #print("after ", rep.alp) #print("before ", rep.ord) new_ord = ['e'] for a in rep.ord[1:]: if a[0] != 's': new_ord.append(a[0] + str(rep.dim) + a[2:]) else: new_ord.append(a) rep.ord = new_ord #print("after ", rep.ord) new_guard = {} for g in rep.guard: new_guard[g[0] + str(rep.dim) + g[2:]] = rep.guard[g] rep.guard = new_guard new_rcall = {} for r in rep.rcall: new_rcall[r[0] + str(rep.dim) + r[2:]] = rep.rcall[r] rep.rcall = new_rcall new_tcall = {} for t in rep.tcall: new_tcall[t[0] + str(rep.dim) + t[2:]] = rep.tcall[t] rep.tcall = new_tcall new_reps = {} for d in range(1, self.analyze.dims + 1): if d <= dim_strip: new_reps[d] = self.analyze.representation[d] else: new_reps[d + 1] = self.analyze.representation[d] self.analyze.representation = new_reps # set dims self.analyze.dims += 1 # construct new dimension self.analyze.representation[dim_strip + 1] = copy.deepcopy( self.analyze.representation[dim_strip]) self.analyze.representation[dim_strip + 1].fname += str(dim_strip + 1) # change call names in new dim strip_dim_name = self.analyze.representation[dim_strip].fname new_dim_name = self.analyze.representation[dim_strip + 1].fname changercall = ChangeCallee(strip_dim_name, new_dim_name) for r in self.analyze.representation[dim_strip + 1].rcall: self.analyze.representation[ dim_strip + 1].rcall[r] = changercall.visit( self.analyze.representation[dim_strip + 1].rcall[r]) # change tcall in dim_strip if dim_strip + 1 < self.analyze.dims: old_tcall_name = self.analyze.representation[dim_strip + 2].fname new_tcall_name = self.analyze.representation[dim_strip + 1].fname changetcall = ChangeCallee(old_tcall_name, new_tcall_name) for t in self.analyze.representation[dim_strip].tcall: self.analyze.representation[dim_strip].tcall[ t] = changetcall.visit( self.analyze.representation[dim_strip].tcall[t]) else: # last dimension strip mining must be handled separately pass # fix stride in dim_strip changestride = ChangeStride(strip_size) for r in self.analyze.representation[dim_strip].rcall: self.analyze.representation[dim_strip].rcall[r].args[ dim_strip - 1] = changestride.visit( self.analyze.representation[dim_strip].rcall[r].args[ dim_strip - 1]) indvar_labels = self.analyze.getindvar() for r in self.analyze.representation[dim_strip + 1].rcall: call = self.analyze.representation[dim_strip + 1].rcall[r] call.args[dim_strip - 1] = ast.Name(id=indvar_labels[dim_strip - 1], ctx=ast.Load()) call.args[dim_strip] = ast.BinOp(left=ast.Name( id=indvar_labels[dim_strip], ctx=ast.Load()), op=ast.Add(), right=ast.Num(n=1)) # if new dimension has work if self.analyze.representation[dim_strip + 1].work != {}: pass oname = indvar_labels[dim_strip - 1] nname = indvar_labels[dim_strip] changename = ReplaceVar(oname, nname) for s in self.analyze.representation[dim_strip + 1].work: self.analyze.representation[ dim_strip + 1].work[s] = changename.visit( self.analyze.representation[dim_strip + 1].work[s]) #if following dimension has work if dim_strip + 1 < self.analyze.dims: if self.analyze.representation[dim_strip + 2].work != {}: oname = indvar_labels[dim_strip - 1] nname = indvar_labels[dim_strip] changename = ReplaceVar(oname, nname) for s in self.analyze.representation[dim_strip + 2].work: self.analyze.representation[ dim_strip + 2].work[s] = changename.visit( self.analyze.representation[dim_strip + 2].work[s]) # fix bounds in dim_strip+1 indvar_new_dim = indvar_labels[dim_strip] newbound = ast.Compare(left=ast.Name(id=indvar_new_dim, ctx=ast.Load()), ops=[ast.GtE()], comparators=[ast.Num(n=strip_size)]) oguard = self.analyze.representation[dim_strip].guard['g' + str(dim_strip)] nguard = ast.BoolOp(op=ast.Or(), values=[oguard, newbound]) self.analyze.representation[dim_strip + 1].guard[ 'g' + str(dim_strip)] = nguard # labels are not fixed yet # fix labels for new dimension rep = self.analyze.representation[dim_strip + 1] #print("before ", rep.dim) rep.dim += 1 #print("after ", rep.dim) #print("before ", rep.alp) new_alp = ['e'] for a in rep.alp[1:]: if a[0] != 's': new_alp.append(a[0] + str(rep.dim) + a[2:]) else: new_alp.append(a) rep.alp = new_alp #print("after ", rep.alp) #print("before ", rep.ord) new_ord = ['e'] for a in rep.ord[1:]: if a[0] != 's': new_ord.append(a[0] + str(rep.dim) + a[2:]) else: new_ord.append(a) rep.ord = new_ord #print("after ", rep.ord) new_guard = {} for g in rep.guard: new_guard[g[0] + str(rep.dim) + g[2:]] = rep.guard[g] rep.guard = new_guard new_rcall = {} for r in rep.rcall: new_rcall[r[0] + str(rep.dim) + r[2:]] = rep.rcall[r] rep.rcall = new_rcall new_tcall = {} for t in rep.tcall: new_tcall[t[0] + str(rep.dim) + t[2:]] = rep.tcall[t] rep.tcall = new_tcall
def visit_Div(self, node): return ast.Add()
def visit_Subscript(self, node): """ If self.visit(node.value) returns a Subscript, flatten that expression into the current subscript node. """ node.value = self.visit(node.value) if isinstance(node.value, ast.Subscript): value = node.value # append or extend the indexing expressions for *current* node to child node if isinstance(node.slice.value, (ast.Name, ast.Num)): value.slice.value.elts.append(node.slice.value) elif isinstance(node.slice.value, ast.Tuple): value.slice.value.elts.extend(node.slice.value.elts) else: raise NotImplementedError(node.slice.value) field = value.value.id.replace(self.ensemble.name, '') if field in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[field]: # dim += 1 # offset for batch dimension if field in self.ensemble.private_info: dim += 1 # offset for omp_get_thread_num() elif field in self.ensemble.batch_fields: dim += 1 index = value.slice.value.elts[dim] if isinstance(index, ast.Name): orig_var = index.id #Anand: modifying below, tiled variable names reflected only if #they are mapping dims #if "_neuron_index_" in orig_var: value.slice.value.elts[dim] = ast.Name( orig_var + "_outer", ast.Load()) value.slice.value.elts.append( ast.Name(orig_var + "_inner", ast.Load())) self.tiled_vars[orig_var] = dim #else: # value.slice.value.elts.append(ast.Name(orig_var, ast.Load())) elif isinstance(value.slice.value.elts[dim], ast.Num) and \ index.n == 0: value.slice.value.elts.append(ast.Num(0)) else: raise NotImplementedError( type(value.slice.value.elts[dim])) if "inputs" in value.value.id or "grad_inputs" in value.value.id: # Add the input offsets defined by user's mapping for the # connection ndim = self.ensemble.ndim # if isinstance(value.slice.value.elts[1], ast.Num) and value.slice.value.elts[1].n == 0: # value.slice.value.elts.append(ast.Name("_input_offset_1_inner", ast.Load())) #if not isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): for i in range(1, ndim + 1): elem = value.slice.value.elts[i] tile = False if field in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[field]: if dim + 1 == i: tile = True if tile: length = 0 if len(self.connections[0].mapping.shape) > i: if len(self.connections[0].mapping.shape[i - 1]) == 1: length = 1 if length == 0: value.slice.value.elts[i] = ast.BinOp( elem, ast.Add(), ast.Name("_input_offset_{}_outer".format(i), ast.Load())) value.slice.value.elts[i + ndim] = ast.BinOp( value.slice.value.elts[i + ndim], ast.Add(), ast.Name("_input_offset_{}_inner".format(i), ast.Load())) else: value.slice.value.elts[i] = ast.Name( "_neuron_index_{}_outer".format(i), ast.Load()) value.slice.value.elts[i + ndim] = ast.Name( "_neuron_index_{}_inner".format(i), ast.Load()) else: value.slice.value.elts[i] = ast.BinOp( elem, ast.Add(), ast.Name("_input_offset_{}".format(i), ast.Load())) return value else: raise NotImplementedError() return node
def multiply(a, b): #print(f"multiplying {a} and {b}") leftmul = ast.BinOp(left=a, right=derive(b), op=ast.Mult()) rightmul = ast.BinOp(left=b, right=derive(a), op=ast.Mult()) return ast.BinOp(left=leftmul, right=rightmul, op=ast.Add()) return f"({a} * {derive(b)} + {b} * {derive(a)})"
def visit_Attribute(self, node): """ A reference `self.field[...]` will be replaced with an array reference ensemble_namefield[...] to reflect SOA (struct of array) layout. """ if node.value.id == "self": # name is ensemble name + attribute name = self.ensemble.name + node.attr ndim = self.ensemble.ndim offset = 0 if node.attr.endswith("input"): assert isinstance(self.ensemble, latte.ensemble.ActivationEnsemble) # ActivationEnsembles support the self.input construct that is # equivalent to self.inputs[neuron_index...][0] name += "s" self.seen_vars.add(name) args = [ ast.Name("_neuron_index_{}".format(i), ast.Load()) for i in range(ndim + 1) ] field = name.replace(self.ensemble.name, "") if field in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[field]: dim += 1 # offset for batch dimension args[dim].id += "_outer" args.append( ast.Name("_neuron_index_{}_inner".format(dim), ast.Load())) for i, p in enumerate(self.ensemble.pad): if p[0] > 0: #ANAND: if both tiling and padding on pad has to be divided by tile factor if field in self.ensemble.tiling_info: found = False for dim, factor in self.ensemble.tiling_info[ field]: if dim == i: found = True pad = p[0] // factor args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(pad)) pad2 = p[0] % factor args[i + ndim] = ast.BinOp( args[i + ndim], ast.Add(), ast.Num(pad2)) if found == False: args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(p[0])) else: args[i + 1] = ast.BinOp(args[i + 1], ast.Add(), ast.Num(p[0])) return ast.Subscript(ast.Name(name, ast.Load()), ast.Index(ast.Tuple(args, ast.Load())), node.ctx) # mark as seen self.seen_vars.add(name) if node.attr in ["inputs", "grad_inputs"]: # only generate batch index for inputs/grad_inputs because # the user will provide rest of indices in expression ndim = 1 elif node.attr in self.ensemble.batch_fields or node.attr in self.ensemble.private_info: # increment ndim for fields that have a batch dimension ndim += 1 else: # fields that don't have a batch dimension start at an offset 1 # as 0 is the batch dimension offset = 1 if isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): if "inputs" in node.attr or "grad_inputs" in node.attr: ndim = 1 offset = 0 else: ndim = self.ensemble.ndim + 1 offset = 0 args = [] # if "grad_" in node.attr and not node.attr.endswith("inputs"): # if node.attr in self.ensemble.private_info: # # We privatize these buffers and reduce across threads at the # # end, removing need for synchronization. This is done by # # adding an outer dimension of size num_threads to the buffer # # args.append(ast.Call(ast.Name("omp_get_thread_num", ast.Load()), [], [])) # args.append(ast.Name("_neuron_index_0", ast.Load())) # only append dimensions if it is not fixed in self.buffer_dim_info # (used for values shared across a dimension) #if not isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): #if node.attr not in ["value", "grad"]: if isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): for i in range(ndim): if name not in self.buffer_dim_info or not self.buffer_dim_info[ name][i]: if node.attr in ["value", "grad"] and i == 1: name2 = "_output_offset_{}".format(i) while name2 in self.seen_vars2: name2 += str(i) name3 = "_neuron_index_{}".format(i + offset) if node.attr in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[ node.attr]: if dim == 0: name3 += "_outer" args.append( ast.BinOp(ast.Name(name3, ast.Load()), ast.Add(), ast.Name(name2, ast.Load()))) self.seen_vars2.add(name2) else: args.append( ast.Name("_neuron_index_{}".format(i + offset), ast.Load())) else: for i in range(ndim): if name not in self.buffer_dim_info or not self.buffer_dim_info[ name][i]: args.append( ast.Name("_neuron_index_{}".format(i + offset), ast.Load())) if node.attr in self.ensemble.scalar_fields and \ node.attr in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[node.attr]: if node.attr in self.ensemble.batch_fields or node.attr in self.ensemble.private_info: dim += 1 # offset for batch dimension if not dim == 1 or not isinstance( self.ensemble, latte.ensemble.ConcatEnsemble): idx = args[dim].id args[dim].id = idx + "_outer" args.append(ast.Name(idx + "_inner", ast.Load())) else: args.append( ast.Name("_neuron_index_1_inner", ast.Load())) if node.attr in ["value", "grad"]: for i, p in enumerate(self.ensemble.pad): if p[0] > 0: #ANAND 10/11/2016: Adding pading update by tiling factor if node.attr in self.ensemble.tiling_info: found = False for dim, factor in self.ensemble.tiling_info[ node.attr]: if dim == i: found = True #factor = self.ensemble.tiling_info[node.attr] pad = p[0] // factor args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(pad)) pad2 = p[0] % factor args[i + ndim] = ast.BinOp( args[i + ndim], ast.Add(), ast.Num(pad2)) if found == False: args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(p[0])) else: args[i + 1] = ast.BinOp(args[i + 1], ast.Add(), ast.Num(p[0])) # return updated indedxing expression return ast.Subscript(ast.Name(name, ast.Load()), ast.Index(ast.Tuple(args, ast.Load())), node.ctx) else: raise Exception("Unsupported Attribute node")
def add_mapping(self, args, target): if len(target) > 1: raise ValueError("Only mapping to single target is supported") arg_idxs = [] for arg in args: arg_idxs.append(self.variable_names[arg]) target_idx = self.variable_names[target[0]] print(target_idx) if len(args) == 1: self.body.append(ast.Assign(targets=[ast.Subscript(value=GLOBAL_ARRAY, slice=ast.Index( value=ast.Constant(value=target_idx, kind=None)))], value=ast.Subscript(value=GLOBAL_ARRAY, slice=ast.Index( value=ast.Constant(value=arg_idxs[0], kind=None))) , lineno=0)) else: self.body.append(ast.Assign(targets=[ast.Subscript(value=GLOBAL_ARRAY, slice=ast.Index( value=ast.Constant(value=target_idx, kind=None)))], value=ast.BinOp(left=self._generate_sum_left(arg_idxs[1:]), op=ast.Add(), right=ast.Subscript(value=GLOBAL_ARRAY, slice=ast.Index( value=ast.Constant(value=arg_idxs[0], kind=None)))) , lineno=0))
def isl2py_exp(e): ty = e.get_type() # returns isl_ast_expr_type # Expression if ty == isl.ast_expr_type.op: op_ty = e.get_op_type() # AND if op_ty == isl.ast_expr_op_type.and_: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.BoolOp(pyast.And(), [op0, op1]) return expr # LESS EQUAL elif op_ty == isl.ast_expr_op_type.le: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.Compare(left=op0, ops=[ pyast.LtE(), ], comparators=[op1]) return expr # LESS elif op_ty == isl.ast_expr_op_type.lt: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.Compare(left=op0, ops=[ pyast.Lt(), ], comparators=[op1]) return expr # EQUALITY elif op_ty == isl.ast_expr_op_type.eq: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.Compare(left=op0, ops=[ pyast.Eq(), ], comparators=[op1]) return expr # GREATER EQUAL elif op_ty == isl.ast_expr_op_type.ge: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.Compare(left=op0, ops=[ pyast.GtE(), ], comparators=[op1]) return expr # Minus elif op_ty == isl.ast_expr_op_type.minus: op0 = isl2py_exp(e.get_op_arg(0)) expr = pyast.UnaryOp(pyast.USub(), op0) return expr # ADD elif op_ty == isl.ast_expr_op_type.add: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.BinOp(op0, pyast.Add(), op1) return expr # SUB elif op_ty == isl.ast_expr_op_type.sub: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.BinOp(op0, pyast.Sub(), op1) return expr # MUL elif op_ty == isl.ast_expr_op_type.mul: op0 = isl2py_exp(e.get_op_arg(0)) op1 = isl2py_exp(e.get_op_arg(1)) expr = pyast.BinOp(op0, pyast.Mult(), op1) return expr # MAX elif op_ty == isl.ast_expr_op_type.max: # NB: Not sure if max can actually have more than one args. nargs = e.get_op_n_arg() args = [isl2py_exp(e.get_op_arg(i)) for i in range(nargs)] expr = pyast.Call( func=pyast.Name(id="max", ctx=pyast.Load()), args=args, keywords=[], ) return expr # MIN elif op_ty == isl.ast_expr_op_type.min: # NB: Not sure if max can actually have more than one args. nargs = e.get_op_n_arg() args = [isl2py_exp(e.get_op_arg(i)) for i in range(nargs)] expr = pyast.Call( func=pyast.Name(id="min", ctx=pyast.Load()), args=args, keywords=[], ) return expr else: raise NotImplementedError("No support for op_ty=%d" % (op_ty, )) # ID elif ty == isl.ast_expr_type.id: name = e.get_id().name if name == "": return pyast.NameConstant(None) else: return pyast.Name(name, pyast.Load()) # INT elif ty == isl.ast_expr_type.int: val = e.get_val().to_python() return pyast.Num(val) elif ty == isl.ast_expr_type.error: raise NotImplementedError else: raise AssertionError("uknown ISL expr type: %d" % (ty, ))
def visit_Assert(self, assert_): """Return the AST statements to replace the ast.Assert instance. This rewrites the test of an assertion to provide intermediate values and replace it with an if statement which raises an assertion error with a detailed explanation in case the expression is false. """ if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1: from _pytest.warning_types import PytestAssertRewriteWarning import warnings warnings.warn_explicit( PytestAssertRewriteWarning( "assertion is always true, perhaps remove parentheses?"), category=None, filename=str(self.module_path), lineno=assert_.lineno, ) self.statements = [] self.variables = [] self.variable_counter = itertools.count() self.stack = [] self.on_failure = [] self.push_format_context() # Rewrite assert into a bunch of statements. top_condition, explanation = self.visit(assert_.test) # If in a test module, check if directly asserting None, in order to warn [Issue #3191] if self.module_path is not None: self.statements.append( self.warn_about_none_ast(top_condition, module_path=self.module_path, lineno=assert_.lineno)) # Create failure message. body = self.on_failure negation = ast.UnaryOp(ast.Not(), top_condition) self.statements.append(ast.If(negation, body, [])) if assert_.msg: assertmsg = self.helper("_format_assertmsg", assert_.msg) explanation = "\n>assert " + explanation else: assertmsg = ast.Str("") explanation = "assert " + explanation template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation)) msg = self.pop_format_context(template) fmt = self.helper("_format_explanation", msg) err_name = ast.Name("AssertionError", ast.Load()) exc = ast_Call(err_name, [fmt], []) if sys.version_info[0] >= 3: raise_ = ast.Raise(exc, None) else: raise_ = ast.Raise(exc, None, None) body.append(raise_) # Clear temporary variables by setting them to None. if self.variables: variables = [ ast.Name(name, ast.Store()) for name in self.variables ] clear = ast.Assign(variables, _NameConstant(None)) self.statements.append(clear) # Fix line numbers. for stmt in self.statements: set_location(stmt, assert_.lineno, assert_.col_offset) return self.statements
def compile_statements(self, node_list): full_code = [] for node in node_list: if type(node) is ast.Assign: if type(node.targets[0]) is ast.Name: target = (node.targets[0].id, 'var') elif type(node.targets[0]) is ast.Subscript: code1, var1 = self.compile_int_expr( ast.BinOp(node.targets[0].value, ast.Add(), node.targets[0].slice.value), None) full_code += code1 target = (var1, 'array') else: raise Babeception( str(node.targets[0]) + ' not supported as target of assignment') code = self.compile_int_expr(node.value, target) full_code += code elif type(node) is ast.Return: if node.value: code = self.compile_int_expr(node.value, 'register') op = ('return_write', ) else: code = [] op = ('return', ) full_code += code + [op] elif type(node) is ast.Expr: code = self.compile_int_expr( node.value, 'register') #TODO: maybe make this more elegant full_code += code elif type(node) is ast.If: if len(node.orelse) == 0: test, _ = preprocess_boolean(node.test, True) end = (self.jmp_cnt, 'label') self.jmp_cnt += 1 logic_code = self.compile_boolean(test, end) body_code = self.compile_statements(node.body) full_code += logic_code + body_code + [('label', end)] else: test, _ = preprocess_boolean(node.test, False) end = (self.jmp_cnt, 'label') body = (self.jmp_cnt + 1, 'label') self.jmp_cnt += 2 logic_code = self.compile_boolean(test, body) else_code = self.compile_statements(node.orelse) body_code = self.compile_statements(node.body) full_code += logic_code + else_code + [ ('dir_jump', end), ('label', body) ] + body_code + [('label', end)] elif type(node) is ast.While: test_pre, _ = preprocess_boolean(node.test, False) test = (self.jmp_cnt, 'label') body = (self.jmp_cnt + 1, 'label') self.jmp_cnt += 2 body_code = self.compile_statements(node.body) logic_code = self.compile_boolean(test_pre, body) full_code += [('dir_jump', test), ('label', body) ] + body_code + [('label', test)] + logic_code elif type(node) is ast.For: # target, iter, body if type(node.target) is not ast.Name or type( node.iter) is not ast.Tuple: raise Babeception('Wrong For Syntax!') code = self.compile_int_expr(node.iter.elts[0], 'register') full_code += code test = (self.jmp_cnt, 'label') body = (self.jmp_cnt + 1, 'label') self.jmp_cnt += 2 test_pre = ast.Compare(node.target, [ast.Lt()], [node.iter.elts[1]]) body_code = self.compile_statements(node.body) logic_code = self.compile_boolean(test_pre, body) inc_code = self.compile_int_expr(node.iter.elts[2], 'register') inc_code2 = [('ADD', (node.target.id, 'var')), ('WRITE', (node.target.id, 'var'))] full_code += [('WRITE', (node.target.id, 'var')), ('dir_jump', test), ('label', body)]\ + body_code + inc_code + inc_code2 + [('label', test)] + logic_code return full_code
def visit_BinOp(self, node): if type(node.op).__name__ == "Sub": node.op = ast.Add() elif type(node.op).__name__ == "Add": node.op = ast.Sub() return node
def compile_int_expr(self, node, target): if type(node) is ast.BinOp: op_to_func = { ast.Mult: 'MUL', ast.Div: 'DIV', ast.Mod: 'MOD', ast.Pow: 'POW' } if type(node.op) in op_to_func: return self.compile_int_expr( ast.Call(ast.Name(op_to_func[type(node.op)]), [node.left, node.right]), target) # TODO: add shift, maybe second mul? elif type(node.op) is ast.Add: code_right, var1 = self.compile_int_expr(node.right, False) code_left = self.compile_int_expr(node.left, 'register') op = ('ADD', var1) return self.save_output(code_right + code_left + [op], target) elif type(node.op) is ast.Sub: return self.compile_int_expr( ast.BinOp(ast.UnaryOp(ast.USub(), node.right), ast.Add(), node.left), target) else: raise Babeception( str(type(node.op)) + ' is not supported! ' + str(node.op)) elif type(node) is ast.UnaryOp: if type(node.op) is ast.USub: code_prev = self.compile_int_expr( node.operand, 'register') #TODO: check for negative constants op = ('INV', ) return self.save_output(code_prev + [op], target) else: raise Babeception(str(type(node.op)) + ' is not supported!') elif type(node) is ast.Call: jmp_code = self.jmp_cnt code = [('LOAD', ((jmp_code, 'label'), 'mem_jmp')), ('WRITE', (self.tmp_cnt, 'tmp'))] self.jmp_cnt += 1 old_cnt = self.tmp_cnt + 1 self.tmp_cnt += len(node.args) + 1 for i in range(len(node.args)): part_code = self.compile_int_expr(node.args[i], (old_cnt + i, 'tmp')) code += part_code par = (self.tmp_cnt, 'tmp') code += [ ('call', node.func.id, list(range(old_cnt - 1, old_cnt + len(node.args))), par), # maybe start location is enough ('label_back', (jmp_code, 'label')) ] self.tmp_cnt += 1 load = ('LOAD', par) if target == 'register': return code + [load] elif target: write = ('WRITE', target) return code + [load, write] else: return code, par elif type(node) in [ast.Constant, ast.Name, ast.Subscript]: code = [] if type(node) is ast.Constant: par = (node.value, 'const') elif type(node) is ast.Name: par = (node.id, 'var') else: code1, var1 = self.compile_int_expr( ast.BinOp(node.value, ast.Add(), node.slice.value), None) code += code1 par = (var1, 'array') load = ('LOAD', par) if target == 'register': return code + [load] elif target: write = ('WRITE', target) return code + [load, write] else: return code, par else: raise Babeception(str(type(node)) + ' is not supported!')
def add(left, right): return ast.BinOp(left=left, right=right, op=ast.Add())
def eval_function_2(node): if isinstance(node.op, ast.Add): node.op = ast.Mult() elif isinstance(node.op, ast.Mult): node.op = ast.Add()