kw_defaults=[], kwarg=None, defaults=[], ), body=ast.Str(s="None"), ), ast.Lambda( args=ast.arguments( args=[ast.arg(arg="s", annotation=None)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[], ), body=ast.Num(n=0), ), ast.Lambda( args=ast.arguments( args=[ast.arg(arg="s", annotation=None)], vararg=ast.arg(arg="a", annotation=None), kwonlyargs=[], kw_defaults=[], kwarg=ast.arg(arg="k", annotation=None), defaults=[], ), body=ast.Name(id="s", ctx=ast.Load()), ), ], ), ],
def p_num(t): '''num : INTEGER''' t[0] = ast.Num(eval(t[1])) t[0].lineno = t.lineno(1) t[0].col_offset = -1 # XXX
def mutate_Num(self, node): return ast.Num(n=node.n + 1)
def collect(node): """ Attempts to collect commutative operations into one and simplifies them. For example, if x and y are scalars, and z is a vector, then (x*z)*y should be rewritten as (x*y)*z to minimise the number of vector operations. Similarly, ((x*2)*3)*4 should be rewritten as x*24. Works for either multiplication/division or addition/subtraction nodes. The final output is a subexpression of the following maximal form: (((numerical_value*(product of scalars))/(product of scalars))*(product of vectors))/(product of vectors) Any possible cancellations will have been done. Parameters ---------- node : Brian AST node The node to be collected/simplified. Returns ------- node : Brian AST node Simplified node. """ node.collected = True orignode_dtype = node.dtype # we only work on */ or +- ops, which are both BinOp if node.__class__.__name__ != 'BinOp': return node # primary would be the * or + nodes, and inverted would be the / or - nodes terms_primary = [] terms_inverted = [] # we handle both multiplicative and additive nodes in the same way by using these variables if node.op.__class__.__name__ in ['Mult', 'Div']: op_primary = ast.Mult op_inverted = ast.Div op_null = prefs.core.default_float_dtype( 1.0) # the identity for the operator op_py_primary = lambda x, y: x * y op_py_inverted = lambda x, y: x / y elif node.op.__class__.__name__ in ['Add', 'Sub']: op_primary = ast.Add op_inverted = ast.Sub op_null = prefs.core.default_float_dtype(0.0) op_py_primary = lambda x, y: x + y op_py_inverted = lambda x, y: x - y else: return node if node.dtype == 'integer': op_null_with_dtype = int(op_null) else: op_null_with_dtype = op_null # recursively collect terms into the terms_primary and terms_inverted lists collect_commutative(node, op_primary, op_inverted, terms_primary, terms_inverted) x = op_null # extract the numerical nodes and fully evaluate remaining_terms_primary = [] remaining_terms_inverted = [] for term in terms_primary: if term.__class__.__name__ == 'Num': x = op_py_primary(x, term.n) elif term.__class__.__name__ == 'Constant': x = op_py_primary(x, term.value) else: remaining_terms_primary.append(term) for term in terms_inverted: if term.__class__.__name__ == 'Num': x = op_py_inverted(x, term.n) elif term.__class__.__name__ == 'Constant': x = op_py_inverted(x, term.value) else: remaining_terms_inverted.append(term) # if the fully evaluated node is just the identity/null element then we # don't have to make it into an explicit term if x != op_null: if hasattr(ast, 'Constant'): num_node = ast.Constant(x) else: num_node = ast.Num(x) else: num_node = None terms_primary = remaining_terms_primary terms_inverted = remaining_terms_inverted node = num_node for scalar in (True, False): primary_terms = [ term for term in terms_primary if term.scalar == scalar ] inverted_terms = [ term for term in terms_inverted if term.scalar == scalar ] primary_terms, inverted_terms = cancel_identical_terms( primary_terms, inverted_terms) # produce nodes that are the reduction of the operator on these subsets prod_primary = reduced_node(primary_terms, op_primary) prod_inverted = reduced_node(inverted_terms, op_primary) # construct the simplest version of the fully simplified node (only doing operations where necessary) node = reduced_node([node, prod_primary], op_primary) if prod_inverted is not None: if node is None: if hasattr(ast, 'Constant'): node = ast.Constant(op_null_with_dtype) else: node = ast.Num(op_null_with_dtype) node = ast.BinOp(node, op_inverted(), prod_inverted) if node is None: # everything cancelled if hasattr(ast, 'Constant'): node = ast.Constant(op_null_with_dtype) else: node = ast.Num(op_null_with_dtype) if hasattr(node, 'dtype') and dtype_hierarchy[ node.dtype] < dtype_hierarchy[orignode_dtype]: node = ast.BinOp(ast.Num(op_null_with_dtype), op_primary(), node) node.collected = True return node
def visit_Attribute(self, node): """ A reference `self.field[...]` will be replaced with an array reference ensemble_namefield[...] to reflect SOA (struct of array) layout. """ if node.value.id == "self": # name is ensemble name + attribute name = self.ensemble.name + node.attr ndim = self.ensemble.ndim offset = 0 if node.attr.endswith("input"): assert isinstance(self.ensemble, latte.ensemble.ActivationEnsemble) # ActivationEnsembles support the self.input construct that is # equivalent to self.inputs[neuron_index...][0] name += "s" self.seen_vars.add(name) args = [ ast.Name("_neuron_index_{}".format(i), ast.Load()) for i in range(ndim + 1) ] field = name.replace(self.ensemble.name, "") if field in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[field]: dim += 1 # offset for batch dimension args[dim].id += "_outer" args.append( ast.Name("_neuron_index_{}_inner".format(dim), ast.Load())) for i, p in enumerate(self.ensemble.pad): if p[0] > 0: #ANAND: if both tiling and padding on pad has to be divided by tile factor if field in self.ensemble.tiling_info: found = False for dim, factor in self.ensemble.tiling_info[ field]: if dim == i: found = True pad = p[0] // factor args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(pad)) pad2 = p[0] % factor args[i + ndim] = ast.BinOp( args[i + ndim], ast.Add(), ast.Num(pad2)) if found == False: args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(p[0])) else: args[i + 1] = ast.BinOp(args[i + 1], ast.Add(), ast.Num(p[0])) return ast.Subscript(ast.Name(name, ast.Load()), ast.Index(ast.Tuple(args, ast.Load())), node.ctx) # mark as seen self.seen_vars.add(name) if node.attr in ["inputs", "grad_inputs"]: # only generate batch index for inputs/grad_inputs because # the user will provide rest of indices in expression ndim = 1 elif node.attr in self.ensemble.batch_fields or node.attr in self.ensemble.private_info: # increment ndim for fields that have a batch dimension ndim += 1 else: # fields that don't have a batch dimension start at an offset 1 # as 0 is the batch dimension offset = 1 if isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): if "inputs" in node.attr or "grad_inputs" in node.attr: ndim = 1 offset = 0 else: ndim = self.ensemble.ndim + 1 offset = 0 args = [] # if "grad_" in node.attr and not node.attr.endswith("inputs"): # if node.attr in self.ensemble.private_info: # # We privatize these buffers and reduce across threads at the # # end, removing need for synchronization. This is done by # # adding an outer dimension of size num_threads to the buffer # # args.append(ast.Call(ast.Name("omp_get_thread_num", ast.Load()), [], [])) # args.append(ast.Name("_neuron_index_0", ast.Load())) # only append dimensions if it is not fixed in self.buffer_dim_info # (used for values shared across a dimension) #if not isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): #if node.attr not in ["value", "grad"]: if isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): for i in range(ndim): if name not in self.buffer_dim_info or not self.buffer_dim_info[ name][i]: if node.attr in ["value", "grad"] and i == 1: name2 = "_output_offset_{}".format(i) while name2 in self.seen_vars2: name2 += str(i) name3 = "_neuron_index_{}".format(i + offset) if node.attr in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[ node.attr]: if dim == 0: name3 += "_outer" args.append( ast.BinOp(ast.Name(name3, ast.Load()), ast.Add(), ast.Name(name2, ast.Load()))) self.seen_vars2.add(name2) else: args.append( ast.Name("_neuron_index_{}".format(i + offset), ast.Load())) else: for i in range(ndim): if name not in self.buffer_dim_info or not self.buffer_dim_info[ name][i]: args.append( ast.Name("_neuron_index_{}".format(i + offset), ast.Load())) if node.attr in self.ensemble.scalar_fields and \ node.attr in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[node.attr]: if node.attr in self.ensemble.batch_fields or node.attr in self.ensemble.private_info: dim += 1 # offset for batch dimension if not dim == 1 or not isinstance( self.ensemble, latte.ensemble.ConcatEnsemble): idx = args[dim].id args[dim].id = idx + "_outer" args.append(ast.Name(idx + "_inner", ast.Load())) else: args.append( ast.Name("_neuron_index_1_inner", ast.Load())) if node.attr in ["value", "grad"]: for i, p in enumerate(self.ensemble.pad): if p[0] > 0: #ANAND 10/11/2016: Adding pading update by tiling factor if node.attr in self.ensemble.tiling_info: found = False for dim, factor in self.ensemble.tiling_info[ node.attr]: if dim == i: found = True #factor = self.ensemble.tiling_info[node.attr] pad = p[0] // factor args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(pad)) pad2 = p[0] % factor args[i + ndim] = ast.BinOp( args[i + ndim], ast.Add(), ast.Num(pad2)) if found == False: args[i + 1] = ast.BinOp( args[i + 1], ast.Add(), ast.Num(p[0])) else: args[i + 1] = ast.BinOp(args[i + 1], ast.Add(), ast.Num(p[0])) # return updated indedxing expression return ast.Subscript(ast.Name(name, ast.Load()), ast.Index(ast.Tuple(args, ast.Load())), node.ctx) else: raise Exception("Unsupported Attribute node")
def parse_formula(self): if not self.needs_parse: return try: dataset = self._parent if self._formula_status is FormulaStatus.OK: self._node._remove_node_parent(self) self._node = None parent = None trans_error = False trans_name = None if self.column_type == ColumnType.RECODED: if self._transform != 0: trans = dataset.get_transform_by_id(self._transform) if trans.status == FormulaStatus.ERROR: trans_name = trans.name trans_error = True if self._parent_id != 0: parent = dataset.get_column_by_id(self._parent_id) self.formula = trans.produce_formula(parent) else: self.formula = '' elif self._parent_id != 0: parent = dataset.get_column_by_id(self._parent_id) self.formula = '`{0}`'.format(parent.name) else: self.formula = '' node = Parser.parse(self.formula) if trans_error: self.formula_message = "'%s' is in error" % (trans_name) else: self.formula_message = '' if node is not None: node = Transfudgifier().visit(node) if self.column_type is ColumnType.FILTER: if node is None: node = ast.Num(1) # 1 is true else: node = ast.Call( # if missing, treat as 0 (false) func=ast.Name(id='IFMISS', ctx=ast.Load()), args=[ node, ast.Num(0), ast.Call( # convert value to int func=ast.Name(id='INT', ctx=ast.Load()), args=[node], keywords=[]) ], keywords=[]) # here we determine the parent filters parent_filter_start = None parent_filter_end = None parent_filter_no = None for i in range(self.index - 1, -1, -1): column = self._parent[i] if column.active: if parent_filter_no is None: if column.filter_no != self.filter_no: parent_filter_no = column.filter_no parent_filter_end = i else: if column.filter_no != parent_filter_no: parent_filter_start = i break if parent_filter_no is not None: if parent_filter_start is None: parent_filter_start = 0 parents = map( lambda i: self._parent[i], range(parent_filter_start, parent_filter_end + 1)) parents = filter(lambda p: p.active, parents) parents = map( lambda p: ast.Name(id=p.name, ctx=ast.Load()), parents) parents = list(parents) ops = list(map(lambda i: ast.Eq(), range(0, len(parents)))) Transfilterifier(parents).visit(node) # we construct a new node, which incorporates all the # parent filters node = ast.Call(func=ast.Name(id='IF', ctx=ast.Load()), args=[ ast.Compare(left=ast.Num(1), ops=ops, comparators=parents), node, ast.Num(-2147483648) ], keywords=[]) if node is None: self._formula_status = FormulaStatus.EMPTY else: Checker.check(node, self) node = Transmogrifier(dataset, parent).visit(node) self._node = node self._node._add_node_parent(self) self._formula_status = FormulaStatus.OK for dep in self.dependencies: if dep.needs_parse: dep.parse_formula() self.set_data_type(self._node.data_type) self.set_measure_type(self._node.measure_type) except BaseException as e: self._formula_status = FormulaStatus.ERROR self._child.formula_message = Messages.create_from(e) # import traceback # print(traceback.format_exc()) self._needs_parse = False
def test_expr(self): expr = ast3.Expr(ast3.Num(8)) self.assertEqual(evaluator.literal_eval(expr), 8)
class TestExportJson(unittest.TestCase): maxDiff = None def test_export_json(self): for i, test in enumerate(self.tests): with self.subTest(test=i): result = json.loads(export_json(test.input)) expected = test.output self.assertEqual(result, expected) tests = [ TestIO(input=ast.Module(body=[ ast.Assign(targets=[ast.Name(ctx=ast.Store(), id="x")], value=ast.Num(n=5)) ]), output={ "ast_type": "Module", "body": [{ "ast_type": "Assign", "col_offset": None, "lineno": None, "targets": [{ "ast_type": "Name", "col_offset": None, "ctx": { "ast_type": "Store" }, "id": "x", "lineno": None }], "value": { "ast_type": "Num", "col_offset": None, "lineno": None, "n": { "ast_type": "int", "n": 5, "n_str": "5" } } }] }), TestIO(input=ast.Module(body=[ ast.Expr(col_offset=0, lineno=1, value=ast.Call(col_offset=0, lineno=1, func=ast.Name(col_offset=0, lineno=1, id="foobar", ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None)) ]), output={ "ast_type": "Module", "body": [{ "ast_type": "Expr", "col_offset": 0, "lineno": 1, "value": { "args": [], "ast_type": "Call", "col_offset": 0, "func": { "ast_type": "Name", "col_offset": 0, "ctx": { "ast_type": "Load" }, "id": "foobar", "lineno": 1 }, "keywords": [], "lineno": 1 } }] }), TestIO(input=ast.NameConstant(None), output={ "ast_type": "NameConstant", "col_offset": None, "lineno": None, "value": "None" }), TestIO(input=ast.NameConstant(True), output={ "ast_type": "NameConstant", "col_offset": None, "lineno": None, "value": "True" }), TestIO(input=ast.NameConstant(False), output={ "ast_type": "NameConstant", "col_offset": None, "lineno": None, "value": "False" }), TestIO(input=ast.Module( body=[ast.Global(names=["x"], lineno=1, col_offset=0)]), output={ "body": [{ "lineno": 1, "col_offset": 0, "ast_type": "Global", "names": ["x"] }], "ast_type": "Module" }), TestIO(input=ast.FunctionDef(name="function", args=ast.arguments( args=[], vararg=None, kwonlyargs=[ ast.arg(arg="x", annotation=None, lineno=1, col_offset=16) ], kw_defaults=[None], kwarg=None, defaults=[]), body=[ast.Pass(lineno=1, col_offset=20)], decorator_list=[], returns=None, lineno=1, col_offset=0), output={ "args": { "args": [], "ast_type": "arguments", "defaults": [], "kw_defaults": [None], "kwarg": None, "kwonlyargs": [{ "annotation": None, "arg": "x", "ast_type": "arg", "col_offset": 16, "lineno": 1 }], "vararg": None }, "ast_type": "FunctionDef", "body": [{ "ast_type": "Pass", "col_offset": 20, "lineno": 1 }], "col_offset": 0, "decorator_list": [], "lineno": 1, "name": "function", "returns": None }), ]
def to_python_ast(self): # print(self.text) if self.text == 'define': fn_node = self.children[0] fn_name = fn_node.text param_nodes = fn_node.children ast_args = ast.arguments(args=[ ast.arg(arg=x.text, annotation=None) for x in param_nodes ], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]) return ast.FunctionDef( name=fn_name, args=ast_args, body=[ast.Return(value=self.children[1].to_python_ast())], decorator_list=[], returns=None, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == 'if': tst = self.children[0].to_python_ast() body = self.children[1].to_python_ast() if len(self.children) == 3: orelse = self.children[2].to_python_ast() else: orelse = ast.NameConstant(value=None) return ast.IfExp(test=tst, body=body, orelse=orelse, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == 'true': return ast.NameConstant(value=True, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == 'false': return ast.NameConstant(value=False, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text.isdigit(): return ast.Num(n=int(self.text), lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '=': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.Compare(left=left, ops=[ast.Eq()], comparators=[right], lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '+': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.BinOp(left=left, op=ast.Add(), right=right, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '-': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.BinOp(left=left, op=ast.Sub(), right=right, lineno=self.centroid[1], col_offset=self.centroid[0]) elif self.text == '*': left = self.children[0].to_python_ast() right = self.children[1].to_python_ast() return ast.BinOp(left=left, op=ast.Mult(), right=right, lineno=self.centroid[1], col_offset=self.centroid[0]) else: if len(self.children) == 0: # Local var return ast.Name(id=self.text, ctx=ast.Load(), lineno=self.centroid[1], col_offset=self.centroid[0]) else: # Function call if self.children[0].text == '': # Empty parameters if child is blank. args = [] else: args = [x.to_python_ast() for x in self.children] return ast.Call(func=ast.Name(id=self.text, ctx=ast.Load()), args=args, keywords=[], lineno=self.centroid[1], col_offset=self.centroid[0])
def test_module(self): body = [ast.Num(42)] x = ast.Module(body) self.assertEqual(x.body, body)
def test_starred(self): left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store()) assign = ast.Assign([left], ast.Num(4)) self.stmt(assign, "must have Store context")
def test_field_attr_writable(self): x = ast.Num() # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666)
def test_constant_func_applied(object_stream): s = statement_constant(ast.Num(n=10), 10, int) t = s.apply_as_function(new_term(object)) assert t.term == '10' assert t.type is int
def test_constant_func(object_stream): s = statement_constant(ast.Num(n=10), 10, int) assert s.apply(object_stream) == 10
def visit_BinOp(self, node): if node.op.__class__ in self.operators: sympy_class = self.operators[node.op.__class__] right = self.visit(node.right) left = self.visit(node.left) if isinstance(node.left, ast.UnaryOp) and (isinstance( node.right, ast.UnaryOp) == 0) and sympy_class in ('Mul', ): left, right = right, left if isinstance(node.op, ast.Sub): right = ast.Call(func=ast.Name(id='Mul', ctx=ast.Load()), args=[ ast.UnaryOp(op=ast.USub(), operand=ast.Num(1)), right ], keywords=[ ast.keyword(arg='evaluate', value=ast.NameConstant( value=False, ctx=ast.Load())) ], starargs=None, kwargs=None) if isinstance(node.op, ast.Div): if isinstance(node.left, ast.UnaryOp): if isinstance(node.right, ast.UnaryOp): left, right = right, left left = ast.Call(func=ast.Name(id='Pow', ctx=ast.Load()), args=[ left, ast.UnaryOp(op=ast.USub(), operand=ast.Num(1)) ], keywords=[ ast.keyword(arg='evaluate', value=ast.NameConstant( value=False, ctx=ast.Load())) ], starargs=None, kwargs=None) else: right = ast.Call(func=ast.Name(id='Pow', ctx=ast.Load()), args=[ right, ast.UnaryOp(op=ast.USub(), operand=ast.Num(1)) ], keywords=[ ast.keyword(arg='evaluate', value=ast.NameConstant( value=False, ctx=ast.Load())) ], starargs=None, kwargs=None) new_node = ast.Call(func=ast.Name(id=sympy_class, ctx=ast.Load()), args=[left, right], keywords=[ ast.keyword(arg='evaluate', value=ast.NameConstant( value=False, ctx=ast.Load())) ], starargs=None, kwargs=None) if sympy_class in ('Add', 'Mul'): # Denest Add or Mul as appropriate new_node.args = self.flatten(new_node.args, sympy_class) return new_node return node
def arithmetic(self): pre_alloc_left, left = self.arithmetic_get_reference(self.expr.left) pre_alloc_right, right = self.arithmetic_get_reference(self.expr.right) if not is_numeric_type(left.typ) or not is_numeric_type(right.typ): raise TypeMismatchException( "Unsupported types for arithmetic op: %r %r" % (left.typ, right.typ), self.expr, ) arithmetic_pair = {left.typ.typ, right.typ.typ} # Special Case: Simplify any literal to literal arithmetic at compile time. if left.typ.is_literal and right.typ.is_literal and \ isinstance(right.value, int) and isinstance(left.value, int): if isinstance(self.expr.op, ast.Add): val = left.value + right.value elif isinstance(self.expr.op, ast.Sub): val = left.value - right.value elif isinstance(self.expr.op, ast.Mult): val = left.value * right.value elif isinstance(self.expr.op, ast.Div): val = left.value // right.value elif isinstance(self.expr.op, ast.Mod): val = left.value % right.value elif isinstance(self.expr.op, ast.Pow): val = left.value ** right.value else: raise ParserException( 'Unsupported literal operator: %s' % str(type(self.expr.op)), self.expr, ) num = ast.Num(val) num.source_code = self.expr.source_code num.lineno = self.expr.lineno num.col_offset = self.expr.col_offset return Expr.parse_value_expr(num, self.context) # Special case with uint256 were int literal may be casted. if arithmetic_pair == {'uint256', 'int128'}: # Check right side literal. if right.typ.is_literal and SizeLimits.in_bounds('uint256', right.value): right = LLLnode.from_list( right.value, typ=BaseType('uint256', None, is_literal=True), pos=getpos(self.expr), ) # Check left side literal. elif left.typ.is_literal and SizeLimits.in_bounds('uint256', left.value): left = LLLnode.from_list( left.value, typ=BaseType('uint256', None, is_literal=True), pos=getpos(self.expr), ) # Only allow explicit conversions to occur. if left.typ.typ != right.typ.typ: raise TypeMismatchException( "Cannot implicitly convert {} to {}.".format(left.typ.typ, right.typ.typ), self.expr, ) ltyp, rtyp = left.typ.typ, right.typ.typ if isinstance(self.expr.op, (ast.Add, ast.Sub)): if left.typ.unit != right.typ.unit and left.typ.unit != {} and right.typ.unit != {}: raise TypeMismatchException( "Unit mismatch: %r %r" % (left.typ.unit, right.typ.unit), self.expr, ) if left.typ.positional and right.typ.positional and isinstance(self.expr.op, ast.Add): raise TypeMismatchException( "Cannot add two positional units!", self.expr, ) new_unit = left.typ.unit or right.typ.unit # xor, as subtracting two positionals gives a delta new_positional = left.typ.positional ^ right.typ.positional op = 'add' if isinstance(self.expr.op, ast.Add) else 'sub' if ltyp == 'uint256' and isinstance(self.expr.op, ast.Add): o = LLLnode.from_list([ 'seq', # Checks that: a + b >= a ['assert', ['ge', ['add', left, right], left]], ['add', left, right], ], typ=BaseType('uint256', new_unit, new_positional), pos=getpos(self.expr)) elif ltyp == 'uint256' and isinstance(self.expr.op, ast.Sub): o = LLLnode.from_list([ 'seq', # Checks that: a >= b ['assert', ['ge', left, right]], ['sub', left, right] ], typ=BaseType('uint256', new_unit, new_positional), pos=getpos(self.expr)) elif ltyp == rtyp: o = LLLnode.from_list( [op, left, right], typ=BaseType(ltyp, new_unit, new_positional), pos=getpos(self.expr), ) else: raise Exception("Unsupported Operation '%r(%r, %r)'" % (op, ltyp, rtyp)) elif isinstance(self.expr.op, ast.Mult): if left.typ.positional or right.typ.positional: raise TypeMismatchException("Cannot multiply positional values!", self.expr) new_unit = combine_units(left.typ.unit, right.typ.unit) if ltyp == rtyp == 'uint256': o = LLLnode.from_list([ 'if', ['eq', left, 0], [0], [ 'seq', ['assert', ['eq', ['div', ['mul', left, right], left], right]], ['mul', left, right] ], ], typ=BaseType('uint256', new_unit), pos=getpos(self.expr)) elif ltyp == rtyp == 'int128': o = LLLnode.from_list( ['mul', left, right], typ=BaseType('int128', new_unit), pos=getpos(self.expr), ) elif ltyp == rtyp == 'decimal': o = LLLnode.from_list([ 'with', 'r', right, [ 'with', 'l', left, [ 'with', 'ans', ['mul', 'l', 'r'], [ 'seq', [ 'assert', ['or', ['eq', ['sdiv', 'ans', 'l'], 'r'], ['iszero', 'l']] ], ['sdiv', 'ans', DECIMAL_DIVISOR], ], ], ], ], typ=BaseType('decimal', new_unit), pos=getpos(self.expr)) else: raise Exception("Unsupported Operation 'mul(%r, %r)'" % (ltyp, rtyp)) elif isinstance(self.expr.op, ast.Div): if left.typ.positional or right.typ.positional: raise TypeMismatchException("Cannot divide positional values!", self.expr) new_unit = combine_units(left.typ.unit, right.typ.unit, div=True) if ltyp == rtyp == 'uint256': o = LLLnode.from_list([ 'seq', # Checks that: b != 0 ['assert', right], ['div', left, right], ], typ=BaseType('uint256', new_unit), pos=getpos(self.expr)) elif ltyp == rtyp == 'int128': o = LLLnode.from_list( ['sdiv', left, ['clamp_nonzero', right]], typ=BaseType('int128', new_unit), pos=getpos(self.expr), ) elif ltyp == rtyp == 'decimal': o = LLLnode.from_list([ 'with', 'l', left, [ 'with', 'r', ['clamp_nonzero', right], [ 'sdiv', ['mul', 'l', DECIMAL_DIVISOR], 'r', ], ] ], typ=BaseType('decimal', new_unit), pos=getpos(self.expr)) else: raise Exception("Unsupported Operation 'div(%r, %r)'" % (ltyp, rtyp)) elif isinstance(self.expr.op, ast.Mod): if left.typ.positional or right.typ.positional: raise TypeMismatchException( "Cannot use positional values as modulus arguments!", self.expr, ) if not are_units_compatible(left.typ, right.typ) and not (left.typ.unit or right.typ.unit): # noqa: E501 raise TypeMismatchException("Modulus arguments must have same unit", self.expr) new_unit = left.typ.unit or right.typ.unit if ltyp == rtyp == 'uint256': o = LLLnode.from_list([ 'seq', ['assert', right], ['mod', left, right] ], typ=BaseType('uint256', new_unit), pos=getpos(self.expr)) elif ltyp == rtyp: o = LLLnode.from_list( ['smod', left, ['clamp_nonzero', right]], typ=BaseType(ltyp, new_unit), pos=getpos(self.expr), ) else: raise Exception("Unsupported Operation 'mod(%r, %r)'" % (ltyp, rtyp)) elif isinstance(self.expr.op, ast.Pow): if left.typ.positional or right.typ.positional: raise TypeMismatchException( "Cannot use positional values as exponential arguments!", self.expr, ) if right.typ.unit: raise TypeMismatchException( "Cannot use unit values as exponents", self.expr, ) if ltyp != 'int128' and ltyp != 'uint256' and isinstance(self.expr.right, ast.Name): raise TypeMismatchException( "Cannot use dynamic values as exponents, for unit base types", self.expr, ) if ltyp == rtyp == 'uint256': o = LLLnode.from_list([ 'seq', [ 'assert', [ 'or', ['or', ['eq', right, 1], ['iszero', right]], ['lt', left, ['exp', left, right]] ], ], ['exp', left, right], ], typ=BaseType('uint256'), pos=getpos(self.expr)) elif ltyp == rtyp == 'int128': new_unit = left.typ.unit if left.typ.unit and not isinstance(self.expr.right, ast.Name): new_unit = {left.typ.unit.copy().popitem()[0]: self.expr.right.n} o = LLLnode.from_list( ['exp', left, right], typ=BaseType('int128', new_unit), pos=getpos(self.expr), ) else: raise TypeMismatchException('Only whole number exponents are supported', self.expr) else: raise ParserException("Unsupported binary operator: %r" % self.expr.op, self.expr) p = ['seq'] if pre_alloc_left: p.append(pre_alloc_left) if pre_alloc_right: p.append(pre_alloc_right) if o.typ.typ == 'int128': p.append([ 'clamp', ['mload', MemoryPositions.MINNUM], o, ['mload', MemoryPositions.MAXNUM], ]) return LLLnode.from_list(p, typ=o.typ, pos=getpos(self.expr)) elif o.typ.typ == 'decimal': p.append([ 'clamp', ['mload', MemoryPositions.MINDECIMAL], o, ['mload', MemoryPositions.MAXDECIMAL], ]) return LLLnode.from_list(p, typ=o.typ, pos=getpos(self.expr)) if o.typ.typ == 'uint256': p.append(o) return LLLnode.from_list(p, typ=o.typ, pos=getpos(self.expr)) else: raise Exception("%r %r" % (o, o.typ))
def visit_For(self, node): for loop in self._loops: print(loop.tagged_node) if node is loop.tagged_node: # generate call to generated function self._functions.append(generate_parallel_function(loop)) slices = generate_slices(loop, cpus) arr_args = [] for slc in slices: arr_args.append([append_slice(name.id, slc) for name in loop.lists]) stmts = [] parsed_import = ast.Import(names=[ast.alias(name='multiprocessing', asname=None)]) parsed_pool = ast.Assign(targets=[ast.Name(id='pool', ctx=ast.Store())], value=ast.Call(func=ast.Attribute(value=ast.Name(id='multiprocessing', ctx=ast.Load()), attr='Pool', ctx=ast.Load()), args=[ast.Num(n=cpus)], keywords=[], starargs=None, kwargs=None)) # stmts.append(parsed_import) stmts.append(parsed_pool) resnames = [] for i in range(cpus): # generate call to apply_async resname = "%s%i" % (ForTransformer.RETURN_STUB, i) resnames.append(resname) fn_call = generate_function_call(id(loop), arr_args[i], i) stmts.append(ast.Assign(targets=[ast.Name(id=resname,ctx=ast.Store())], value=fn_call)) for i, arr in enumerate(loop.lists): print("lists %i" % i) stmts.append(ast.parse(generate_template(resnames, i, arr)).body[0]) return stmts else: print("HELLOOO!!!") self.generic_visit(node)
def visit_Subscript(self, node: ast.Subscript) -> ast.AST: """Subscript slice operations e.g., ``x[1:]`` or ``y[::2]``.""" self.generic_visit(node) log_header = f"visit_Subscript: {self.src_file}:" idx = None # Subscripts have slice properties with col/lineno, slice itself does not have line/col # Index is also a valid Subscript slice property slice = node.slice if not isinstance(slice, ast.Slice): LOGGER.debug("%s (%s, %s): not a slice node.", log_header, node.lineno, node.col_offset) return node # Built "on the fly" based on the various conditions for operation types # The RangeChange options are added in the later if/else cases slice_mutations: Dict[str, ast.Slice] = { "Slice_UnboundUpper": ast.Slice(lower=slice.upper, upper=None, step=slice.step), "Slice_UnboundLower": ast.Slice(lower=None, upper=slice.lower, step=slice.step), "Slice_Unbounded": ast.Slice(lower=None, upper=None, step=slice.step), } node_span = NodeSpan(node) locidx_kwargs = { "lineno": node_span.lineno, "col_offset": node_span.col_offset, "end_lineno": node_span.end_lineno, "end_col_offset": node_span.end_col_offset, } # Unbounded Swap Operation # upper slice range e.g. x[:2] will become x[2:] if slice.lower is None and slice.upper is not None: idx = LocIndex( ast_class="SliceUS", op_type="Slice_UnboundLower", **locidx_kwargs # type: ignore ) self.locs.add(idx) # lower slice range e.g. x[1:] will become x[:1] if slice.upper is None and slice.lower is not None: idx = LocIndex( ast_class="SliceUS", op_type="Slice_UnboundUpper", **locidx_kwargs # type: ignore ) self.locs.add(idx) # Range Change Operation # range upper bound move towards zero from absolute value e.g. x[2,4] becomes x[2,3] # and x[-4, -3] becomes x[-4, -2]. # More likely to generate useful mutants in the positive case. if slice.lower is not None and slice.upper is not None: if isinstance(slice.upper, ast.Num): idx = LocIndex( ast_class="SliceRC", op_type="Slice_UPosToZero", **locidx_kwargs # type: ignore ) slice_mutations["Slice_UPosToZero"] = ast.Slice( lower=slice.lower, upper=ast.Num(n=slice.upper.n - 1), step=slice.step) LOGGER.debug("RangeChange UPosToZero: %s", ast.dump(slice_mutations["Slice_UPosToZero"])) self.locs.add(idx) if isinstance(slice.upper, ast.UnaryOp): idx = LocIndex( ast_class="SliceRC", op_type="Slice_UNegToZero", **locidx_kwargs # type: ignore ) slice_mutations["Slice_UNegToZero"] = ast.Slice( lower=slice.lower, upper=ast.UnaryOp( op=ast.USub(), operand=ast.Num(n=slice.upper.operand.n - 1) # type: ignore ), step=slice.step, ) LOGGER.debug("RangeChange UNegToZero: %s", ast.dump(slice_mutations["Slice_UNegToZero"])) self.locs.add(idx) # Apply Mutation if idx == self.target_idx and not self.readonly: LOGGER.debug("%s mutating idx: %s with %s", log_header, self.target_idx, self.mutation) mutation = slice_mutations[str(self.mutation)] # uses AST.fix_missing_locations since the values of ast.Num and ast.UnaryOp also need # lineno and col-offset values. This is a recursive fix. return ast.fix_missing_locations( ast.copy_location( ast.Subscript(value=node.value, slice=mutation, ctx=node.ctx), node)) LOGGER.debug("%s (%s, %s): no mutations applied.", log_header, node.lineno, node.col_offset) return node
def visitAssign(self, node): return ast.AnnAssign(ast.Name("foo", ast.Store), ast.Str("foo"), ast.Num(1), True)
def gen_unpack_spec(self, tpl): """Generate a specification for '__rl_unpack_sequence__'. This spec is used to protect sequence unpacking. The primary goal of this spec is to tell which elements in a sequence are sequences again. These 'child' sequences have to be protected again. For example there is a sequence like this: (a, (b, c), (d, (e, f))) = g On a higher level the spec says: - There is a sequence of len 3 - The element at index 1 is a sequence again with len 2 - The element at index 2 is a sequence again with len 2 - The element at index 1 in this subsequence is a sequence again with len 2 With this spec '__rl_unpack_sequence__' does something like this for protection (len checks are omitted): t = list(__rl_getiter__(g)) t[1] = list(__rl_getiter__(t[1])) t[2] = list(__rl_getiter__(t[2])) t[2][1] = list(__rl_getiter__(t[2][1])) return t The 'real' spec for the case above is then: spec = { 'min_len': 3, 'childs': ( (1, {'min_len': 2, 'childs': ()}), (2, { 'min_len': 2, 'childs': ( (1, {'min_len': 2, 'childs': ()}) ) } ) ) } So finally the assignment above is converted into: (a, (b, c), (d, (e, f))) = __rl_unpack_sequence__(g, spec) """ spec = ast.Dict(keys=[], values=[]) spec.keys.append(ast.Str('childs')) spec.values.append(ast.Tuple([], ast.Load())) # starred elements in a sequence do not contribute into the min_len. # For example a, b, *c = g # g must have at least 2 elements, not 3. 'c' is empyt if g has only 2. min_len = len([ob for ob in tpl.elts if not self.is_starred(ob)]) offset = 0 for idx, val in enumerate(tpl.elts): # After a starred element specify the child index from the back. # Since it is unknown how many elements from the sequence are # consumed by the starred element. # For example a, *b, (c, d) = g # Then (c, d) has the index '-1' if self.is_starred(val): offset = min_len + 1 elif isinstance(val, ast.Tuple): el = ast.Tuple([], ast.Load()) el.elts.append(ast.Num(idx - offset)) el.elts.append(self.gen_unpack_spec(val)) spec.values[0].elts.append(el) spec.keys.append(ast.Str('min_len')) spec.values.append(ast.Num(min_len)) return spec
def strip_mining(self, xf): assert xf.in_dim == self.analyze.dims assert xf.name == "sm" strip_size = xf.strip_size dim_strip = xf.dim_strip + 1 # 1 based indexing assert self.analyze.representation[dim_strip].loop #for k in self.analyze.indvars: # print(ast.dump(self.analyze.indvars[k])) # add new induction variable dim_strip_indvar = self.analyze.indvars[dim_strip] new_dim_indvar = copy.deepcopy(dim_strip_indvar) new_dim_indvar.arg = new_dim_indvar.arg + str(dim_strip + 1) self.analyze.indvars = shift(self.analyze.indvars, dim_strip) self.analyze.indvars[dim_strip + 1] = new_dim_indvar # change induction variable order new_indvar_id = self.analyze.indvars[dim_strip + 1].arg addarg = CallAddArg(new_indvar_id, dim_strip) for d in range(1, self.analyze.dims + 1): rep = self.analyze.representation[d] for r in rep.rcall: rep.rcall[r] = addarg.visit(rep.rcall[r]) for t in rep.tcall: rep.tcall[t] = addarg.visit(rep.tcall[t]) # shift dimension (first labels, then objects) for d in range(1, self.analyze.dims + 1): if d > dim_strip: rep = self.analyze.representation[d] #print("before ", rep.dim) rep.dim += 1 #print("after ", rep.dim) #print("before ", rep.alp) new_alp = ['e'] for a in rep.alp[1:]: if a[0] != 's': new_alp.append(a[0] + str(rep.dim) + a[2:]) else: new_alp.append(a) rep.alp = new_alp #print("after ", rep.alp) #print("before ", rep.ord) new_ord = ['e'] for a in rep.ord[1:]: if a[0] != 's': new_ord.append(a[0] + str(rep.dim) + a[2:]) else: new_ord.append(a) rep.ord = new_ord #print("after ", rep.ord) new_guard = {} for g in rep.guard: new_guard[g[0] + str(rep.dim) + g[2:]] = rep.guard[g] rep.guard = new_guard new_rcall = {} for r in rep.rcall: new_rcall[r[0] + str(rep.dim) + r[2:]] = rep.rcall[r] rep.rcall = new_rcall new_tcall = {} for t in rep.tcall: new_tcall[t[0] + str(rep.dim) + t[2:]] = rep.tcall[t] rep.tcall = new_tcall new_reps = {} for d in range(1, self.analyze.dims + 1): if d <= dim_strip: new_reps[d] = self.analyze.representation[d] else: new_reps[d + 1] = self.analyze.representation[d] self.analyze.representation = new_reps # set dims self.analyze.dims += 1 # construct new dimension self.analyze.representation[dim_strip + 1] = copy.deepcopy( self.analyze.representation[dim_strip]) self.analyze.representation[dim_strip + 1].fname += str(dim_strip + 1) # change call names in new dim strip_dim_name = self.analyze.representation[dim_strip].fname new_dim_name = self.analyze.representation[dim_strip + 1].fname changercall = ChangeCallee(strip_dim_name, new_dim_name) for r in self.analyze.representation[dim_strip + 1].rcall: self.analyze.representation[ dim_strip + 1].rcall[r] = changercall.visit( self.analyze.representation[dim_strip + 1].rcall[r]) # change tcall in dim_strip if dim_strip + 1 < self.analyze.dims: old_tcall_name = self.analyze.representation[dim_strip + 2].fname new_tcall_name = self.analyze.representation[dim_strip + 1].fname changetcall = ChangeCallee(old_tcall_name, new_tcall_name) for t in self.analyze.representation[dim_strip].tcall: self.analyze.representation[dim_strip].tcall[ t] = changetcall.visit( self.analyze.representation[dim_strip].tcall[t]) else: # last dimension strip mining must be handled separately pass # fix stride in dim_strip changestride = ChangeStride(strip_size) for r in self.analyze.representation[dim_strip].rcall: self.analyze.representation[dim_strip].rcall[r].args[ dim_strip - 1] = changestride.visit( self.analyze.representation[dim_strip].rcall[r].args[ dim_strip - 1]) indvar_labels = self.analyze.getindvar() for r in self.analyze.representation[dim_strip + 1].rcall: call = self.analyze.representation[dim_strip + 1].rcall[r] call.args[dim_strip - 1] = ast.Name(id=indvar_labels[dim_strip - 1], ctx=ast.Load()) call.args[dim_strip] = ast.BinOp(left=ast.Name( id=indvar_labels[dim_strip], ctx=ast.Load()), op=ast.Add(), right=ast.Num(n=1)) # if new dimension has work if self.analyze.representation[dim_strip + 1].work != {}: pass oname = indvar_labels[dim_strip - 1] nname = indvar_labels[dim_strip] changename = ReplaceVar(oname, nname) for s in self.analyze.representation[dim_strip + 1].work: self.analyze.representation[ dim_strip + 1].work[s] = changename.visit( self.analyze.representation[dim_strip + 1].work[s]) #if following dimension has work if dim_strip + 1 < self.analyze.dims: if self.analyze.representation[dim_strip + 2].work != {}: oname = indvar_labels[dim_strip - 1] nname = indvar_labels[dim_strip] changename = ReplaceVar(oname, nname) for s in self.analyze.representation[dim_strip + 2].work: self.analyze.representation[ dim_strip + 2].work[s] = changename.visit( self.analyze.representation[dim_strip + 2].work[s]) # fix bounds in dim_strip+1 indvar_new_dim = indvar_labels[dim_strip] newbound = ast.Compare(left=ast.Name(id=indvar_new_dim, ctx=ast.Load()), ops=[ast.GtE()], comparators=[ast.Num(n=strip_size)]) oguard = self.analyze.representation[dim_strip].guard['g' + str(dim_strip)] nguard = ast.BoolOp(op=ast.Or(), values=[oguard, newbound]) self.analyze.representation[dim_strip + 1].guard[ 'g' + str(dim_strip)] = nguard # labels are not fixed yet # fix labels for new dimension rep = self.analyze.representation[dim_strip + 1] #print("before ", rep.dim) rep.dim += 1 #print("after ", rep.dim) #print("before ", rep.alp) new_alp = ['e'] for a in rep.alp[1:]: if a[0] != 's': new_alp.append(a[0] + str(rep.dim) + a[2:]) else: new_alp.append(a) rep.alp = new_alp #print("after ", rep.alp) #print("before ", rep.ord) new_ord = ['e'] for a in rep.ord[1:]: if a[0] != 's': new_ord.append(a[0] + str(rep.dim) + a[2:]) else: new_ord.append(a) rep.ord = new_ord #print("after ", rep.ord) new_guard = {} for g in rep.guard: new_guard[g[0] + str(rep.dim) + g[2:]] = rep.guard[g] rep.guard = new_guard new_rcall = {} for r in rep.rcall: new_rcall[r[0] + str(rep.dim) + r[2:]] = rep.rcall[r] rep.rcall = new_rcall new_tcall = {} for t in rep.tcall: new_tcall[t[0] + str(rep.dim) + t[2:]] = rep.tcall[t] rep.tcall = new_tcall
compiled = compile( # this feels in missing line, col. no type junk ast.fix_missing_locations(tree), # req. but doesn't matter what gets put filename='<ast>', # series of commands (exec) or value of an expression (eval), also # hook for interactive that prints an expression result (single) mode='eval' ) print(eval(compiled)) # 3 def ast_compile(_ast): return compile( ast.fix_missing_locations(_ast), filename='<ast>', mode='eval', ) mantree = ast.Expression( ast.BinOp( op=ast.Add(), left=ast.Num(1), right=ast.Num(2), ) ) mancomp = ast_compile(mantree) print(eval(mancomp))
def visit_Subscript(self, node): """ If self.visit(node.value) returns a Subscript, flatten that expression into the current subscript node. """ node.value = self.visit(node.value) if isinstance(node.value, ast.Subscript): value = node.value # append or extend the indexing expressions for *current* node to child node if isinstance(node.slice.value, (ast.Name, ast.Num)): value.slice.value.elts.append(node.slice.value) elif isinstance(node.slice.value, ast.Tuple): value.slice.value.elts.extend(node.slice.value.elts) else: raise NotImplementedError(node.slice.value) field = value.value.id.replace(self.ensemble.name, '') if field in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[field]: # dim += 1 # offset for batch dimension if field in self.ensemble.private_info: dim += 1 # offset for omp_get_thread_num() elif field in self.ensemble.batch_fields: dim += 1 index = value.slice.value.elts[dim] if isinstance(index, ast.Name): orig_var = index.id #Anand: modifying below, tiled variable names reflected only if #they are mapping dims #if "_neuron_index_" in orig_var: value.slice.value.elts[dim] = ast.Name( orig_var + "_outer", ast.Load()) value.slice.value.elts.append( ast.Name(orig_var + "_inner", ast.Load())) self.tiled_vars[orig_var] = dim #else: # value.slice.value.elts.append(ast.Name(orig_var, ast.Load())) elif isinstance(value.slice.value.elts[dim], ast.Num) and \ index.n == 0: value.slice.value.elts.append(ast.Num(0)) else: raise NotImplementedError( type(value.slice.value.elts[dim])) if "inputs" in value.value.id or "grad_inputs" in value.value.id: # Add the input offsets defined by user's mapping for the # connection ndim = self.ensemble.ndim # if isinstance(value.slice.value.elts[1], ast.Num) and value.slice.value.elts[1].n == 0: # value.slice.value.elts.append(ast.Name("_input_offset_1_inner", ast.Load())) #if not isinstance(self.ensemble, latte.ensemble.ConcatEnsemble): for i in range(1, ndim + 1): elem = value.slice.value.elts[i] tile = False if field in self.ensemble.tiling_info: for dim, _ in self.ensemble.tiling_info[field]: if dim + 1 == i: tile = True if tile: length = 0 if len(self.connections[0].mapping.shape) > i: if len(self.connections[0].mapping.shape[i - 1]) == 1: length = 1 if length == 0: value.slice.value.elts[i] = ast.BinOp( elem, ast.Add(), ast.Name("_input_offset_{}_outer".format(i), ast.Load())) value.slice.value.elts[i + ndim] = ast.BinOp( value.slice.value.elts[i + ndim], ast.Add(), ast.Name("_input_offset_{}_inner".format(i), ast.Load())) else: value.slice.value.elts[i] = ast.Name( "_neuron_index_{}_outer".format(i), ast.Load()) value.slice.value.elts[i + ndim] = ast.Name( "_neuron_index_{}_inner".format(i), ast.Load()) else: value.slice.value.elts[i] = ast.BinOp( elem, ast.Add(), ast.Name("_input_offset_{}".format(i), ast.Load())) return value else: raise NotImplementedError() return node
def _create_assign_lambda( s, o, lamb ): assert isinstance( o, Signal ), "You can only assign(//=) a lambda function to a Wire/InPort/OutPort." srcs, line = inspect.getsourcelines( lamb ) src = compiled_re.sub( r'\2', ''.join(srcs) ).lstrip(' ') root = ast.parse(src) assert isinstance( root, ast.Module ) and len(root.body) == 1, "We only support single-statement lambda." root = root.body[0] assert isinstance( root, ast.AugAssign ) and isinstance( root.op, ast.FloorDiv ) lhs, rhs = root.target, root.value # We expect the lambda to have no argument: # {'args': [], 'vararg': None, 'kwonlyargs': [], 'kw_defaults': [], 'kwarg': None, 'defaults': []} assert isinstance( rhs, ast.Lambda ) and not rhs.args.args and rhs.args.vararg is None, \ "The lambda shouldn't contain any argument." rhs = rhs.body # Compose a new and valid function based on the lambda's lhs and rhs # Note that we don't need to add those source code of closure var # assignment to linecache. To get the matching line number in the # error message, we set the line number of update block # Shunning: bugfix: blk_name = "_lambda__{}".format( repr(o).replace(".","_").replace("[", "_").replace("]", "_").replace(":", "_") ) lambda_upblk = ast.FunctionDef( name=blk_name, args=ast.arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[ast.Assign(targets=[lhs], value=rhs, lineno=2, col_offset=6)], decorator_list=[], returns=None, lineno=1, col_offset=4, ) lambda_upblk_module = ast.Module(body=[ lambda_upblk ]) # Manually wrap the lambda upblk with a closure function that adds the # desired variables to the closure of `_lambda__*` # We construct AST for the following function to add free variables in the # closure of the lambda function to the closure of the generated lambda # update block. # # def closure( lambda_closure ): # <FreeVarName1> = lambda_closure[<Idx1>].cell_contents # <FreeVarName2> = lambda_closure[<Idx2>].cell_contents # ... # <FreeVarNameN> = lambda_closure[<IdxN>].cell_contents # def _lambda__<lambda_blk_name>(): # # the assignment statement appears here # return _lambda__<lambda_blk_name> new_root = ast.Module( body=[ ast.FunctionDef( name="closure", args=ast.arguments(args=[ast.arg(arg="lambda_closure", annotation=None, lineno=1, col_offset=12)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[ ast.Assign( targets=[ast.Name(id=var, ctx=ast.Store(), lineno=1+idx, col_offset=2)], value=ast.Attribute( value=ast.Subscript( value=ast.Name( id='lambda_closure', ctx=ast.Load(), lineno=1+idx, col_offset=5+len(var), ), slice=ast.Index( value=ast.Num( n=idx, lineno=1+idx, col_offset=19+len(var), ), ), ctx=ast.Load(), lineno=1+idx, col_offset=5+len(var), ), attr='cell_contents', ctx=ast.Load(), lineno=1+idx, col_offset=5+len(var), ), lineno=1+idx, col_offset=2, ) for idx, var in enumerate(lamb.__code__.co_freevars) ] + [ lambda_upblk ] + [ ast.Return( value=ast.Name( id=blk_name, ctx=ast.Load(), lineno=4+len(lamb.__code__.co_freevars), col_offset=9, ), lineno=4+len(lamb.__code__.co_freevars), col_offset=2, ) ], decorator_list=[], returns=None, lineno=1, col_offset=0, ) ] ) # In Python 3 we need to supply a dict as local to get the newly # compiled function from closure. # Then `closure(lamb.__closure__)` returns the lambda update block with # the correct free variables in its closure. dict_local = {} custom_exec( compile(new_root, blk_name, "exec"), lamb.__globals__, dict_local ) blk = dict_local[ 'closure' ]( lamb.__closure__ ) # Add the source code to linecache for the compiled function new_src = "def {}():\n {}\n".format( blk_name, src.replace("//=", "=") ) linecache.cache[ blk_name ] = (len(new_src), None, new_src.splitlines(), blk_name) ComponentLevel1.update( s, blk ) # This caching here does no caching because the block name contains # the signal name intentionally to avoid conflicts. With //= it is # more possible than normal update block to have conflicts: # if param == 1: s.out //= s.in_ + 1 # else: s.out //= s.out + 100 # Here these two blocks will implicity have the same name but they # have different contents based on different param. # So the cache call here is just to reuse the existing interface to # register the AST/src of the generated block for elaborate or passes # to use. s._cache_func_meta( blk, is_update_ff=False, given=("".join(srcs), lambda_upblk_module, line, inspect.getsourcefile( lamb )) ) return blk
def visit_For(self, node): first = node i = first.target.id i_len = first.iter.args[0] if not util.has_nested_for(node.body): return node second = node.body[0] j = second.target.id j_len = second.iter.args[0] if not util.has_nested_for(second.body): return node third = second.body[0] if isinstance(third, latte.transformers.neuron.RangeDim): mapping_func = util.get_ast(third.mapping).body[0] ndim = len(mapping_func.args.args) dim = third.child_for.iter.args[1].n k_len = ast.Num(len(third.mapping(*[1 for _ in range(ndim)])[dim])) third = third.child_for else: k_len = third.iter.args[0] k = third.target.id if isinstance(third.body[0], ast.AugAssign) and \ isinstance(third.body[0].op, ast.Add) and \ isinstance(third.body[0].value, ast.BinOp) and \ isinstance(third.body[0].value.op, ast.Mult): A = third.body[0].value.left B = third.body[0].value.right C = third.body[0].target A_idx = [idx.id for idx in A.slice.value.elts] B_idx = [idx.id for idx in B.slice.value.elts] C_idx = [idx.id for idx in C.slice.value.elts] if C_idx == [i, j]: ldc = j_len if A_idx == [k, i]: trans_A = ast.Name("True", ast.Load()) lda = i_len elif A_idx == [i, k]: trans_A = ast.Name("False", ast.Load()) lda = k_len else: raise NotImplementedError() if B_idx == [j, k]: trans_B = ast.Name("True", ast.Load()) ldb = k_len elif B_idx == [k, j]: trans_B = ast.Name("False", ast.Load()) ldb = j_len else: raise NotImplementedError() A.value.id = "_" + A.value.id B.value.id = "_" + B.value.id C.value.id = "_" + C.value.id gemm_call = ast.Call(ast.Name("sgemm", ast.Load()), [ trans_A, trans_B, i_len, j_len, k_len, ast.Num(1.0), A.value, lda, B.value, ldb, ast.Num(1.0), C.value, ldc ], []) return ast.Expr(gemm_call) elif C_idx == [i, k]: ldc = k_len if A_idx == [j, i]: trans_A = ast.Name("True", ast.Load()) lda = i_len elif A_idx == [i, j]: trans_A = ast.Name("False", ast.Load()) lda = j_len else: raise NotImplementedError() if B_idx == [k, j]: trans_B = ast.Name("True", ast.Load()) ldb = j_len elif B_idx == [j, k]: trans_B = ast.Name("False", ast.Load()) ldb = k_len else: raise NotImplementedError() A.value.id = "_" + A.value.id B.value.id = "_" + B.value.id C.value.id = "_" + C.value.id gemm_call = ast.Call(ast.Name("sgemm", ast.Load()), [ trans_A, trans_B, i_len, k_len, j_len, ast.Num(1.0), A.value, lda, B.value, ldb, ast.Num(1.0), C.value, ldc ], []) return ast.Expr(gemm_call) elif C_idx == [j, k]: ldc = k_len if A_idx == [i, j]: trans_A = ast.Name("True", ast.Load()) lda = j_len elif A_idx == [j, i]: trans_A = ast.Name("False", ast.Load()) lda = i_len else: raise NotImplementedError() if B_idx == [k, i]: trans_B = ast.Name("True", ast.Load()) ldb = i_len elif B_idx == [i, k]: trans_B = ast.Name("False", ast.Load()) ldb = k_len else: raise NotImplementedError() A.value.id = "_" + A.value.id B.value.id = "_" + B.value.id C.value.id = "_" + C.value.id gemm_call = ast.Call(ast.Name("sgemm", ast.Load()), [ trans_A, trans_B, j_len, k_len, i_len, ast.Num(1.0), A.value, lda, B.value, ldb, ast.Num(1.0), C.value, ldc ], []) return ast.Expr(gemm_call) else: raise NotImplementedError(C_idx, [i, j, k]) # raise NotImplementedError(astor.to_source(node)) return node
def f(tree, **kw): assert macropy.core.unparse( tree) == "(1 * max(1, 2, 3))", macropy.core.unparse(tree) return ast.Num(n=10)
def to_ast(self): return ast.Num( int(self.value) if isinstance(self.value, bool) else self.value)
def convert_range(child): if len(child) == 1: child.insert(0, ast.Num(n=0)) return child
def test_number(self): tree = transform('a=<:number:>', dict(number=ast.Num(3))) env = {} exec(compile(tree, '<string>', 'exec'), env) assert env['a'] == 3
def from_phpast(node): if node is None: return py.Pass(**pos(node)) if isinstance(node, basestring): return py.Str(node, **pos(node)) if isinstance(node, (int, float)): return py.Num(node, **pos(node)) if isinstance(node, php.Array): if node.nodes: if node.nodes[0].key is not None: keys = [] values = [] for elem in node.nodes: keys.append(from_phpast(elem.key)) values.append(from_phpast(elem.value)) return py.Dict(keys, values, **pos(node)) else: return py.List([from_phpast(x.value) for x in node.nodes], py.Load(**pos(node)), **pos(node)) else: return py.List([], py.Load(**pos(node)), **pos(node)) if isinstance(node, php.InlineHTML): args = [py.Str(node.data, **pos(node))] return py.Call( py.Name('inline_html', py.Load(**pos(node)), **pos(node)), args, [], None, None, **pos(node)) if isinstance(node, php.Echo): return py.Call(py.Name('echo', py.Load(**pos(node)), **pos(node)), map(from_phpast, node.nodes), [], None, None, **pos(node)) if isinstance(node, php.Print): return py.Print(None, [from_phpast(node.node)], True, **pos(node)) if isinstance(node, php.Exit): args = [] if node.expr is not None: args.append(from_phpast(node.expr)) return py.Raise( py.Call(py.Name('Exit', py.Load(**pos(node)), **pos(node)), args, [], None, None, **pos(node)), None, None, **pos(node)) if isinstance(node, php.Return): if node.node is None: return py.Return(None, **pos(node)) else: return py.Return(from_phpast(node.node), **pos(node)) if isinstance(node, php.Break): assert node.node is None, 'level on break not supported' return py.Break(**pos(node)) if isinstance(node, php.Continue): assert node.node is None, 'level on continue not supported' return py.Continue(**pos(node)) if isinstance(node, php.Silence): return from_phpast(node.expr) if isinstance(node, php.Block): return from_phpast(php.If(1, node, [], None, lineno=node.lineno)) if isinstance(node, php.Unset): return py.Delete(map(from_phpast, node.nodes), **pos(node)) if isinstance(node, php.IsSet) and len(node.nodes) == 1: if isinstance(node.nodes[0], php.ArrayOffset): return py.Compare(from_phpast(node.nodes[0].expr), [py.In(**pos(node))], [from_phpast(node.nodes[0].node)], **pos(node)) if isinstance(node.nodes[0], php.ObjectProperty): return py.Call( py.Name('hasattr', py.Load(**pos(node)), **pos(node)), [ from_phpast(node.nodes[0].node), from_phpast(node.nodes[0].name) ], [], None, None, **pos(node)) if isinstance(node.nodes[0], php.Variable): return py.Compare(py.Str( node.nodes[0].name[1:], **pos(node)), [py.In(**pos(node))], [ py.Call(py.Name('vars', py.Load(**pos(node)), **pos(node)), [], [], None, None, **pos(node)) ], **pos(node)) return py.Compare(from_phpast(node.nodes[0]), [py.IsNot(**pos(node))], [py.Name('None', py.Load(**pos(node)), **pos(node))], **pos(node)) if isinstance(node, php.Empty): return from_phpast( php.UnaryOp('!', php.BinaryOp('&&', php.IsSet([node.expr], lineno=node.lineno), node.expr, lineno=node.lineno), lineno=node.lineno)) if isinstance(node, php.Assignment): if (isinstance(node.node, php.ArrayOffset) and node.node.expr is None): return py.Call( py.Attribute(from_phpast(node.node.node), 'append', py.Load(**pos(node)), **pos(node)), [from_phpast(node.expr)], [], None, None, **pos(node)) if (isinstance(node.node, php.ObjectProperty) and isinstance(node.node.name, php.BinaryOp)): return to_stmt( py.Call(py.Name('setattr', py.Load(**pos(node)), **pos(node)), [ from_phpast(node.node.node), from_phpast(node.node.name), from_phpast(node.expr) ], [], None, None, **pos(node))) return py.Assign([store(from_phpast(node.node))], from_phpast(node.expr), **pos(node)) if isinstance(node, php.ListAssignment): return py.Assign([ py.Tuple(map(store, map(from_phpast, node.nodes)), py.Store(**pos(node)), **pos(node)) ], from_phpast(node.expr), **pos(node)) if isinstance(node, php.AssignOp): return from_phpast( php.Assignment(node.left, php.BinaryOp(node.op[:-1], node.left, node.right, lineno=node.lineno), False, lineno=node.lineno)) if isinstance(node, (php.PreIncDecOp, php.PostIncDecOp)): return from_phpast( php.Assignment(node.expr, php.BinaryOp(node.op[0], node.expr, 1, lineno=node.lineno), False, lineno=node.lineno)) if isinstance(node, php.ArrayOffset): return py.Subscript(from_phpast(node.node), py.Index(from_phpast(node.expr), **pos(node)), py.Load(**pos(node)), **pos(node)) if isinstance(node, php.ObjectProperty): if isinstance(node.name, (php.Variable, php.BinaryOp)): return py.Call( py.Name('getattr', py.Load(**pos(node)), **pos(node)), [from_phpast(node.node), from_phpast(node.name)], [], None, None, **pos(node)) return py.Attribute(from_phpast(node.node), node.name, py.Load(**pos(node)), **pos(node)) if isinstance(node, php.Constant): name = node.name if name.lower() == 'true': name = 'True' if name.lower() == 'false': name = 'False' if name.lower() == 'null': name = 'None' return py.Name(name, py.Load(**pos(node)), **pos(node)) if isinstance(node, php.Variable): name = node.name[1:] if name == 'this': name = 'self' return py.Name(name, py.Load(**pos(node)), **pos(node)) if isinstance(node, php.Global): return py.Global([var.name[1:] for var in node.nodes], **pos(node)) if isinstance(node, php.Include): once = py.Name('True' if node.once else 'False', py.Load(**pos(node)), **pos(node)) return py.Call(py.Name('include', py.Load(**pos(node)), **pos(node)), [from_phpast(node.expr), once], [], None, None, **pos(node)) if isinstance(node, php.Require): once = py.Name('True' if node.once else 'False', py.Load(**pos(node)), **pos(node)) return py.Call(py.Name('require', py.Load(**pos(node)), **pos(node)), [from_phpast(node.expr), once], [], None, None, **pos(node)) if isinstance(node, php.UnaryOp): op = unary_ops.get(node.op) assert op is not None, "unknown unary operator: '%s'" % node.op op = op(**pos(node)) return py.UnaryOp(op, from_phpast(node.expr), **pos(node)) if isinstance(node, php.BinaryOp): if node.op == '.': pattern, pieces = build_format(node.left, node.right) if pieces: return py.BinOp( py.Str(pattern, **pos(node)), py.Mod(**pos(node)), py.Tuple(map(from_phpast, pieces), py.Load(**pos(node)), **pos(node)), **pos(node)) else: return py.Str(pattern % (), **pos(node)) if node.op in bool_ops: op = bool_ops[node.op](**pos(node)) return py.BoolOp(op, [from_phpast(node.left), from_phpast(node.right)], **pos(node)) if node.op in cmp_ops: op = cmp_ops[node.op](**pos(node)) return py.Compare(from_phpast(node.left), [op], [from_phpast(node.right)], **pos(node)) op = binary_ops.get(node.op) assert op is not None, "unknown binary operator: '%s'" % node.op op = op(**pos(node)) return py.BinOp(from_phpast(node.left), op, from_phpast(node.right), **pos(node)) if isinstance(node, php.TernaryOp): return py.IfExp(from_phpast(node.expr), from_phpast(node.iftrue), from_phpast(node.iffalse), **pos(node)) if isinstance(node, php.Cast): return py.Call( py.Name(casts.get(node.type, node.type), py.Load(**pos(node)), **pos(node)), [from_phpast(node.expr)], [], None, None, **pos(node)) if isinstance(node, php.If): orelse = [] if node.else_: for else_ in map(from_phpast, deblock(node.else_.node)): orelse.append(to_stmt(else_)) for elseif in reversed(node.elseifs): orelse = [ py.If(from_phpast(elseif.expr), map(to_stmt, map(from_phpast, deblock(elseif.node))), orelse, **pos(node)) ] return py.If(from_phpast(node.expr), map(to_stmt, map(from_phpast, deblock(node.node))), orelse, **pos(node)) if isinstance(node, php.For): assert node.test is None or len(node.test) == 1, \ 'only a single test is supported in for-loops' return from_phpast( php.Block((node.start or []) + [ php.While(node.test[0] if node.test else 1, php.Block(deblock(node.node) + (node.count or []), lineno=node.lineno), lineno=node.lineno) ], lineno=node.lineno)) if isinstance(node, php.Foreach): if node.keyvar is None: target = py.Name(node.valvar.name[1:], py.Store(**pos(node)), **pos(node)) else: target = py.Tuple([ py.Name(node.keyvar.name[1:], py.Store(**pos(node))), py.Name(node.valvar.name[1:], py.Store(**pos(node))) ], py.Store(**pos(node)), **pos(node)) return py.For(target, from_phpast(node.expr), map(to_stmt, map(from_phpast, deblock(node.node))), [], **pos(node)) if isinstance(node, php.While): return py.While(from_phpast(node.expr), map(to_stmt, map(from_phpast, deblock(node.node))), [], **pos(node)) if isinstance(node, php.DoWhile): condition = php.If(php.UnaryOp('!', node.expr, lineno=node.lineno), php.Break(None, lineno=node.lineno), [], None, lineno=node.lineno) return from_phpast( php.While(1, php.Block(deblock(node.node) + [condition], lineno=node.lineno), lineno=node.lineno)) if isinstance(node, php.Try): return py.TryExcept(map(to_stmt, map(from_phpast, node.nodes)), [ py.ExceptHandler( py.Name(catch.class_, py.Load(**pos(node)), **pos(node)), store(from_phpast(catch.var)), map(to_stmt, map(from_phpast, catch.nodes)), **pos(node)) for catch in node.catches ], [], **pos(node)) if isinstance(node, php.Throw): return py.Raise(from_phpast(node.node), None, None, **pos(node)) if isinstance(node, php.Function): args = [] defaults = [] for param in node.params: args.append( py.Name(param.name[1:], py.Param(**pos(node)), **pos(node))) if param.default is not None: defaults.append(from_phpast(param.default)) body = map(to_stmt, map(from_phpast, node.nodes)) if not body: body = [py.Pass(**pos(node))] return py.FunctionDef(node.name, py.arguments(args, None, None, defaults), body, [], **pos(node)) if isinstance(node, php.Method): args = [] defaults = [] decorator_list = [] if 'static' in node.modifiers: decorator_list.append( py.Name('classmethod', py.Load(**pos(node)), **pos(node))) args.append(py.Name('cls', py.Param(**pos(node)), **pos(node))) else: args.append(py.Name('self', py.Param(**pos(node)), **pos(node))) for param in node.params: args.append( py.Name(param.name[1:], py.Param(**pos(node)), **pos(node))) if param.default is not None: defaults.append(from_phpast(param.default)) body = map(to_stmt, map(from_phpast, node.nodes)) if not body: body = [py.Pass(**pos(node))] return py.FunctionDef(node.name, py.arguments(args, None, None, defaults), body, decorator_list, **pos(node)) if isinstance(node, php.Class): name = node.name bases = [] extends = node.extends or 'object' bases.append(py.Name(extends, py.Load(**pos(node)), **pos(node))) body = map(to_stmt, map(from_phpast, node.nodes)) for stmt in body: if (isinstance(stmt, py.FunctionDef) and stmt.name in (name, '__construct')): stmt.name = '__init__' if not body: body = [py.Pass(**pos(node))] return py.ClassDef(name, bases, body, [], **pos(node)) if isinstance(node, (php.ClassConstants, php.ClassVariables)): assert len(node.nodes) == 1, \ 'only one class-level assignment supported per line' if isinstance(node.nodes[0], php.ClassConstant): name = php.Constant(node.nodes[0].name, lineno=node.lineno) else: name = php.Variable(node.nodes[0].name, lineno=node.lineno) initial = node.nodes[0].initial if initial is None: initial = php.Constant('None', lineno=node.lineno) return py.Assign([store(from_phpast(name))], from_phpast(initial), **pos(node)) if isinstance(node, (php.FunctionCall, php.New)): if isinstance(node.name, basestring): name = py.Name(node.name, py.Load(**pos(node)), **pos(node)) else: name = py.Subscript( py.Call(py.Name('vars', py.Load(**pos(node)), **pos(node)), [], [], None, None, **pos(node)), py.Index(from_phpast(node.name), **pos(node)), py.Load(**pos(node)), **pos(node)) args, kwargs = build_args(node.params) return py.Call(name, args, kwargs, None, None, **pos(node)) if isinstance(node, php.MethodCall): args, kwargs = build_args(node.params) return py.Call( py.Attribute(from_phpast(node.node), node.name, py.Load(**pos(node)), **pos(node)), args, kwargs, None, None, **pos(node)) if isinstance(node, php.StaticMethodCall): class_ = node.class_ if class_ == 'self': class_ = 'cls' args, kwargs = build_args(node.params) return py.Call( py.Attribute(py.Name(class_, py.Load(**pos(node)), **pos(node)), node.name, py.Load(**pos(node)), **pos(node)), args, kwargs, None, None, **pos(node)) if isinstance(node, php.StaticProperty): class_ = node.node name = node.name if isinstance(name, php.Variable): name = name.name[1:] return py.Attribute(py.Name(class_, py.Load(**pos(node)), **pos(node)), name, py.Load(**pos(node)), **pos(node)) return py.Call(py.Name('XXX', py.Load(**pos(node)), **pos(node)), [py.Str(str(node), **pos(node))], [], None, None, **pos(node))