def DISABLED_visit_Compare(self, node): "Reduce cascaded comparisons into single comparisons" # Process children self.generic_visit(node) # TODO: We can't generate temporaries from subexpressions since # this may invalidate execution order. For now, set the type so # we can clone for c in node.comparators: c.type = None compare_nodes = [] comparators = [nodes.CloneableNode(c) for c in node.comparators] # Build comparison nodes left = node.left for op, right in zip(node.ops, comparators): node = self.ir.Compare(left=left, ops=[op], comparators=[right]) # We shouldn't need to type this... node = nodes.typednode(node, typesystem.bool_) left = right.clone compare_nodes.append(node) # AND the comparisons together boolop = lambda left, right: self.ir.BoolOp(ast.And(), [left, right]) node = reduce(boolop, reversed(compare_nodes)) return node
def visit_ArrayNewNode(self, node): if self.nopython: raise error.NumbaError( node, "Cannot yet allocate new array in nopython context") PyArray_Type = nodes.ObjectInjectNode(np.ndarray) descr = nodes.ObjectInjectNode(node.type.dtype.get_dtype()).cloneable ndim = nodes.const(node.type.ndim, int_) flags = nodes.const(0, int_) args = [PyArray_Type, descr.clone, ndim, node.shape, node.strides, node.data, flags] incref_descr = nodes.IncrefNode(descr) incref_base = None setbase = None if node.base is None: args.append(nodes.NULL_obj) else: base = nodes.CloneableNode(node.base) incref_base = nodes.IncrefNode(base) args.append(base.clone) array = nodes.PyArray_NewFromDescr(args) array = nodes.ObjectTempNode(array).cloneable body = [incref_descr, incref_base, array, setbase] if node.base is not None: body.append(nodes.PyArray_SetBaseObject([array.clone, base.clone])) # TODO: PyArray_UpdateFlags() result = nodes.ExpressionNode(filter(None, body), array.clone) return self.visit(result)
def handle_method_call(self, env, node, call_node): """ Resolve an extension method of a static (C++/Cython-like) vtable: typedef { double (*method1)(double); ... } vtab_struct; vtab_struct *vtab = *(vtab_struct **) (((char *) obj) + vtab_offset) void *method = vtab[index] """ # Make the object we call the method on clone-able node.value = nodes.CloneableNode(node.value) ext_type = node.value.type offset = ext_type.vtab_offset vtable_struct = ext_type.vtab_type.to_struct() vtable_struct_type = vtable_struct.ref() vtab_struct_pointer_pointer = nodes.value_at_offset( node.value, offset, vtable_struct_type.pointer()) vtab_struct_pointer = nodes.DereferenceNode( vtab_struct_pointer_pointer) vmethod = nodes.StructAttribute(vtab_struct_pointer, node.attr, ast.Load(), vtable_struct_type) # Insert first argument 'self' in args list args = call_node.args args.insert(0, nodes.CloneNode(node.value)) result = nodes.NativeFunctionCallNode(node.type, vmethod, args) return result
def extract(complex_node): complex_node = nodes.CloneableNode(complex_node) real = nodes.ComplexAttributeNode(complex_node, 'real') imag = nodes.ComplexAttributeNode(complex_node.clone, 'imag') return real, imag
def getiter(self, context, for_node, llvm_module): iterator = function_util.external_call(context, llvm_module, self.getiter_func, args=[for_node.iter]) iterator = nodes.CloneableNode(iterator) self.iterator = iterator.clone return iterator
def next(self, context, for_node, llvm_module): "Index element and update index" index = self.index.load value = nodes.CloneableNode(index(for_node.iter, index)) add = ast.BinOp(index, ast.Add(), nodes.const(1, Py_ssize_t)) return nodes.ExpressionNode( stmts=[value, assign(self.index.store, add)], expr=value.clone)
def visit_PyErr_OccurredNode(self, node): check_err = nodes.CheckErrorNode(nodes.ptrtoint( function_util.external_call(self.context, self.llvm_module, 'PyErr_Occurred')), goodval=nodes.ptrtoint(nodes.NULL)) result = nodes.CloneableNode(node.node) result = nodes.ExpressionNode(stmts=[result, check_err], expr=result.clone) return self.visit(result)
def visit_ListComp(self, node): """ Rewrite list comprehensions to the equivalent for loops. AST syntax: ListComp(expr elt, comprehension* generators) comprehension = (expr target, expr iter, expr* ifs) 'ifs' represent a chain of ANDs """ assert len(node.generators) > 0 # Create innermost body, i.e. list.append(expr) # TODO: size hint for PyList_New list_create = ast.List(elts=[], ctx=ast.Load()) list_create.type = typesystem.object_ # typesystem.list_() list_create = nodes.CloneableNode(list_create) list_value = nodes.CloneNode(list_create) list_append = ast.Attribute(list_value, "append", ast.Load()) append_call = ast.Call(func=list_append, args=[node.elt], keywords=[], starargs=None, kwargs=None) # Build up the loops from inwards to outwards body = append_call for comprehension in reversed(node.generators): # Hanlde the 'if' clause ifs = comprehension.ifs if len(ifs) > 1: make_boolop = lambda op1_op2: ast.BoolOp(op=ast.And(), values=op1_op2) if_test = reduce(make_boolop, ifs) elif len(ifs) == 1: if_test, = ifs else: if_test = None if if_test is not None: body = ast.If(test=if_test, body=[body], orelse=[]) # Wrap list.append() call or inner loops body = ast.For(target=comprehension.target, iter=comprehension.iter, body=[body], orelse=[]) expr = nodes.ExpressionNode(stmts=[list_create, body], expr=list_value) return self.visit(expr)
def __init__(self, type, value, subslices, nopython, **kwargs): super(NativeSliceNode, self).__init__(**kwargs) value = nodes.CloneableNode(value) self.type = type self.value = value self.subslices = subslices self.shape_type = numba.carray(npy_intp, type.ndim) self.nopython = nopython if not nopython: self.build_array_node = self.build_array() else: self.build_array_node = None
def visit_CoerceToObject(self, node): new_node = node node_type = node.node.type if node_type.is_bool: new_node = function_util.external_call(self.context, self.llvm_module, "PyBool_FromLong", args=[node.node]) elif node_type.is_numeric: cls = None args = node.node, if node_type.is_int: cls = self._get_int_conversion_func(node_type, pyapi._from_long) elif node_type.is_float: cls = pyapi.PyFloat_FromDouble elif node_type.is_complex: cls = pyapi.PyComplex_FromDoubles complex_value = nodes.CloneableNode(node.node) args = [ nodes.ComplexAttributeNode(complex_value, "real"), nodes.ComplexAttributeNode(complex_value.clone, "imag") ] else: raise error.NumbaError( node, "Don't know how to coerce type %r to PyObject" % node_type) if cls: new_node = function_util.external_call(self.context, self.llvm_module, cls.__name__, args=args) elif node_type.is_pointer and not node_type.is_string(): # Create ctypes pointer object ctypes_pointer_type = node_type.to_ctypes() args = [ nodes.CoercionNode(node.node, int64), nodes.ObjectInjectNode(ctypes_pointer_type, object_) ] new_node = nodes.call_pyfunc(ctypes.cast, args) self.generic_visit(new_node) return new_node
def visit_Compare(self, node): "Reduce cascaded comparisons into single comparisons" # Process children self.generic_visit(node) compare_nodes = [] comparators = [nodes.CloneableNode(c) for c in node.comparators] if len(node.comparators) > 1: if node.type.is_array: raise error.NumbaError( node, "Cannot determine truth value of boolean array " "(use any or all)") # Build comparison nodes left = node.left for op, right in zip(node.ops, comparators): node = ast.Compare(left=left, ops=[op], comparators=[right]) # Set result type of comparison: # bool array of array comparison # bool otherwise if left.type.is_array or right.type.is_array: # array < x -> Array(bool_, array.ndim) result_type = self.env.crnt.typesystem.promote( left.type, right.type) else: result_type = bool_ nodes.typednode(node, result_type) # Handle comparisons specially based on their types node = self.single_compare(node) compare_nodes.append(node) left = right.clone # AND the comparisons together node = reduce(build_boolop, reversed(compare_nodes)) return node
def visit_ArrayNewNode(self, node): if self.nopython: # Give the codegen (subclass) a chance to handle this self.generic_visit(node) return node PyArray_Type = nodes.ObjectInjectNode(np.ndarray) descr = nodes.ObjectInjectNode(node.type.dtype.get_dtype()).cloneable ndim = nodes.const(node.type.ndim, int_) flags = nodes.const(0, int_) args = [ PyArray_Type, descr.clone, ndim, node.shape, node.strides, node.data, flags ] incref_descr = nodes.IncrefNode(descr) incref_base = None setbase = None if node.base is None: args.append(nodes.NULL_obj) else: base = nodes.CloneableNode(node.base) incref_base = nodes.IncrefNode(base) args.append(base.clone) array = nodes.PyArray_NewFromDescr(args) array = nodes.ObjectTempNode(array).cloneable body = [incref_descr, incref_base, array, setbase] if node.base is not None: body.append(nodes.PyArray_SetBaseObject([array.clone, base.clone])) # TODO: PyArray_UpdateFlags() result = nodes.ExpressionNode(filter(None, body), array.clone) return self.visit(result)
def handle_method_call(self, env, node, call_node): """ Resolve an extension method of a dynamic hash-based vtable: PyCustomSlots_Table ***vtab_slot = (((char *) obj) + vtab_offset) lookup_virtual_method(*vtab_slot) We may cache (*vtab_slot), but we may not cache (**vtab_slot), since compilations may regenerate the table. However, we could *preload* (**vtab_slot), where function calls invalidate the preload, if we were so inclined. """ # Make the object we call the method on clone-able node.value = nodes.CloneableNode(node.value) ext_type = node.ext_type func_signature = node.type #typesystem.extmethod_to_function(node.type) offset = ext_type.vtab_offset # __________________________________________________________________ # Retrieve vtab vtab_ppp = nodes.value_at_offset(node.value, offset, void.pointer().pointer()) vtab_struct_pp = nodes.DereferenceNode(vtab_ppp) # __________________________________________________________________ # Calculate pre-hash prehash = virtual.hash_signature(func_signature, func_signature.name) prehash_node = nodes.ConstNode(prehash, uint64) # __________________________________________________________________ # Retrieve method pointer # A method is always present when it was given a static signature, # e.g. @double(double) always_present = node.attr in ext_type.vtab_type.methodnames args = [vtab_struct_pp, prehash_node] # lookup_impl = NumbaVirtualLookup() lookup_impl = DebugVirtualLookup() ptr = lookup_impl.lookup(env, always_present, node, args) vmethod = ptr.coerce(func_signature.pointer()) vmethod = vmethod.cloneable # __________________________________________________________________ # Call method pointer # Insert first argument 'self' in args list args = call_node.args args.insert(0, nodes.CloneNode(node.value)) method_call = nodes.NativeFunctionCallNode(func_signature, vmethod, args) # __________________________________________________________________ # Generate fallback # TODO: Subclassing! # if not always_present: # # TODO: Enable this path and generate a phi for the result # # Generate object call # obj_args = [nodes.CoercionNode(arg, object_) for arg in args] # obj_args.append(nodes.NULL) # object_call = function_util.external_call( # env.context, env.crnt.llvm_module, # 'PyObject_CallMethodObjArgs', obj_args) # # # if vmethod != NULL: vmethod(obj, ...) # # else: obj.method(...) # method_call = nodes.if_else( # ast.NotEq(), # vmethod.clone, nodes.NULL, # lhs=method_call, rhs=object_call) return method_call
def __init__(self, substitutions): self.substitutions = substitutions for name, replacement in substitutions.iteritems(): if (not isinstance(replacement, nodes.CloneableNode) and hasattr(replacement, 'type')): substitutions[name] = nodes.CloneableNode(replacement)
def rewrite_range_iteration(self, node): """ Handle range iteration: for i in range(start, stop, step): ... becomes nsteps = compute_nsteps(start, stop, step) temp = 0 while temp < nsteps: target = start + temp * step ... temp += 1 """ self.generic_visit(node) temp = nodes.TempNode(node.target.type, 'target_temp') nsteps = nodes.TempNode(Py_ssize_t, 'nsteps') start, stop, step = unpack_range_args(node.iter) if isinstance(step, nodes.ConstNode): have_step = step.pyval != 1 else: have_step = True start, stop, step = [ nodes.CloneableNode(n) for n in (start, stop, step) ] if have_step: compute_nsteps = """ $length = {{stop}} - {{start}} {{nsteps}} = $length / {{step}} if {{nsteps_load}} * {{step}} != $length: #$length % {{step}}: # Test for truncation {{nsteps}} = {{nsteps_load}} + 1 # print "nsteps", {{nsteps_load}} """ else: compute_nsteps = "{{nsteps}} = {{stop}} - {{start}}" if node.orelse: else_clause = "else: {{else_body}}" else: else_clause = "" templ = textwrap.dedent(""" %s {{temp}} = 0 while {{temp_load}} < {{nsteps_load}}: {{target}} = {{start}} + {{temp_load}} * {{step}} {{body}} {{temp}} = {{temp_load}} + 1 %s """) % (textwrap.dedent(compute_nsteps), else_clause) # Leave the bodies empty, they are already analyzed body = ast.Suite(body=[]) else_body = ast.Suite(body=[]) #-------------------------------------------------------------------- # Substitute template and infer types #-------------------------------------------------------------------- result = self.run_template(templ, vars=dict(length=Py_ssize_t), start=start, stop=stop, step=step, nsteps=nsteps.store(), nsteps_load=nsteps.load(), temp=temp.store(), temp_load=temp.load(), target=node.target, body=body, else_body=else_body) #-------------------------------------------------------------------- # Patch the body and else clause #-------------------------------------------------------------------- body.body.extend(node.body) else_body.body.extend(node.orelse) while_node = result.body[-1] assert isinstance(while_node, ast.While) target_increment = while_node.body[-1] assert isinstance(target_increment, ast.Assign) # Add target variable increment basic block node.incr_block.body = [target_increment] while_node.body[-1] = node.incr_block #-------------------------------------------------------------------- # Create a While with the ForNode's cfg blocks merged in #-------------------------------------------------------------------- while_node = make_while_loop(while_node) copy_basic_blocks(node, while_node) while_node = nodes.build_while(**vars(while_node)) # Create the place to jump to for 'continue' while_node.continue_block = node.incr_block # Set the new while loop in the templated Suite result.body[-1] = while_node return result
def visit_CoerceToNative(self, node): """ Try to perform fast coercion using e.g. PyLong_AsLong(), with a fallback to PyArg_ParseTuple(). """ new_node = None from_type = node.node.type node_type = node.type if node_type.is_numeric: cls = None if node_type == size_t: node_type = ulonglong if node_type.is_int: # and not new_node = self.object_to_int(node.node, node_type) elif node_type.is_float: cls = pyapi.PyFloat_AsDouble elif node_type.is_complex: # FIXME: This conversion has to be pretty slow. We # need to move towards being ABI-savvy enough to just # call PyComplex_AsCComplex(). cloneable = nodes.CloneableNode(node.node) new_node = nodes.ComplexNode( real=function_util.external_call(self.context, self.llvm_module, "PyComplex_RealAsDouble", args=[cloneable]), imag=function_util.external_call(self.context, self.llvm_module, "PyComplex_ImagAsDouble", args=[cloneable.clone])) else: raise error.NumbaError( node, "Don't know how to coerce a Python object to a %r" % node_type) if cls: # TODO: error checking! new_node = function_util.external_call(self.context, self.llvm_module, cls.__name__, args=[node.node]) elif node_type.is_pointer: if from_type.is_jit_function and node_type.base_type.is_function: new_node = self.coerce_to_function_pointer( node, from_type, node_type) else: raise error.NumbaError( node, "Obtaining pointers from objects " "is not yet supported") elif node_type.is_void: raise error.NumbaError(node, "Cannot coerce %s to void" % (from_type, )) if new_node is None: # Create a tuple for PyArg_ParseTuple new_node = node new_node.node = ast.Tuple(elts=[node.node], ctx=ast.Load()) self.generic_visit(node) return node if new_node.type != node.type: # Fast native coercion. E.g. coercing an object to an int_ # will use PyLong_AsLong, but that will return a long_. We # need to coerce the long_ to an int_ new_node = nodes.CoercionNode(new_node, node.type) # Specialize replacement node new_node = self.visit(new_node) return new_node
def register_array_expression(self, node, lhs=None): super(ArrayExpressionRewriteNative, self).register_array_expression( node, lhs) lhs_type = lhs.type if lhs else node.type is_expr = lhs is None if node.type.is_array and lhs_type.ndim < node.type.ndim: # TODO: this is valid in NumPy if the leading dimensions of the # TODO: RHS have extent 1 raise error.NumbaError( node, "Right hand side must have a " "dimensionality <= %d" % lhs_type.ndim) # Create ufunc scalar kernel ufunc_ast, signature, ufunc_builder = self.get_py_ufunc_ast(lhs, node) signature.struct_by_reference = True # Compile ufunc scalar kernel with numba ast.fix_missing_locations(ufunc_ast) func_env, (_, _, _) = pipeline.run_pipeline2( self.env, None, ufunc_ast, signature, function_globals={}, ) # Manual linking lfunc = func_env.lfunc # print lfunc operands = ufunc_builder.operands functions.keep_alive(self.func, lfunc) operands = [nodes.CloneableNode(operand) for operand in operands] if lhs is not None: lhs = nodes.CloneableNode(lhs) broadcast_operands = [lhs] + operands lhs = lhs.clone else: broadcast_operands = operands[:] shape = slicenodes.BroadcastNode(lhs_type, broadcast_operands) operands = [op.clone for op in operands] if lhs is None and self.nopython: raise error.NumbaError( node, "Cannot allocate new memory in nopython context") elif lhs is None: # TODO: determine best output order at runtime shape = shape.cloneable lhs = nodes.ArrayNewEmptyNode(lhs_type, shape.clone, lhs_type.is_f_contig).cloneable # Build minivect wrapper kernel context = NumbaproStaticArgsContext() context.llvm_module = self.env.llvm_context.module # context.debug = True context.optimize_broadcasting = False b = context.astbuilder variables = [b.variable(name_node.type, "op%d" % i) for i, name_node in enumerate([lhs] + operands)] miniargs = [b.funcarg(variable) for variable in variables] body = miniutils.build_kernel_call(lfunc.name, signature, miniargs, b) minikernel = b.function_from_numpy( templating.temp_name("array_expression"), body, miniargs) lminikernel, ctypes_kernel = context.run_simple( minikernel, specializers.StridedSpecializer) # Build call to minivect kernel operands.insert(0, lhs) args = [shape] scalar_args = [] for operand in operands: if operand.type.is_array: data_p = self.array_attr(operand, 'data') data_p = nodes.CoercionNode(data_p, operand.type.dtype.pointer()) if not isinstance(operand, nodes.CloneNode): operand = nodes.CloneNode(operand) strides_p = self.array_attr(operand, 'strides') args.extend((data_p, strides_p)) else: scalar_args.append(operand) args.extend(scalar_args) result = nodes.NativeCallNode(minikernel.type, args, lminikernel) # Use native slicing in array expressions slicenodes.mark_nopython(ast.Suite(body=result.args)) if not is_expr: # a[:] = b[:] * c[:] return result # b[:] * c[:], return new array as expression return nodes.ExpressionNode(stmts=[result], expr=lhs.clone)