def __init__(self, typingctx, targetctx, library, args, return_type, flags, locals): # Make sure the environment is reloaded config.reload_config() typingctx.refresh() targetctx.refresh() self.typingctx = typingctx self.targetctx = _make_subtarget(targetctx, flags) self.library = library self.args = args self.return_type = return_type self.flags = flags self.locals = locals # Results of various steps of the compilation pipeline self.bc = None self.func_id = None self.func_ir = None self.func_ir_original = None # used for fallback self.lifted = None self.lifted_from = None self.typemap = None self.calltypes = None self.type_annotation = None self.metadata = {} # holds arbitrary inter-pipeline stage meta data # parfor diagnostics info, add to metadata self.parfor_diagnostics = ParforDiagnostics() self.metadata['parfor_diagnostics'] = self.parfor_diagnostics self.status = _CompileStatus( can_fallback=self.flags.enable_pyobject, can_giveup=config.COMPATIBILITY_MODE )
def __init__(self, typingctx, targetctx, library, args, return_type, flags, locals): # Make sure the environment is reloaded config.reload_config() typingctx.refresh() targetctx.refresh() self.typingctx = typingctx self.targetctx = _make_subtarget(targetctx, flags) self.library = library self.args = args self.return_type = return_type self.flags = flags self.locals = locals # Results of various steps of the compilation pipeline self.bc = None self.func_id = None self.func_ir = None self.func_ir_original = None # used for fallback self.lifted = None self.lifted_from = None self.typemap = None self.calltypes = None self.type_annotation = None self.metadata = {} # holds arbitrary inter-pipeline stage meta data # parfor diagnostics info, add to metadata self.parfor_diagnostics = ParforDiagnostics() self.metadata['parfor_diagnostics'] = self.parfor_diagnostics self.status = _CompileStatus( can_fallback=self.flags.enable_pyobject, can_giveup=config.COMPATIBILITY_MODE )
class BasePipeline(object): """ Stores and manages states for the compiler pipeline """ def __init__(self, typingctx, targetctx, library, args, return_type, flags, locals): # Make sure the environment is reloaded config.reload_config() typingctx.refresh() targetctx.refresh() self.typingctx = typingctx self.targetctx = _make_subtarget(targetctx, flags) self.library = library self.args = args self.return_type = return_type self.flags = flags self.locals = locals # Results of various steps of the compilation pipeline self.bc = None self.func_id = None self.func_ir = None self.func_ir_original = None # used for fallback self.lifted = None self.lifted_from = None self.typemap = None self.calltypes = None self.type_annotation = None self.metadata = {} # holds arbitrary inter-pipeline stage meta data # parfor diagnostics info, add to metadata self.parfor_diagnostics = ParforDiagnostics() self.metadata['parfor_diagnostics'] = self.parfor_diagnostics self.status = _CompileStatus( can_fallback=self.flags.enable_pyobject, can_giveup=config.COMPATIBILITY_MODE ) @contextmanager def fallback_context(self, msg): """ Wraps code that would signal a fallback to object mode """ try: yield except BaseException as e: if not self.status.can_fallback: raise else: if utils.PYVERSION >= (3,): # Clear all references attached to the traceback e = e.with_traceback(None) warnings.warn_explicit('%s: %s' % (msg, e), errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) raise @contextmanager def giveup_context(self, msg): """ Wraps code that would signal a fallback to interpreter mode """ try: yield except BaseException as e: if not self.status.can_giveup: raise else: if utils.PYVERSION >= (3,): # Clear all references attached to the traceback e = e.with_traceback(None) warnings.warn_explicit('%s: %s' % (msg, e), errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) raise def extract_bytecode(self, func_id): """ Extract bytecode from function """ bc = bytecode.ByteCode(func_id) if config.DUMP_BYTECODE: print(bc.dump()) return bc def compile_extra(self, func): self.func_id = bytecode.FunctionIdentity.from_function(func) try: bc = self.extract_bytecode(self.func_id) except BaseException as e: if self.status.can_giveup: self.stage_compile_interp_mode() return self.cr else: raise e self.bc = bc self.lifted = () self.lifted_from = None return self._compile_bytecode() def compile_ir(self, func_ir, lifted=(), lifted_from=None): self.func_id = func_ir.func_id self.lifted = lifted self.lifted_from = lifted_from self._set_and_check_ir(func_ir) return self._compile_ir() def stage_analyze_bytecode(self): """ Analyze bytecode and translating to Numba IR """ func_ir = translate_stage(self.func_id, self.bc) self._set_and_check_ir(func_ir) def _set_and_check_ir(self, func_ir): self.func_ir = func_ir self.nargs = self.func_ir.arg_count if not self.args and self.flags.force_pyobject: # Allow an empty argument types specification when object mode # is explicitly requested. self.args = (types.pyobject,) * self.nargs elif len(self.args) != self.nargs: raise TypeError("Signature mismatch: %d argument types given, " "but function takes %d arguments" % (len(self.args), self.nargs)) def stage_process_ir(self): ir_processing_stage(self.func_ir) def stage_preserve_ir(self): self.func_ir_original = self.func_ir.copy() def frontend_looplift(self): """ Loop lifting analysis and transformation """ loop_flags = self.flags.copy() outer_flags = self.flags.copy() # Do not recursively loop lift outer_flags.unset('enable_looplift') loop_flags.unset('enable_looplift') if not self.flags.enable_pyobject_looplift: loop_flags.unset('enable_pyobject') main, loops = transforms.loop_lifting(self.func_ir, typingctx=self.typingctx, targetctx=self.targetctx, locals=self.locals, flags=loop_flags) if loops: # Some loops were extracted if config.DEBUG_FRONTEND or config.DEBUG: for loop in loops: print("Lifting loop", loop.get_source_location()) cres = compile_ir(self.typingctx, self.targetctx, main, self.args, self.return_type, outer_flags, self.locals, lifted=tuple(loops), lifted_from=None) return cres def stage_frontend_withlift(self): """ Extract with-contexts """ main, withs = transforms.with_lifting( func_ir=self.func_ir, typingctx=self.typingctx, targetctx=self.targetctx, flags=self.flags, locals=self.locals, ) if withs: cres = compile_ir(self.typingctx, self.targetctx, main, self.args, self.return_type, self.flags, self.locals, lifted=tuple(withs), lifted_from=None, pipeline_class=type(self)) raise _EarlyPipelineCompletion(cres) def stage_objectmode_frontend(self): """ Front-end: Analyze bytecode, generate Numba IR, infer types """ self.func_ir = self.func_ir_original or self.func_ir if self.flags.enable_looplift: assert not self.lifted cres = self.frontend_looplift() if cres is not None: raise _EarlyPipelineCompletion(cres) # Fallback typing: everything is a python object self.typemap = defaultdict(lambda: types.pyobject) self.calltypes = defaultdict(lambda: types.pyobject) self.return_type = types.pyobject def stage_nopython_frontend(self): """ Type inference and legalization """ with self.fallback_context('Function "%s" failed type inference' % (self.func_id.func_name,)): # Type inference typemap, return_type, calltypes = type_inference_stage( self.typingctx, self.func_ir, self.args, self.return_type, self.locals) self.typemap = typemap self.return_type = return_type self.calltypes = calltypes with self.fallback_context('Function "%s" has invalid return type' % (self.func_id.func_name,)): legalize_return_type(self.return_type, self.func_ir, self.targetctx) def stage_generic_rewrites(self): """ Perform any intermediate representation rewrites before type inference. """ assert self.func_ir msg = ('Internal error in pre-inference rewriting ' 'pass encountered during compilation of ' 'function "%s"' % (self.func_id.func_name,)) with self.fallback_context(msg): rewrites.rewrite_registry.apply('before-inference', self, self.func_ir) def stage_nopython_rewrites(self): """ Perform any intermediate representation rewrites after type inference. """ # Ensure we have an IR and type information. assert self.func_ir assert isinstance(getattr(self, 'typemap', None), dict) assert isinstance(getattr(self, 'calltypes', None), dict) msg = ('Internal error in post-inference rewriting ' 'pass encountered during compilation of ' 'function "%s"' % (self.func_id.func_name,)) with self.fallback_context(msg): rewrites.rewrite_registry.apply('after-inference', self, self.func_ir) def stage_pre_parfor_pass(self): """ Preprocessing for data-parallel computations. """ # Ensure we have an IR and type information. assert self.func_ir preparfor_pass = PreParforPass( self.func_ir, self.type_annotation.typemap, self.type_annotation.calltypes, self.typingctx, self.flags.auto_parallel, self.parfor_diagnostics.replaced_fns ) preparfor_pass.run() def stage_parfor_pass(self): """ Convert data-parallel computations into Parfor nodes """ # Ensure we have an IR and type information. assert self.func_ir parfor_pass = ParforPass(self.func_ir, self.type_annotation.typemap, self.type_annotation.calltypes, self.return_type, self.typingctx, self.flags.auto_parallel, self.flags, self.parfor_diagnostics) parfor_pass.run() if config.WARNINGS: # check the parfor pass worked and warn if it didn't has_parfor = False for blk in self.func_ir.blocks.values(): for stmnt in blk.body: if isinstance(stmnt, Parfor): has_parfor = True break else: continue break if not has_parfor: # parfor calls the compiler chain again with a string if not self.func_ir.loc.filename == '<string>': msg = ("parallel=True was specified but no transformation" " for parallel execution was possible.") warnings.warn_explicit( msg, errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno ) def stage_inline_pass(self): """ Inline calls to locally defined closures. """ # Ensure we have an IR and type information. assert self.func_ir inline_pass = InlineClosureCallPass(self.func_ir, self.flags.auto_parallel, self.parfor_diagnostics.replaced_fns) inline_pass.run() # Remove all Dels, and re-run postproc post_proc = postproc.PostProcessor(self.func_ir) post_proc.run() if config.DEBUG or config.DUMP_IR: name = self.func_ir.func_id.func_qualname print(("IR DUMP: %s" % name).center(80, "-")) self.func_ir.dump() def stage_annotate_type(self): """ Create type annotation after type inference """ self.type_annotation = type_annotations.TypeAnnotation( func_ir=self.func_ir, typemap=self.typemap, calltypes=self.calltypes, lifted=self.lifted, lifted_from=self.lifted_from, args=self.args, return_type=self.return_type, html_output=config.HTML) if config.ANNOTATE: print("ANNOTATION".center(80, '-')) print(self.type_annotation) print('=' * 80) if config.HTML: with open(config.HTML, 'w') as fout: self.type_annotation.html_annotate(fout) def stage_dump_diagnostics(self): if self.flags.auto_parallel.enabled: if config.PARALLEL_DIAGNOSTICS: if self.parfor_diagnostics is not None: self.parfor_diagnostics.dump(config.PARALLEL_DIAGNOSTICS) else: raise RuntimeError("Diagnostics failed.") def backend_object_mode(self): """ Object mode compilation """ with self.giveup_context("Function %s failed at object mode lowering" % (self.func_id.func_name,)): if len(self.args) != self.nargs: # append missing self.args = (tuple(self.args) + (types.pyobject,) * (self.nargs - len(self.args))) return py_lowering_stage(self.targetctx, self.library, self.func_ir, self.flags) def backend_nopython_mode(self): """Native mode compilation""" msg = ("Function %s failed at nopython " "mode lowering" % (self.func_id.func_name,)) with self.fallback_context(msg): return native_lowering_stage( self.targetctx, self.library, self.func_ir, self.typemap, self.return_type, self.calltypes, self.flags, self.metadata) def _backend(self, lowerfn, objectmode): """ Back-end: Generate LLVM IR from Numba IR, compile to machine code """ if self.library is None: codegen = self.targetctx.codegen() self.library = codegen.create_library(self.func_id.func_qualname) # Enable object caching upfront, so that the library can # be later serialized. self.library.enable_object_caching() lowered = lowerfn() signature = typing.signature(self.return_type, *self.args) self.cr = compile_result( typing_context=self.typingctx, target_context=self.targetctx, entry_point=lowered.cfunc, typing_error=self.status.fail_reason, type_annotation=self.type_annotation, library=self.library, call_helper=lowered.call_helper, signature=signature, objectmode=objectmode, interpmode=False, lifted=self.lifted, fndesc=lowered.fndesc, environment=lowered.env, has_dynamic_globals=lowered.has_dynamic_globals, metadata=self.metadata, ) def stage_objectmode_backend(self): """ Lowering for object mode """ lowerfn = self.backend_object_mode self._backend(lowerfn, objectmode=True) # Warn if compiled function in object mode and force_pyobject not set if not self.flags.force_pyobject: if len(self.lifted) > 0: warn_msg = ('Function "%s" was compiled in object mode without' ' forceobj=True, but has lifted loops.' % (self.func_id.func_name,)) else: warn_msg = ('Function "%s" was compiled in object mode without' ' forceobj=True.' % (self.func_id.func_name,)) warnings.warn_explicit(warn_msg, errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) if self.flags.release_gil: warn_msg = ("Code running in object mode won't allow parallel" " execution despite nogil=True.") warnings.warn_explicit(warn_msg, errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) def stage_nopython_backend(self): """ Do lowering for nopython """ lowerfn = self.backend_nopython_mode self._backend(lowerfn, objectmode=False) def stage_compile_interp_mode(self): """ Just create a compile result for interpreter mode """ args = [types.pyobject] * len(self.args) signature = typing.signature(types.pyobject, *args) self.cr = compile_result(typing_context=self.typingctx, target_context=self.targetctx, entry_point=self.func_id.func, typing_error=self.status.fail_reason, type_annotation="<Interpreter mode function>", signature=signature, objectmode=False, interpmode=True, lifted=(), fndesc=None,) def stage_ir_legalization(self): raise_on_unsupported_feature(self.func_ir) def stage_cleanup(self): """ Cleanup intermediate results to release resources. """ def define_pipelines(self, pm): """Child classes override this to customize the pipeline. """ raise NotImplementedError() def add_preprocessing_stage(self, pm): """Add the preprocessing stage that analyzes the bytecode to prepare the Numba IR. """ if self.func_ir is None: pm.add_stage(self.stage_analyze_bytecode, "analyzing bytecode") pm.add_stage(self.stage_process_ir, "processing IR") def add_pre_typing_stage(self, pm): """Add any stages that go before type-inference. The current stages contain type-agnostic rewrite passes. """ if not self.flags.no_rewrites: if self.status.can_fallback: pm.add_stage(self.stage_preserve_ir, "preserve IR for fallback") pm.add_stage(self.stage_generic_rewrites, "nopython rewrites") pm.add_stage(self.stage_inline_pass, "inline calls to locally defined closures") def add_typing_stage(self, pm): """Add the type-inference stage necessary for nopython mode. """ pm.add_stage(self.stage_nopython_frontend, "nopython frontend") pm.add_stage(self.stage_annotate_type, "annotate type") def add_optimization_stage(self, pm): """Add optimization stages. """ if self.flags.auto_parallel.enabled: pm.add_stage(self.stage_pre_parfor_pass, "Preprocessing for parfors") if not self.flags.no_rewrites: pm.add_stage(self.stage_nopython_rewrites, "nopython rewrites") if self.flags.auto_parallel.enabled: pm.add_stage(self.stage_parfor_pass, "convert to parfors") def add_lowering_stage(self, pm): """Add the lowering (code-generation) stage for nopython-mode """ pm.add_stage(self.stage_nopython_backend, "nopython mode backend") def add_cleanup_stage(self, pm): """Add the clean-up stage to remove intermediate results. """ pm.add_stage(self.stage_cleanup, "cleanup intermediate results") def add_with_handling_stage(self, pm): pm.add_stage(self.stage_frontend_withlift, "Handle with contexts") def define_nopython_pipeline(self, pm, name='nopython'): """Add the nopython-mode pipeline to the pipeline manager """ pm.create_pipeline(name) self.add_preprocessing_stage(pm) self.add_with_handling_stage(pm) self.add_pre_typing_stage(pm) self.add_typing_stage(pm) self.add_optimization_stage(pm) pm.add_stage(self.stage_ir_legalization, "ensure IR is legal prior to lowering") self.add_lowering_stage(pm) pm.add_stage(self.stage_dump_diagnostics, "dump diagnostics") self.add_cleanup_stage(pm) def define_objectmode_pipeline(self, pm, name='object'): """Add the object-mode pipeline to the pipeline manager """ pm.create_pipeline(name) self.add_preprocessing_stage(pm) pm.add_stage(self.stage_objectmode_frontend, "object mode frontend") pm.add_stage(self.stage_inline_pass, "inline calls to locally defined closures") pm.add_stage(self.stage_annotate_type, "annotate type") pm.add_stage(self.stage_ir_legalization, "ensure IR is legal prior to lowering") pm.add_stage(self.stage_objectmode_backend, "object mode backend") self.add_cleanup_stage(pm) def define_interpreted_pipeline(self, pm, name="interp"): """Add the interpreted-mode (fallback) pipeline to the pipeline manager """ pm.create_pipeline(name) pm.add_stage(self.stage_compile_interp_mode, "compiling with interpreter mode") self.add_cleanup_stage(pm) def _compile_core(self): """ Populate and run compiler pipeline """ pm = _PipelineManager() self.define_pipelines(pm) pm.finalize() res = pm.run(self.status) if res is not None: # Early pipeline completion return res else: assert self.cr is not None return self.cr def _compile_bytecode(self): """ Populate and run pipeline for bytecode input """ assert self.func_ir is None return self._compile_core() def _compile_ir(self): """ Populate and run pipeline for IR input """ assert self.func_ir is not None return self._compile_core()
class BasePipeline(object): """ Stores and manages states for the compiler pipeline """ def __init__(self, typingctx, targetctx, library, args, return_type, flags, locals): # Make sure the environment is reloaded config.reload_config() typingctx.refresh() targetctx.refresh() self.typingctx = typingctx self.targetctx = _make_subtarget(targetctx, flags) self.library = library self.args = args self.return_type = return_type self.flags = flags self.locals = locals # Results of various steps of the compilation pipeline self.bc = None self.func_id = None self.func_ir = None self.func_ir_original = None # used for fallback self.lifted = None self.lifted_from = None self.typemap = None self.calltypes = None self.type_annotation = None self.metadata = {} # holds arbitrary inter-pipeline stage meta data # parfor diagnostics info, add to metadata self.parfor_diagnostics = ParforDiagnostics() self.metadata['parfor_diagnostics'] = self.parfor_diagnostics self.status = _CompileStatus( can_fallback=self.flags.enable_pyobject, can_giveup=config.COMPATIBILITY_MODE ) @contextmanager def fallback_context(self, msg): """ Wraps code that would signal a fallback to object mode """ try: yield except BaseException as e: if not self.status.can_fallback: raise else: if utils.PYVERSION >= (3,): # Clear all references attached to the traceback e = e.with_traceback(None) warnings.warn_explicit('%s: %s' % (msg, e), errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) raise @contextmanager def giveup_context(self, msg): """ Wraps code that would signal a fallback to interpreter mode """ try: yield except BaseException as e: if not self.status.can_giveup: raise else: if utils.PYVERSION >= (3,): # Clear all references attached to the traceback e = e.with_traceback(None) warnings.warn_explicit('%s: %s' % (msg, e), errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) raise def extract_bytecode(self, func_id): """ Extract bytecode from function """ bc = bytecode.ByteCode(func_id) if config.DUMP_BYTECODE: print(bc.dump()) return bc def compile_extra(self, func): self.func_id = bytecode.FunctionIdentity.from_function(func) try: bc = self.extract_bytecode(self.func_id) except BaseException as e: if self.status.can_giveup: self.stage_compile_interp_mode() return self.cr else: raise e self.bc = bc self.lifted = () self.lifted_from = None return self._compile_bytecode() def compile_ir(self, func_ir, lifted=(), lifted_from=None): self.func_id = func_ir.func_id self.lifted = lifted self.lifted_from = lifted_from self._set_and_check_ir(func_ir) return self._compile_ir() def stage_analyze_bytecode(self): """ Analyze bytecode and translating to Numba IR """ func_ir = translate_stage(self.func_id, self.bc) self._set_and_check_ir(func_ir) def _set_and_check_ir(self, func_ir): self.func_ir = func_ir self.nargs = self.func_ir.arg_count if not self.args and self.flags.force_pyobject: # Allow an empty argument types specification when object mode # is explicitly requested. self.args = (types.pyobject,) * self.nargs elif len(self.args) != self.nargs: raise TypeError("Signature mismatch: %d argument types given, " "but function takes %d arguments" % (len(self.args), self.nargs)) def stage_process_ir(self): ir_processing_stage(self.func_ir) def stage_preserve_ir(self): self.func_ir_original = self.func_ir.copy() def frontend_looplift(self): """ Loop lifting analysis and transformation """ loop_flags = self.flags.copy() outer_flags = self.flags.copy() # Do not recursively loop lift outer_flags.unset('enable_looplift') loop_flags.unset('enable_looplift') if not self.flags.enable_pyobject_looplift: loop_flags.unset('enable_pyobject') main, loops = transforms.loop_lifting(self.func_ir, typingctx=self.typingctx, targetctx=self.targetctx, locals=self.locals, flags=loop_flags) if loops: # Some loops were extracted if config.DEBUG_FRONTEND or config.DEBUG: for loop in loops: print("Lifting loop", loop.get_source_location()) cres = compile_ir(self.typingctx, self.targetctx, main, self.args, self.return_type, outer_flags, self.locals, lifted=tuple(loops), lifted_from=None) return cres def stage_frontend_withlift(self): """ Extract with-contexts """ main, withs = transforms.with_lifting( func_ir=self.func_ir, typingctx=self.typingctx, targetctx=self.targetctx, flags=self.flags, locals=self.locals, ) if withs: cres = compile_ir(self.typingctx, self.targetctx, main, self.args, self.return_type, self.flags, self.locals, lifted=tuple(withs), lifted_from=None, pipeline_class=type(self)) raise _EarlyPipelineCompletion(cres) def stage_objectmode_frontend(self): """ Front-end: Analyze bytecode, generate Numba IR, infer types """ self.func_ir = self.func_ir_original or self.func_ir if self.flags.enable_looplift: assert not self.lifted cres = self.frontend_looplift() if cres is not None: raise _EarlyPipelineCompletion(cres) # Fallback typing: everything is a python object self.typemap = defaultdict(lambda: types.pyobject) self.calltypes = defaultdict(lambda: types.pyobject) self.return_type = types.pyobject def stage_nopython_frontend(self): """ Type inference and legalization """ with self.fallback_context('Function "%s" failed type inference' % (self.func_id.func_name,)): # Type inference typemap, return_type, calltypes = type_inference_stage( self.typingctx, self.func_ir, self.args, self.return_type, self.locals) self.typemap = typemap self.return_type = return_type self.calltypes = calltypes with self.fallback_context('Function "%s" has invalid return type' % (self.func_id.func_name,)): legalize_return_type(self.return_type, self.func_ir, self.targetctx) def stage_generic_rewrites(self): """ Perform any intermediate representation rewrites before type inference. """ assert self.func_ir msg = ('Internal error in pre-inference rewriting ' 'pass encountered during compilation of ' 'function "%s"' % (self.func_id.func_name,)) with self.fallback_context(msg): rewrites.rewrite_registry.apply('before-inference', self, self.func_ir) def stage_nopython_rewrites(self): """ Perform any intermediate representation rewrites after type inference. """ # Ensure we have an IR and type information. assert self.func_ir assert isinstance(getattr(self, 'typemap', None), dict) assert isinstance(getattr(self, 'calltypes', None), dict) msg = ('Internal error in post-inference rewriting ' 'pass encountered during compilation of ' 'function "%s"' % (self.func_id.func_name,)) with self.fallback_context(msg): rewrites.rewrite_registry.apply('after-inference', self, self.func_ir) def stage_pre_parfor_pass(self): """ Preprocessing for data-parallel computations. """ # Ensure we have an IR and type information. assert self.func_ir preparfor_pass = PreParforPass( self.func_ir, self.type_annotation.typemap, self.type_annotation.calltypes, self.typingctx, self.flags.auto_parallel, self.parfor_diagnostics.replaced_fns ) preparfor_pass.run() def stage_parfor_pass(self): """ Convert data-parallel computations into Parfor nodes """ # Ensure we have an IR and type information. assert self.func_ir parfor_pass = ParforPass(self.func_ir, self.type_annotation.typemap, self.type_annotation.calltypes, self.return_type, self.typingctx, self.flags.auto_parallel, self.flags, self.parfor_diagnostics) parfor_pass.run() if config.WARNINGS: # check the parfor pass worked and warn if it didn't has_parfor = False for blk in self.func_ir.blocks.values(): for stmnt in blk.body: if isinstance(stmnt, Parfor): has_parfor = True break else: continue break if not has_parfor: # parfor calls the compiler chain again with a string if not self.func_ir.loc.filename == '<string>': msg = ("parallel=True was specified but no transformation" " for parallel execution was possible.") warnings.warn_explicit( msg, errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno ) def stage_inline_pass(self): """ Inline calls to locally defined closures. """ # Ensure we have an IR and type information. assert self.func_ir inline_pass = InlineClosureCallPass(self.func_ir, self.flags.auto_parallel, self.parfor_diagnostics.replaced_fns) inline_pass.run() # Remove all Dels, and re-run postproc post_proc = postproc.PostProcessor(self.func_ir) post_proc.run() if config.DEBUG or config.DUMP_IR: name = self.func_ir.func_id.func_qualname print(("IR DUMP: %s" % name).center(80, "-")) self.func_ir.dump() def stage_annotate_type(self): """ Create type annotation after type inference """ self.type_annotation = type_annotations.TypeAnnotation( func_ir=self.func_ir, typemap=self.typemap, calltypes=self.calltypes, lifted=self.lifted, lifted_from=self.lifted_from, args=self.args, return_type=self.return_type, html_output=config.HTML) if config.ANNOTATE: print("ANNOTATION".center(80, '-')) print(self.type_annotation) print('=' * 80) if config.HTML: with open(config.HTML, 'w') as fout: self.type_annotation.html_annotate(fout) def stage_dump_diagnostics(self): if self.flags.auto_parallel.enabled: if config.PARALLEL_DIAGNOSTICS: if self.parfor_diagnostics is not None: self.parfor_diagnostics.dump(config.PARALLEL_DIAGNOSTICS) else: raise RuntimeError("Diagnostics failed.") def backend_object_mode(self): """ Object mode compilation """ with self.giveup_context("Function %s failed at object mode lowering" % (self.func_id.func_name,)): if len(self.args) != self.nargs: # append missing self.args = (tuple(self.args) + (types.pyobject,) * (self.nargs - len(self.args))) return py_lowering_stage(self.targetctx, self.library, self.func_ir, self.flags) def backend_nopython_mode(self): """Native mode compilation""" msg = ("Function %s failed at nopython " "mode lowering" % (self.func_id.func_name,)) with self.fallback_context(msg): return native_lowering_stage( self.targetctx, self.library, self.func_ir, self.typemap, self.return_type, self.calltypes, self.flags, self.metadata) def _backend(self, lowerfn, objectmode): """ Back-end: Generate LLVM IR from Numba IR, compile to machine code """ if self.library is None: codegen = self.targetctx.codegen() self.library = codegen.create_library(self.func_id.func_qualname) # Enable object caching upfront, so that the library can # be later serialized. self.library.enable_object_caching() lowered = lowerfn() signature = typing.signature(self.return_type, *self.args) self.cr = compile_result( typing_context=self.typingctx, target_context=self.targetctx, entry_point=lowered.cfunc, typing_error=self.status.fail_reason, type_annotation=self.type_annotation, library=self.library, call_helper=lowered.call_helper, signature=signature, objectmode=objectmode, interpmode=False, lifted=self.lifted, fndesc=lowered.fndesc, environment=lowered.env, metadata=self.metadata, ) def stage_objectmode_backend(self): """ Lowering for object mode """ lowerfn = self.backend_object_mode self._backend(lowerfn, objectmode=True) # Warn if compiled function in object mode and force_pyobject not set if not self.flags.force_pyobject: if len(self.lifted) > 0: warn_msg = ('Function "%s" was compiled in object mode without' ' forceobj=True, but has lifted loops.' % (self.func_id.func_name,)) else: warn_msg = ('Function "%s" was compiled in object mode without' ' forceobj=True.' % (self.func_id.func_name,)) warnings.warn_explicit(warn_msg, errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) if self.flags.release_gil: warn_msg = ("Code running in object mode won't allow parallel" " execution despite nogil=True.") warnings.warn_explicit(warn_msg, errors.NumbaWarning, self.func_id.filename, self.func_id.firstlineno) def stage_nopython_backend(self): """ Do lowering for nopython """ lowerfn = self.backend_nopython_mode self._backend(lowerfn, objectmode=False) def stage_compile_interp_mode(self): """ Just create a compile result for interpreter mode """ args = [types.pyobject] * len(self.args) signature = typing.signature(types.pyobject, *args) self.cr = compile_result(typing_context=self.typingctx, target_context=self.targetctx, entry_point=self.func_id.func, typing_error=self.status.fail_reason, type_annotation="<Interpreter mode function>", signature=signature, objectmode=False, interpmode=True, lifted=(), fndesc=None,) def stage_ir_legalization(self): raise_on_unsupported_feature(self.func_ir) def stage_cleanup(self): """ Cleanup intermediate results to release resources. """ def define_pipelines(self, pm): """Child classes override this to customize the pipeline. """ raise NotImplementedError() def add_preprocessing_stage(self, pm): """Add the preprocessing stage that analyzes the bytecode to prepare the Numba IR. """ if self.func_ir is None: pm.add_stage(self.stage_analyze_bytecode, "analyzing bytecode") pm.add_stage(self.stage_process_ir, "processing IR") def add_pre_typing_stage(self, pm): """Add any stages that go before type-inference. The current stages contain type-agnostic rewrite passes. """ if not self.flags.no_rewrites: if self.status.can_fallback: pm.add_stage(self.stage_preserve_ir, "preserve IR for fallback") pm.add_stage(self.stage_generic_rewrites, "nopython rewrites") pm.add_stage(self.stage_inline_pass, "inline calls to locally defined closures") def add_typing_stage(self, pm): """Add the type-inference stage necessary for nopython mode. """ pm.add_stage(self.stage_nopython_frontend, "nopython frontend") pm.add_stage(self.stage_annotate_type, "annotate type") def add_optimization_stage(self, pm): """Add optimization stages. """ if self.flags.auto_parallel.enabled: pm.add_stage(self.stage_pre_parfor_pass, "Preprocessing for parfors") if not self.flags.no_rewrites: pm.add_stage(self.stage_nopython_rewrites, "nopython rewrites") if self.flags.auto_parallel.enabled: pm.add_stage(self.stage_parfor_pass, "convert to parfors") def add_lowering_stage(self, pm): """Add the lowering (code-generation) stage for nopython-mode """ pm.add_stage(self.stage_nopython_backend, "nopython mode backend") def add_cleanup_stage(self, pm): """Add the clean-up stage to remove intermediate results. """ pm.add_stage(self.stage_cleanup, "cleanup intermediate results") def add_with_handling_stage(self, pm): pm.add_stage(self.stage_frontend_withlift, "Handle with contexts") def define_nopython_pipeline(self, pm, name='nopython'): """Add the nopython-mode pipeline to the pipeline manager """ pm.create_pipeline(name) self.add_preprocessing_stage(pm) self.add_with_handling_stage(pm) self.add_pre_typing_stage(pm) self.add_typing_stage(pm) self.add_optimization_stage(pm) pm.add_stage(self.stage_ir_legalization, "ensure IR is legal prior to lowering") self.add_lowering_stage(pm) pm.add_stage(self.stage_dump_diagnostics, "dump diagnostics") self.add_cleanup_stage(pm) def define_objectmode_pipeline(self, pm, name='object'): """Add the object-mode pipeline to the pipeline manager """ pm.create_pipeline(name) self.add_preprocessing_stage(pm) pm.add_stage(self.stage_objectmode_frontend, "object mode frontend") pm.add_stage(self.stage_inline_pass, "inline calls to locally defined closures") pm.add_stage(self.stage_annotate_type, "annotate type") pm.add_stage(self.stage_ir_legalization, "ensure IR is legal prior to lowering") pm.add_stage(self.stage_objectmode_backend, "object mode backend") self.add_cleanup_stage(pm) def define_interpreted_pipeline(self, pm, name="interp"): """Add the interpreted-mode (fallback) pipeline to the pipeline manager """ pm.create_pipeline(name) pm.add_stage(self.stage_compile_interp_mode, "compiling with interpreter mode") self.add_cleanup_stage(pm) def _compile_core(self): """ Populate and run compiler pipeline """ pm = _PipelineManager() self.define_pipelines(pm) pm.finalize() res = pm.run(self.status) if res is not None: # Early pipeline completion return res else: assert self.cr is not None return self.cr def _compile_bytecode(self): """ Populate and run pipeline for bytecode input """ assert self.func_ir is None return self._compile_core() def _compile_ir(self): """ Populate and run pipeline for IR input """ assert self.func_ir is not None return self._compile_core()