def optimizeShlibModule(module): # Pick up parent package if any. _attemptRecursion(module) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() Plugins.considerImplicitImports(module, signal_change = signalChange)
def addImportedModule(imported_module): key = (Utils.relpath(imported_module.getFilename()), imported_module.getFullName()) if key in imported_modules: assert imported_module is imported_modules[key], key else: Plugins.onModuleDiscovered(imported_module) imported_modules[key] = imported_module imported_by_name[imported_module.getFullName()] = imported_module # We don't expect that to happen. assert not imported_module.isMainModule()
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name = module.getFullName() ) ) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise if not tag_set: break if graph is not None: computation_counters[module] = computation_counters.get(module, 0) + 1 module_graph = module.asGraph(computation_counters[module]) graph.subgraph(module_graph) touched = True if _progress: memory_watch.finish() printLine( "Memory usage changed during optimization of '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) ) Plugins.considerImplicitImports(module, signal_change = signalChange) return touched
def decideRecursion(module_filename, module_name, module_package, module_kind): # Many branches, which make decisions immediately, by returning # pylint: disable=R0911,R0912 Plugins.onModuleEncounter(module_filename, module_name, module_package, module_kind) if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." if module_package is None: full_name = module_name else: full_name = module_package + '.' + module_name no_case_modules = Options.getShallFollowInNoCase() for no_case_module in no_case_modules: if full_name == no_case_module: return (False, "Module listed explicitly to not recurse to.") if full_name.startswith(no_case_module + '.'): return (False, "Module in package listed explicitly to not recurse to.") any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return (True, "Module listed explicitly to recurse to.") if full_name.startswith(any_case_module + '.'): return (True, "Module in package listed explicitly to recurse to.") if Options.shallFollowNoImports(): return (False, "Requested to not recurse at all.") if StandardLibrary.isStandardLibraryPath(module_filename): return (Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ("" if Options.shallFollowStandardLibrary() else "not ")) if Options.shallFollowAllImports(): return (True, "Requested to recurse to all non-standard library modules.") # Means, we were not given instructions how to handle things. return (None, "Default behavior, not recursing without request.")
def addImportedModule(module_relpath, imported_module): if (module_relpath, "__main__") in imported_modules: warning("""\ Re-importing __main__ module via its filename duplicates the module.""") key = module_relpath, imported_module.getFullName() if key in imported_modules: assert imported_module is imported_modules[key], key else: Plugins.onModuleDiscovered(imported_module) imported_modules[key] = imported_module imported_by_name[imported_module.getFullName()] = imported_module
def addImportedModule(module_relpath, imported_module): if (module_relpath, "__main__") in imported_modules: warning("""\ Re-importing __main__ module via its filename duplicates the module.""") key = module_relpath, imported_module.getFullName() if key in imported_modules: assert imported_module is imported_modules[ key ], key else: Plugins.onModuleDiscovered(imported_module) imported_modules[ key ] = imported_module imported_by_name[ imported_module.getFullName() ] = imported_module
def _findModule(module_name): if _debug_module_finding: print( "_findModule: Enter to search '%s'." % ( module_name, ) ) assert not module_name.endswith('.'), module_name key = module_name if key in module_search_cache: result = module_search_cache[key] if _debug_module_finding: print("_findModule: Cached result (see previous call).") if result is ImportError: raise ImportError else: return result try: module_search_cache[key] = _findModule2(module_name) except ImportError: new_module_name = Plugins.considerFailedImportReferrals(module_name) if new_module_name is None: module_search_cache[key] = ImportError raise else: module_search_cache[key] = _findModule(new_module_name) return module_search_cache[key]
def _findModule(module_name): if _debug_module_finding: print("_findModule: Enter to search '%s'." % (module_name, )) assert not module_name.endswith('.'), module_name key = module_name if key in module_search_cache: result = module_search_cache[key] if _debug_module_finding: print("_findModule: Cached result (see previous call).") if result is ImportError: raise ImportError else: return result try: module_search_cache[key] = _findModule2(module_name) except ImportError: new_module_name = Plugins.considerFailedImportReferrals(module_name) if new_module_name is None: module_search_cache[key] = ImportError raise else: module_search_cache[key] = _findModule(new_module_name) return module_search_cache[key]
def readSourceCodeFromFilename(module_name, source_filename): if Utils.python_version < 300: source_code = _readSourceCodeFromFilename2(source_filename) else: source_code = _readSourceCodeFromFilename3(source_filename) # Allow plug-ins to mess with source code. source_code = Plugins.onModuleSourceCode(module_name, source_code) return source_code
def warnAbout(importing, module_name, parent_package, level): # This probably should not be dealt with here. if module_name == "": return if not isWhiteListedNotExistingModule(module_name): key = module_name, parent_package, level if key not in warned_about: warned_about.add(key) if parent_package is None: full_name = module_name else: full_name = module_name if Plugins.suppressUnknownImportWarning(importing, full_name): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if parent_package is not None: warning( "%s: Cannot find '%s' in package '%s' %s.", importing.getSourceReference().getAsString(), module_name, parent_package, level_desc ) else: warning( "%s: Cannot find '%s' %s.", importing.getSourceReference().getAsString(), module_name, level_desc )
def warnAbout(importing, module_name, parent_package, level): # This probably should not be dealt with here. if module_name == "": return if not isWhiteListedNotExistingModule(module_name): key = module_name, parent_package, level if key not in warned_about: warned_about.add(key) if parent_package is None: full_name = module_name else: full_name = module_name if Plugins.suppressUnknownImportWarning(importing, full_name): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if parent_package is not None: warning("%s: Cannot find '%s' in package '%s' %s.", importing.getSourceReference().getAsString(), module_name, parent_package, level_desc) else: warning("%s: Cannot find '%s' %s.", importing.getSourceReference().getAsString(), module_name, level_desc)
def decideRecursion(module_filename, module_name, module_package, module_kind): # Many branches, which make decisions immediately, by returning # pylint: disable=R0911,R0912 Plugins.onModuleEncounter( module_filename, module_name, module_package, module_kind ) if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." if module_package is None: full_name = module_name else: full_name = module_package + '.' + module_name no_case_modules = Options.getShallFollowInNoCase() for no_case_module in no_case_modules: if full_name == no_case_module: return ( False, "Module listed explicitly to not recurse to." ) if full_name.startswith(no_case_module + '.'): return ( False, "Module in package listed explicitly to not recurse to." ) any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return ( True, "Module listed explicitly to recurse to." ) if full_name.startswith(any_case_module + '.'): return ( True, "Module in package listed explicitly to recurse to." ) if Options.shallFollowNoImports(): return ( False, "Requested to not recurse at all." ) if StandardLibrary.isStandardLibraryPath(module_filename): return ( Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ( "" if Options.shallFollowStandardLibrary() else "not " ) ) if Options.shallFollowAllImports(): return ( True, "Requested to recurse to all non-standard library modules." ) # Means, we were not given instructions how to handle things. return ( None, "Default behavior, not recursing without request." )
def _onEnterNode(self, node): # This has many different things it deals with, so there need to be a # lot of branches and statements, pylint: disable=R0912,R0915 # Also all self specific things have been done on the outside, # pylint: disable=R0201 # Find nodes with only compile time constant children, these are # missing some obvious optimization potentially. if False: # For searching only, pylint: disable=W0125 if not node.isStatementReturn() and \ not node.isExpressionYield() and \ not node.isStatementRaiseException() and \ not node.isExpressionCall() and \ not node.isExpressionBuiltinIter1(): children = node.getVisitableNodes() if children: for child in children: if child.isStatement() or child.isStatementsSequence(): break if not child.isCompileTimeConstant(): break else: assert False, (node, node.parent, children) if node.isExpressionFunctionBody(): if node.isUnoptimized(): node.markAsLocalsDict() if node.needsLocalsDict(): provider = node.getParentVariableProvider() if not provider.isCompiledPythonModule(): provider.markAsLocalsDict() if node.isStatementReturn() or node.isStatementGeneratorReturn(): search = node in_tried_block = False # Search up to the containing function, and check for a try/finally # containing the "return" statement. search = search.getParentReturnConsumer() if search.isExpressionGeneratorObjectBody() or \ search.isExpressionCoroutineObjectBody(): if in_tried_block: search.markAsNeedsGeneratorReturnHandling(2) else: search.markAsNeedsGeneratorReturnHandling(1) if node.isExpressionBuiltinImport() and \ not Options.getShallFollowExtra() and \ not Options.getShallFollowExtraFilePatterns() and \ not Options.shallFollowNoImports() and \ not isWhiteListedImport(node) and \ not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()): warning("""Unresolved '__import__' call at '%s' may require use \ of '--recurse-directory'.""" % (node.getSourceReference().getAsString())) if node.isExpressionFunctionCreation(): if not node.getParent().isExpressionFunctionCall() or \ node.getParent().getFunction() is not node: node.getFunctionRef().getFunctionBody().markAsNeedsCreation() if node.isExpressionFunctionCall(): node.getFunction().getFunctionRef().getFunctionBody().\ markAsDirectlyCalled() if node.isExpressionFunctionRef(): function_body = node.getFunctionBody() parent_module = function_body.getParentModule() node_module = node.getParentModule() if node_module is not parent_module: function_body.markAsCrossModuleUsed() node_module.addCrossUsedFunction(function_body) if node.isStatementAssignmentVariable(): target_var = node.getTargetVariableRef().getVariable() assign_source = node.getAssignSource() if assign_source.isExpressionOperationBinary(): left_arg = assign_source.getLeft() if left_arg.isExpressionVariableRef(): if assign_source.getLeft().getVariable().isModuleVariable( ): assign_source.unmarkAsInplaceSuspect() elif assign_source.getLeft().getVariable() is target_var: if assign_source.isInplaceSuspect(): node.markAsInplaceSuspect() if node.isStatementPublishException(): node.getParentStatementsFrame().markAsFrameExceptionPreserving() if python_version >= 300: if node.isExpressionYield() or node.isExpressionYieldFrom(): search = node.getParent() while not search.isExpressionGeneratorObjectBody(): last_search = search search = search.getParent() if search.isStatementTry() and \ last_search == search.getBlockExceptHandler(): node.markAsExceptionPreserving() break
def main(): """ Main program flow of Nuitka At this point, options will be parsed already, Nuitka will be executing in the desired version of Python with desired flags, and we just get to execute the task assigned. We might be asked to only re-compile generated C++, dump only an XML representation of the internal node tree after optimization, etc. """ # Main has to fullfil many options, leading to many branches and statements # to deal with them. pylint: disable=R0912 positional_args = Options.getPositionalArgs() assert len(positional_args) > 0 filename = Options.getPositionalArgs()[0] # Inform the importing layer about the main script directory, so it can use # it when attempting to follow imports. Importing.setMainScriptDirectory( main_dir = Utils.dirname(Utils.abspath(filename)) ) # Detect to be frozen modules if any, so we can consider to not recurse # to them. if Options.isStandaloneMode(): for module in detectEarlyImports(): ModuleRegistry.addUncompiledModule(module) if module.getName() == "site": origin_prefix_filename = Utils.joinpath( Utils.dirname(module.getCompileTimeFilename()), "orig-prefix.txt" ) if Utils.isFile(origin_prefix_filename): data_files.append( (filename, "orig-prefix.txt") ) # Turn that source code into a node tree structure. try: main_module = createNodeTree( filename = filename ) except (SyntaxError, IndentationError) as e: # Syntax or indentation errors, output them to the user and abort. sys.exit( SyntaxErrors.formatOutput(e) ) if Options.shallDumpBuiltTreeXML(): for module in ModuleRegistry.getDoneModules(): dumpTreeXML(module) elif Options.shallDisplayBuiltTree(): displayTree(main_module) else: result, options = compileTree( main_module = main_module ) # Exit if compilation failed. if not result: sys.exit(1) if Options.shallNotDoExecCppCall(): sys.exit(0) # Remove the source directory (now build directory too) if asked to. if Options.isRemoveBuildDir(): shutil.rmtree( getSourceDirectoryPath(main_module) ) if Options.isStandaloneMode(): binary_filename = options["result_name"] + ".exe" standalone_entry_points.insert( 0, (binary_filename, None) ) dist_dir = getStandaloneDirectoryPath(main_module) for module in ModuleRegistry.getDoneUserModules(): standalone_entry_points.extend( Plugins.considerExtraDlls(dist_dir, module) ) if Utils.getOS() == "NetBSD": warning("Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported.") copyUsedDLLs( dist_dir = dist_dir, standalone_entry_points = standalone_entry_points ) for source_filename, target_filename in data_files: shutil.copy2( source_filename, Utils.joinpath( getStandaloneDirectoryPath(main_module), target_filename ) ) # Modules should not be executable, but Scons creates them like it, fix # it up here. if Utils.getOS() != "Windows" and Options.shallMakeModule(): subprocess.call( ( "chmod", "-x", getResultFullpath(main_module) ) ) # Execute the module immediately if option was given. if Options.shallExecuteImmediately(): if Options.shallMakeModule(): executeModule( tree = main_module, clean_path = Options.shallClearPythonPathEnvironment() ) else: executeMain( binary_filename = getResultFullpath(main_module), clean_path = Options.shallClearPythonPathEnvironment() )
def _onEnterNode(self, node): # This has many different things it deals with, so there need to be a # lot of branches and statements, pylint: disable=R0912,R0915 # Also all self specific things have been done on the outside, # pylint: disable=R0201 # Find nodes with only compile time constant children, these are # missing some obvious optimization potentially. if False: if not node.isStatementReturn() and \ not node.isExpressionYield() and \ not node.isStatementRaiseException() and \ not node.isExpressionCall() and \ not node.isExpressionBuiltinIter1(): children = node.getVisitableNodes() if children: for child in children: if child.isStatement() or child.isStatementsSequence(): break if not child.isCompileTimeConstant(): break else: assert False, (node, node.parent, children) if node.isExpressionFunctionBody(): if node.isUnoptimized(): node.markAsLocalsDict() if node.needsLocalsDict(): provider = node.getParentVariableProvider() if provider.isExpressionFunctionBody(): provider.markAsLocalsDict() if node.isStatementReturn() or node.isStatementGeneratorReturn(): search = node in_tried_block = False # Search up to the containing function, and check for a try/finally # containing the "return" statement. search = search.getParentReturnConsumer() if search.isExpressionFunctionBody() and search.isGenerator(): if in_tried_block: search.markAsNeedsGeneratorReturnHandling(2) else: search.markAsNeedsGeneratorReturnHandling(1) if node.isExpressionBuiltinImport() and \ not Options.getShallFollowExtra() and \ not Options.getShallFollowExtraFilePatterns() and \ not Options.shallFollowNoImports() and \ not isWhiteListedImport(node) and \ not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()): warning("""Unresolved '__import__' call at '%s' may require use \ of '--recurse-directory'.""" % ( node.getSourceReference().getAsString() ) ) if node.isExpressionFunctionCreation(): if not node.getParent().isExpressionFunctionCall() or \ node.getParent().getFunction() is not node: node.getFunctionRef().getFunctionBody().markAsNeedsCreation() if node.isExpressionFunctionCall(): node.getFunction().getFunctionRef().getFunctionBody().\ markAsDirectlyCalled() if node.isExpressionFunctionRef(): function_body = node.getFunctionBody() parent_module = function_body.getParentModule() node_module = node.getParentModule() if node_module is not parent_module: function_body.markAsCrossModuleUsed() node_module.addCrossUsedFunction(function_body) if node.isStatementAssignmentVariable(): target_var = node.getTargetVariableRef().getVariable() assign_source = node.getAssignSource() if assign_source.isExpressionOperationBinary(): left_arg = assign_source.getLeft() if left_arg.isExpressionVariableRef(): if assign_source.getLeft().getVariable().isModuleVariable(): assign_source.unmarkAsInplaceSuspect() elif assign_source.getLeft().getVariable() is target_var: if assign_source.isInplaceSuspect(): node.markAsInplaceSuspect() if node.isStatementPublishException(): node.getParentStatementsFrame().markAsFrameExceptionPreserving() if python_version >= 300: if node.isExpressionYield() or node.isExpressionYieldFrom(): search = node.getParent() while not search.isExpressionFunctionBody(): last_search = search search = search.getParent() if search.isStatementTry() and \ last_search == search.getBlockExceptHandler(): node.markAsExceptionPreserving() break