def optimizeShlibModule(module): # Pick up parent package if any. _attemptRecursion(module) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() Plugins.considerImplicitImports(module, signal_change = signalChange)
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name = module.getFullName() ) ) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = MemoryUsage.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise if not tag_set: break if graph is not None: computation_counters[module] = computation_counters.get(module, 0) + 1 module_graph = module.asGraph(computation_counters[module]) graph.subgraph(module_graph) touched = True if _progress: memory_watch.finish() printLine( "Memory usage changed during optimization of '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) ) Plugins.considerImplicitImports(module, signal_change = signalChange) return touched
def optimizeCompiledPythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name = module.getFullName() ) ) touched = False if _progress and Options.isShowMemory(): memory_watch = MemoryUsage.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise Graphs.onModuleOptimizationStep(module) # Search for local change tags. for tag in tag_set: if tag == "new_code": continue break else: break # Otherwise we did stuff, so note that for return value. touched = True if _progress and Options.isShowMemory(): memory_watch.finish() printLine( "Memory usage changed during optimization of '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) ) Plugins.considerImplicitImports(module, signal_change = signalChange) return touched
def _createTriggerLoadedModule(module, trigger_name, code): from nuitka.tree.Building import createModuleTree from nuitka.nodes.ModuleNodes import CompiledPythonModule from nuitka.plugins.Plugins import Plugins module_name = module.getName() + trigger_name source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name) mode = Plugins.decideCompilation(module_name, source_ref) trigger_module = CompiledPythonModule( name = module_name, package_name = module.getPackage(), mode = mode, source_ref = source_ref ) createModuleTree( module = trigger_module, source_ref = module.getSourceReference(), source_code = code, is_main = False ) return trigger_module
def addImportedModule(imported_module): key = ( Utils.relpath(imported_module.getFilename()), imported_module.getFullName() ) if key in imported_modules: assert imported_module is imported_modules[key], key else: Plugins.onModuleDiscovered(imported_module) imported_modules[key] = imported_module imported_by_name[imported_module.getFullName()] = imported_module # We don't expect that to happen. assert not imported_module.isMainModule()
def _findModule(module_name): if _debug_module_finding: print( "_findModule: Enter to search '%s'." % ( module_name, ) ) assert not module_name.endswith('.'), module_name key = module_name if key in module_search_cache: result = module_search_cache[key] if _debug_module_finding: print("_findModule: Cached result (see previous call).") if result is ImportError: raise ImportError else: return result try: module_search_cache[key] = _findModule2(module_name) except ImportError: new_module_name = Plugins.considerFailedImportReferrals(module_name) if new_module_name is None: module_search_cache[key] = ImportError raise else: module_search_cache[key] = _findModule(new_module_name) return module_search_cache[key]
def addImportedModule(imported_module): module_filename = relpath(imported_module.getFilename()) if os.path.basename(module_filename) == "__init__.py": module_filename = os.path.dirname(module_filename) key = (module_filename, imported_module.getFullName()) if key in imported_modules: assert imported_module is imported_modules[key], key else: Plugins.onModuleDiscovered(imported_module) imported_modules[key] = imported_module imported_by_name[imported_module.getFullName()] = imported_module # We don't expect that to happen. assert not imported_module.isMainModule()
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() source_code = readSourceCodeFromFilename(full_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name = full_name, is_package = False, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = full_name, is_package = False, bytecode = bytecode ) uncompiled_module = makeUncompiledPythonModule( module_name = module.getFullName(), filename = filename, bytecode = marshal.dumps(bytecode), is_package = module.isCompiledPythonPackage(), user_provided = True, technical = False ) replaceImportedModule( old = module, new = uncompiled_module ) replaceRootModule( old = module, new = uncompiled_module ) assert module.constraint_collection is not None uncompiled_module.setUsedModules(module.constraint_collection.getUsedModules())
def optimizeCompiledPythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name=module.getFullName())) touched = False if _progress and Options.isShowMemory(): memory_watch = MemoryUsage.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise Graphs.onModuleOptimizationStep(module) # Search for local change tags. for tag in tag_set: if tag == "new_code": continue break else: break # Otherwise we did stuff, so note that for return value. touched = True if _progress and Options.isShowMemory(): memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) Plugins.considerImplicitImports(module, signal_change=signalChange) return touched
def readSourceCodeFromFilename(module_name, source_filename): if python_version < 300: source_code = _readSourceCodeFromFilename2(source_filename) else: source_code = _readSourceCodeFromFilename3(source_filename) # Allow plug-ins to mess with source code. source_code = Plugins.onModuleSourceCode(module_name, source_code) return source_code
def buildModuleTree(filename, package, is_top, is_main): module, source_ref, source_filename = decideModuleTree( filename=filename, package=package, is_top=is_top, is_main=is_main, is_shlib=False, ) if is_top: ModuleRegistry.addRootModule(module) OutputDirectories.setMainModule(module) # Detect to be frozen modules if any, so we can consider to not recurse # to them. if Options.isStandaloneMode(): detectEarlyImports() # If there is source code associated (not the case for namespace packages of # Python3.3 or higher, then read it. if source_filename is not None: source_code = readSourceCodeFromFilename( module_name=module.getFullName(), source_filename=source_filename) if is_main: checkPythonVersionFromCode(source_code) # Read source code. createModuleTree( module=module, source_ref=source_ref, source_code=source_code, is_main=is_main, ) # Main modules do not get added to the import cache, but plugins get to see it. if module.isMainModule(): Plugins.onModuleDiscovered(module) else: addImportedModule(imported_module=module) return module
def decideCompilationMode(is_top, module_name, source_ref): result = Plugins.decideCompilation(module_name, source_ref) if result == "bytecode" and is_top: plugins_logger.warning("""\ Ignoring plugin decision to compile top level package '%s' as bytecode, the extension module entry point is technically required to compiled.""" % module_name) result = "compiled" return result
def getModuleMetapathLoaderEntryCode(module, bytecode_accessor): module_c_name = encodePythonStringToC( Plugins.encodeDataComposerName(module.getFullName().asString())) flags = ["NUITKA_TRANSLATED_FLAG"] if not Options.isStandaloneMode() and python_version >= 0x370: if Options.isWin32Windows(): file_path = encodePythonUnicodeToC(module.getCompileTimeFilename()) else: file_path = encodePythonStringToC( module.getCompileTimeFilename().encode( sys.getfilesystemencoding())) else: file_path = "NULL" if module.isUncompiledPythonModule(): code_data = module.getByteCode() is_package = module.isUncompiledPythonPackage() flags.append("NUITKA_BYTECODE_FLAG") if is_package: flags.append("NUITKA_PACKAGE_FLAG") accessor_code = bytecode_accessor.getBlobDataCode( data=code_data, name="bytecode of module '%s'" % module.getFullName().asString(), ) return template_metapath_loader_bytecode_module_entry % { "module_name": module_c_name, "bytecode": accessor_code[accessor_code.find("[") + 1:-1], "size": len(code_data), "flags": " | ".join(flags), "file_path": file_path, } elif module.isPythonExtensionModule(): flags.append("NUITKA_EXTENSION_MODULE_FLAG") return template_metapath_loader_extension_module_entry % { "module_name": module_c_name, "flags": " | ".join(flags), "file_path": file_path, } else: if module.isCompiledPythonPackage(): flags.append("NUITKA_PACKAGE_FLAG") return template_metapath_loader_compiled_module_entry % { "module_name": module_c_name, "module_identifier": module.getCodeName(), "flags": " | ".join(flags), "file_path": file_path, }
def decideRecursion(module_filename, module_name, module_kind, extra_recursion=False): # Many branches, which make decisions immediately, by returning # pylint: disable=too-many-return-statements if module_name == "__main__": return False, "Main program is not recursed to again." plugin_decision = Plugins.onModuleEncounter( module_filename, module_name, module_kind ) if plugin_decision: return plugin_decision if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." no_case, reason = matchesModuleNameToPatterns( module_name=module_name, patterns=Options.getShallFollowInNoCase() ) if no_case: return (False, "Module %s instructed by user to not recurse to." % reason) any_case, reason = matchesModuleNameToPatterns( module_name=module_name, patterns=Options.getShallFollowModules() ) if any_case: return (True, "Module %s instructed by user to recurse to." % reason) if Options.shallFollowNoImports(): return (False, "Requested to not recurse at all.") if StandardLibrary.isStandardLibraryPath(module_filename): return ( Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ("" if Options.shallFollowStandardLibrary() else "not "), ) if Options.shallFollowAllImports(): return (True, "Requested to recurse to all non-standard library modules.") # Means, we were not given instructions how to handle things. if extra_recursion: return (True, "Lives in plug-in directory.") if Options.shallMakeModule(): return (False, "Making a module, not following any imports by default.") return (None, "Default behavior, not recursing without request.")
def onModuleInitialSet(self): from nuitka.importing.ImportCache import addImportedModule from nuitka.ModuleRegistry import getRootTopModule from nuitka.plugins.Plugins import Plugins from nuitka.tree.Building import ( CompiledPythonModule, createModuleTree, readSourceCodeFromFilename, ) # First, build the module node and then read again from the # source code. root_module = getRootTopModule() module_name = ModuleName("__parents_main__") source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( module_name=module_name, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename(module_name, root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4. if python_version >= 0x340: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) addImportedModule(imported_module=slave_main_module) yield slave_main_module
def readSourceCodeFromFilename(module_name, source_filename): if python_version < 300: source_code = _readSourceCodeFromFilename2(source_filename) else: source_code = _readSourceCodeFromFilename3(source_filename) # Allow plug-ins to mess with source code, test framework usages # will pass None for module name. if module_name is not None: source_code = Plugins.onModuleSourceCode(module_name, source_code) return source_code
def optimizeUncompiledPythonModule(module): if _progress: info( "Doing module dependency considerations for '{module_name}':".format( module_name=module.getFullName() ) ) for used_module_name, used_module_path in module.getUsedModules(): used_module = ImportCache.getImportedModuleByNameAndPath( used_module_name, used_module_path ) ModuleRegistry.addUsedModule(used_module) package_name = module.getPackage() if package_name is not None: used_module = ImportCache.getImportedModuleByName(package_name) ModuleRegistry.addUsedModule(used_module) Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def optimizeUncompiledPythonModule(module): full_name = module.getFullName() progress_logger.info( "Doing module dependency considerations for '{module_name}':".format( module_name=full_name ) ) for used_module_name, used_module_path in module.getUsedModules(): used_module = ImportCache.getImportedModuleByNameAndPath( used_module_name, used_module_path ) ModuleRegistry.addUsedModule(used_module) package_name = full_name.getPackageName() if package_name is not None: used_module = ImportCache.getImportedModuleByName(package_name) ModuleRegistry.addUsedModule(used_module) Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def addImportedModule(imported_module): module_filename = relpath(imported_module.getFilename()) if os.path.basename(module_filename) == "__init__.py": module_filename = os.path.dirname(module_filename) key = ( module_filename, imported_module.getFullName() ) if key in imported_modules: assert imported_module is imported_modules[key], key else: Plugins.onModuleDiscovered(imported_module) imported_modules[key] = imported_module imported_by_name[imported_module.getFullName()] = imported_module # We don't expect that to happen. assert not imported_module.isMainModule()
def _addSlaveMainModule(root_module): from nuitka.tree.Building import ( CompiledPythonModule, readSourceCodeFromFilename, createModuleTree, ) from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins from sys import hexversion # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name=module_name, package_name=None, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename( "__parents_main__", root_module.getFilename() ) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4.0.a2 if hexversion >= 0x030400A2: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) # This is an alternative entry point of course. addRootModule(slave_main_module)
def _addSlaveMainModule(root_module): from nuitka.tree.Building import ( CompiledPythonModule, readSourceCodeFromFilename, createModuleTree, ) from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins from sys import hexversion # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name=module_name, package_name=None, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename("__parents_main__", root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4.0.a2 if hexversion >= 0x030400A2: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) # This is an alternative entry point of course. addRootModule(slave_main_module)
def warnAbout(importing, module_name, parent_package, level, tried_names): # This probably should not be dealt with here, pylint: disable=too-many-branches if module_name == "": return if not isIgnoreListedImportMaker(importing) and not isIgnoreListedNotExistingModule( module_name ): key = module_name, parent_package, level if key not in warned_about: warned_about.add(key) if parent_package is None: full_name = module_name else: full_name = module_name if Plugins.suppressUnknownImportWarning(importing, full_name): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if _debug_module_finding: if parent_package is not None: recursion_logger.warning( "%s: Cannot find '%s' in package '%s' %s (tried %s)." % ( importing.getSourceReference().getAsString(), module_name, parent_package, level_desc, ",".join(tried_names), ) ) else: recursion_logger.warning( "%s: Cannot find '%s' %s." % ( importing.getSourceReference().getAsString(), module_name, level_desc, ) )
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() debug("Demoting module '%s' to bytecode from '%s'.", full_name, filename) source_code = readSourceCodeFromFilename(full_name, filename) source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec", dont_inherit=True) bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name, is_package=False, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=full_name, filename=filename, bytecode=marshal.dumps(bytecode), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) assert module.trace_collection is not None uncompiled_module.setUsedModules(module.trace_collection.getUsedModules())
def buildMainModuleTree(filename, is_main): # Detect to be frozen modules if any, so we can consider to not follow # to them. Plugins.onBeforeCodeParsing() if is_main: # TODO: Doesn't work for deeply nested packages at all. if Options.hasPythonFlagPackageMode(): module_name = ModuleName(os.path.basename(filename) + ".__main__") else: module_name = ModuleName("__main__") else: module_name = Importing.getModuleNameAndKindFromFilename(filename)[0] module, _added = buildModule( module_name=module_name, module_filename=filename, source_code=None, is_top=True, is_main=is_main, is_extension=False, is_fake=False, hide_syntax_error=False, ) if is_main and Options.isStandaloneMode(): module.setEarlyModules(detectEarlyImports()) # Main modules do not get added to the import cache, but plugins get to see it. if module.isMainModule(): Plugins.onModuleDiscovered(module) else: addImportedModule(imported_module=module) return module
def getPackageSpecificDLLDirectories(package_name): scan_dirs = OrderedSet() if package_name is not None: from nuitka.importing.Importing import findModule package_dir = findModule(None, package_name, None, 0, False)[1] if os.path.isdir(package_dir): scan_dirs.add(package_dir) scan_dirs.update( getSubDirectories(package_dir, ignore_dirs=("__pycache__", ))) scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name)) return scan_dirs
def _getPackageSpecificDLLDirectories(package_name): scan_dirs = OrderedSet() if package_name is not None: package_dir = locateModule(module_name=package_name, parent_package=None, level=0)[1] if os.path.isdir(package_dir): scan_dirs.add(package_dir) scan_dirs.update( getSubDirectories(package_dir, ignore_dirs=("__pycache__", ))) scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name)) return scan_dirs
def warnAbout(importing, module_name, parent_package, level, tried_names): # This probably should not be dealt with here, pylint: disable=too-many-branches if module_name == "": return if not isWhiteListedImport(importing) and not isWhiteListedNotExistingModule( module_name ): key = module_name, parent_package, level if key not in warned_about: warned_about.add(key) if parent_package is None: full_name = module_name else: full_name = module_name if Plugins.suppressUnknownImportWarning(importing, full_name): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if _debug_module_finding: if parent_package is not None: warning( "%s: Cannot find '%s' in package '%s' %s (tried %s).", importing.getSourceReference().getAsString(), module_name, parent_package, level_desc, ",".join(tried_names), ) else: warning( "%s: Cannot find '%s' %s.", importing.getSourceReference().getAsString(), module_name, level_desc, )
def warnAbout(importing, module_name, level, source_ref): # This probably should not be dealt with here if module_name == "": return if not isIgnoreListedNotExistingModule( module_name ) and not isIgnoreListedImportMaker(source_ref): key = module_name, level if key not in warned_about: warned_about.add(key) if Plugins.suppressUnknownImportWarning( importing=importing, source_ref=source_ref, module_name=module_name ): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if _debug_module_finding: if importing.getPackageName() is not None: recursion_logger.warning( "%s: Cannot find '%s' in package '%s' %s." % ( importing.getSourceReference().getAsString(), module_name, importing.getPackageName().asString(), level_desc, ) ) else: recursion_logger.warning( "%s: Cannot find '%s' %s." % ( importing.getSourceReference().getAsString(), module_name, level_desc, ) )
def warnAbout(importing, module_name, parent_package, level): # This probably should not be dealt with here. if module_name == "": return if not isWhiteListedImport(importing) and \ not isWhiteListedNotExistingModule(module_name): key = module_name, parent_package, level if key not in warned_about: warned_about.add(key) if parent_package is None: full_name = module_name else: full_name = module_name if Plugins.suppressUnknownImportWarning(importing, full_name): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if parent_package is not None: warning( "%s: Cannot find '%s' in package '%s' %s.", importing.getSourceReference().getAsString(), module_name, parent_package, level_desc ) else: warning( "%s: Cannot find '%s' %s.", importing.getSourceReference().getAsString(), module_name, level_desc )
def warnAbout(importing, module_name, parent_package, level): # This probably should not be dealt with here. if module_name == "": return if not isWhiteListedNotExistingModule(module_name): key = module_name, parent_package, level if key not in warned_about: warned_about.add(key) if parent_package is None: full_name = module_name else: full_name = module_name if Plugins.suppressUnknownImportWarning(importing, full_name): return if level == 0: level_desc = "as absolute import" elif level == -1: level_desc = "as relative or absolute import" elif level == 1: level_desc = "%d package level up" % level else: level_desc = "%d package levels up" % level if parent_package is not None: warning( "%s: Cannot find '%s' in package '%s' %s.", importing.getSourceReference().getAsString(), module_name, parent_package, level_desc ) else: warning( "%s: Cannot find '%s' %s.", importing.getSourceReference().getAsString(), module_name, level_desc )
def _parsePEFileOutput(binary_filename, scan_dirs, result): pe = _getPEFile(binary_filename) # Some DLLs (eg numpy) don't have imports if not hasattr(pe, "DIRECTORY_ENTRY_IMPORT"): info( "Warning: no DIRECTORY_ENTRY_IMPORT PE section for library '%s'!" % binary_filename) return # Get DLL imports from PE file for imported_module in pe.DIRECTORY_ENTRY_IMPORT: dll_filename = imported_module.dll.decode() # Try to guess DLL path from scan dirs for scan_dir in scan_dirs: try: guessed_path = os.path.join(scan_dir, dll_filename) if os.path.isfile(guessed_path): dll_filename = guessed_path break except TypeError: pass dll_name = os.path.basename(dll_filename).upper() # Win API can be assumed. if dll_name.startswith("API-MS-WIN-") or dll_name.startswith( "EXT-MS-WIN-"): continue if dll_name in _win_dll_whitelist: continue # Allow plugins to prevent inclusion. blocked = Plugins.removeDllDependencies(dll_filename=dll_filename, dll_filenames=result) for to_remove in blocked: result.discard(to_remove) result.add(os.path.normcase(os.path.abspath(dll_filename)))
def getModuleMetapathLoaderEntryCode(module, bytecode_accessor): module_c_name = encodePythonStringToC( Plugins.encodeDataComposerName(module.getFullName().asString())) flags = ["NUITKA_TRANSLATED_FLAG"] if module.isUncompiledPythonModule(): code_data = module.getByteCode() is_package = module.isUncompiledPythonPackage() flags.append("NUITKA_BYTECODE_FLAG") if is_package: flags.append("NUITKA_PACKAGE_FLAG") accessor_code = bytecode_accessor.getBlobDataCode( data=code_data, name="bytecode of module '%s'" % module.getFullName().asString(), ) return template_metapath_loader_bytecode_module_entry % { "module_name": module_c_name, "bytecode": accessor_code[accessor_code.find("[") + 1:-1], "size": len(code_data), "flags": " | ".join(flags) or "0", } elif module.isPythonExtensionModule(): flags.append("NUITKA_EXTENSION_MODULE_FLAG") return template_metapath_loader_extension_module_entry % { "module_name": module_c_name, "flags": " | ".join(flags) or "0", } else: if module.isCompiledPythonPackage(): flags.append("NUITKA_PACKAGE_FLAG") return template_metapath_loader_compiled_module_entry % { "module_name": module_c_name, "module_identifier": module.getCodeName(), "flags": " | ".join(flags), }
def _addSlaveMainModule(root_module): from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name = module_name, package_name = None, mode = mode, source_ref = root_module.getSourceReference() ) source_code = readSourceCodeFromFilename( "__parents_main__", root_module.getFilename() ) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.main()""" createModuleTree( module = slave_main_module, source_ref = root_module.getSourceReference(), source_code = source_code, is_main = False ) # This is an alternative entry point of course. addRootModule(slave_main_module)
def readSourceCodeFromFilename(module_name, source_filename): if python_version < 0x300: source_code = _readSourceCodeFromFilename2(source_filename) else: source_code = _readSourceCodeFromFilename3(source_filename) # Allow plugins to mess with source code. Test code calls this # without a module and doesn't want changes from plugins. if module_name is not None: source_code_modified = Plugins.onModuleSourceCode(module_name, source_code) else: source_code_modified = source_code if Options.shallPersistModifications() and source_code_modified != source_code: orig_source_filename = source_filename + ".orig" if not os.path.exists(orig_source_filename): putTextFileContents(filename=orig_source_filename, contents=source_code) putTextFileContents(filename=source_filename, contents=source_code_modified) return source_code_modified
def _createTriggerLoadedModule(module, trigger_name, code): from nuitka.tree.Building import createModuleTree from nuitka.nodes.ModuleNodes import CompiledPythonModule from nuitka.plugins.Plugins import Plugins module_name = module.getName() + trigger_name source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name) mode = Plugins.decideCompilation(module_name, source_ref) trigger_module = CompiledPythonModule(name=module_name, package_name=module.getPackage(), mode=mode, source_ref=source_ref) createModuleTree(module=trigger_module, source_ref=module.getSourceReference(), source_code=code, is_main=False) return trigger_module
def decideCompilationMode(is_top, module_name, for_pgo): """Decide the compilation mode for a module. module_name - The module to decide compilation mode for. for_pgo - consider PGO information or not """ result = Plugins.decideCompilation(module_name) # Cannot change mode of __main__ to bytecode, that is not going # to work currently. if result == "bytecode" and is_top: plugins_logger.warning("""\ Ignoring plugin decision to compile top level package '%s' as bytecode, the extension module entry point is technically required to compiled.""" % module_name) result = "compiled" # Include all of standard library as bytecode, for now. We need to identify # which ones really need that. if not is_top: module_filename = Importing.locateModule(module_name=module_name, parent_package=None, level=0)[1] if module_filename is not None and isStandardLibraryPath( module_filename): result = "bytecode" # Plugins need to win over PGO, as they might know it better if result is None and not for_pgo: result = decideCompilationFromPGO(module_name=module_name) # Default if neither plugins nor PGO have expressed an opinion if result is None: result = "compiled" return result
def _onEnterNode(self, node): # This has many different things it deals with, so there need to be a # lot of branches and statements, pylint: disable=too-many-branches,too-many-statements # Also all self specific things have been done on the outside, # pylint: disable=no-self-use if node.isStatementReturn() or node.isStatementGeneratorReturn(): search = node in_tried_block = False # Search up to the containing function, and check for a try/finally # containing the "return" statement. search = search.getParentReturnConsumer() if ( search.isExpressionGeneratorObjectBody() or search.isExpressionCoroutineObjectBody() or search.isExpressionAsyncgenObjectBody() ): if in_tried_block: search.markAsNeedsGeneratorReturnHandling(2) else: search.markAsNeedsGeneratorReturnHandling(1) if ( node.isExpressionBuiltinImport() and not Options.getShallFollowExtra() and not Options.getShallFollowExtraFilePatterns() and not Options.shallFollowNoImports() and not isWhiteListedImport(node) and not node.recurse_attempted and not Plugins.suppressBuiltinImportWarning( node.getParentModule(), node.getSourceReference() ) ): warning( """Unresolved '__import__' call at '%s' may require use \ of '--include-plugin-directory' or '--include-plugin-files'.""" % (node.getSourceReference().getAsString()) ) if node.isExpressionBuiltinImport() and node.recurse_attempted: module_name = node.getImportName() if module_name.isCompileTimeConstant(): imported_module_name = module_name.getCompileTimeConstant() if type(imported_module_name) in (str, unicode): if imported_module_name: imported_names.add(imported_module_name) if node.isExpressionFunctionCreation(): if ( not node.getParent().isExpressionFunctionCall() or node.getParent().getFunction() is not node ): node.getFunctionRef().getFunctionBody().markAsNeedsCreation() if node.isExpressionFunctionCall(): node.getFunction().getFunctionRef().getFunctionBody().markAsDirectlyCalled() if node.isExpressionFunctionRef(): function_body = node.getFunctionBody() parent_module = function_body.getParentModule() node_module = node.getParentModule() if node_module is not parent_module: function_body.markAsCrossModuleUsed() node_module.addCrossUsedFunction(function_body) if node.isStatementAssignmentVariable(): target_var = node.getVariable() assign_source = node.getAssignSource() if assign_source.isExpressionOperationBinary(): left_arg = assign_source.getLeft() if left_arg.isExpressionVariableRef(): if assign_source.getLeft().getVariable() is target_var: if assign_source.isInplaceSuspect(): node.markAsInplaceSuspect() elif left_arg.isExpressionLocalsVariableRefORFallback(): assign_source.unmarkAsInplaceSuspect() if node.isStatementLocalsDictOperationSet(): assign_source = node.getAssignSource() if assign_source.isExpressionOperationBinary(): assign_source.unmarkAsInplaceSuspect() if python_version < 300 and node.isStatementPublishException(): node.getParentStatementsFrame().markAsFrameExceptionPreserving() if python_version >= 300: if ( node.isExpressionYield() or node.isExpressionYieldFrom() or node.isExpressionYieldFromWaitable() ): search = node.getParent() while ( not search.isExpressionGeneratorObjectBody() and not search.isExpressionCoroutineObjectBody() and not search.isExpressionAsyncgenObjectBody() ): last_search = search search = search.getParent() if ( search.isStatementTry() and last_search == search.getBlockExceptHandler() ): node.markAsExceptionPreserving() break
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename=filename, module_name=ModuleName("_collections_abc"), result=result, user_provided=user_provided, technical=technical, ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = "\n" + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code, ) # Debian stretch site.py source_code = source_code.replace( "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []") # Anaconda3 4.1.2 site.py source_code = source_code.replace( "def main():", "def main():return\n\nif 0:\n def _unused():") if Options.isShowInclusion(): inclusion_logger.info("Freezing module '%s' (from '%s')." % (module_name, filename)) is_package = os.path.basename(filename) == "__init__.py" # Plugins can modify source code: source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name, is_package=is_package, source_code=source_code) bytecode = compileSourceToBytecode( source_code=source_code, filename=module_name.replace(".", os.path.sep) + ".py", ) # Plugins can modify bytecode code: bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name, is_package=is_package, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(bytecode), is_package=is_package, filename=filename, user_provided=user_provided, technical=technical, ) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)
def copyDataFiles(dist_dir): """Copy the data files needed for standalone distribution. Args: dist_dir: The distribution folder under creation Notes: This is for data files only, not DLLs or even extension modules, those must be registered as entry points, and would not go through necessary handling if provided like this. """ # Many details to deal with, pylint: disable=too-many-branches,too-many-locals for pattern, dest, arg in Options.getShallIncludeDataFiles(): filenames = resolveShellPatternToFilenames(pattern) if not filenames: inclusion_logger.warning("No match data file to be included: %r" % pattern) for filename in filenames: file_reason = "specified data file %r on command line" % arg rel_path = dest if rel_path.endswith(("/", os.path.sep)): rel_path = os.path.join(rel_path, os.path.basename(filename)) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(filename, rel_path, file_reason), ) for src, dest in Options.getShallIncludeDataDirs(): filenames = getFileList(src) if not filenames: inclusion_logger.warning("No files in directory" % src) for filename in filenames: relative_filename = relpath(filename, src) file_reason = "specified data dir %r on command line" % src rel_path = os.path.join(dest, relative_filename) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(filename, rel_path, file_reason), ) # Cyclic dependency from nuitka import ModuleRegistry for module in ModuleRegistry.getDoneModules(): for plugin, included_datafile in Plugins.considerDataFiles(module): _handleDataFile(dist_dir=dist_dir, tracer=plugin, included_datafile=included_datafile) for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonPackage( ) or module.isUncompiledPythonPackage(): package_name = module.getFullName() match, reason = package_name.matchesToShellPatterns( patterns=Options.getShallIncludePackageData()) if match: package_directory = module.getCompileTimeDirectory() pkg_filenames = getFileList( package_directory, ignore_dirs=("__pycache__", ), ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") + getSharedLibrarySuffixes(), ) if pkg_filenames: file_reason = "package '%s' %s" % (package_name, reason) for pkg_filename in pkg_filenames: rel_path = os.path.join( package_name.asPath(), os.path.relpath(pkg_filename, package_directory), ) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(pkg_filename, rel_path, file_reason), )
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements assert package is None or type(package) is str assert filename is not None if is_main and os.path.isdir(filename): source_filename = os.path.join(filename, "__main__.py") if not os.path.isfile(source_filename): sys.stderr.write("%s: can't find '__main__' module in '%s'\n" % (os.path.basename(sys.argv[0]), filename)) sys.exit(2) filename = source_filename main_added = True else: main_added = False if os.path.isfile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename(filename=filename, ) if is_main: module_name = "__main__" else: module_name = os.path.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % (module_name)) sys.exit(2) if is_shlib: result = PythonShlibModule(name=module_name, package_name=package, source_ref=source_ref) elif is_main: result = PythonMainModule(main_added=main_added, mode=Plugins.decideCompilation( "__main__", source_ref), future_spec=None, source_ref=source_ref) else: if package is not None: full_name = package + '.' + module_name else: full_name = module_name result = CompiledPythonModule(name=module_name, package_name=package, mode=Plugins.decideCompilation( full_name, source_ref), future_spec=None, source_ref=source_ref) elif Importing.isPackageDir(filename): if is_top: module_name = splitPath(filename)[-1] else: module_name = os.path.basename(filename) source_filename = os.path.join(filename, "__init__.py") if not os.path.isfile(source_filename): source_ref, result = createNamespacePackage( module_name=module_name, package_name=package, module_relpath=filename) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename=os.path.abspath(source_filename), ) if package is not None: full_name = package + '.' + module_name else: full_name = module_name result = CompiledPythonPackage(name=module_name, package_name=package, mode=Plugins.decideCompilation( full_name, source_ref), future_spec=None, source_ref=source_ref) assert result.getFullName() == full_name, result else: sys.stderr.write("%s: can't open file '%s'.\n" % (os.path.basename(sys.argv[0]), filename)) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def setCommonOptions(options): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches if Options.shallRunInDebugger(): options["full_names"] = "true" if Options.assumeYesForDownloads(): options["assume_yes_for_downloads"] = asBoolStr(True) if not Options.shallUseProgressBar(): options["progress_bar"] = "false" if Options.isClang(): options["clang_mode"] = "true" if Options.isShowScons(): options["show_scons"] = "true" if Options.isMingw64(): options["mingw_mode"] = "true" if Options.getMsvcVersion(): options["msvc_version"] = Options.getMsvcVersion() if Options.shallDisableCCacheUsage(): options["disable_ccache"] = asBoolStr(True) if Options.shallDisableConsoleWindow(): options["disable_console"] = asBoolStr(True) if Options.getLtoMode() != "auto": options["lto_mode"] = Options.getLtoMode() cpp_defines = Plugins.getPreprocessorSymbols() if cpp_defines: options["cpp_defines"] = ",".join( "%s%s%s" % (key, "=" if value else "", value or "") for key, value in cpp_defines.items()) cpp_include_dirs = Plugins.getExtraIncludeDirectories() if cpp_include_dirs: options["cpp_include_dirs"] = ",".join(cpp_include_dirs) link_dirs = Plugins.getExtraLinkDirectories() if link_dirs: options["link_dirs"] = ",".join(link_dirs) link_libraries = Plugins.getExtraLinkLibraries() if link_libraries: options["link_libraries"] = ",".join(link_libraries) if Utils.isMacOS(): macos_min_version = detectBinaryMinMacOS(sys.executable) if macos_min_version is None: Tracing.general.sysexit( "Could not detect minimum macOS version for %r." % sys.executable) options["macos_min_version"] = macos_min_version macos_target_arch = Options.getMacOSTargetArch() if macos_target_arch == "universal": Tracing.general.sysexit( "Cannot create universal macOS binaries (yet), please pick an arch and create two binaries." ) options["macos_target_arch"] = macos_target_arch
def decideRecursion( module_filename, module_name, module_package, module_kind, extra_recursion=False ): # Many branches, which make decisions immediately, by returning # pylint: disable=too-many-branches,too-many-return-statements if module_name == "__main__": return False, "Main program is not recursed to again." plugin_decision = Plugins.onModuleEncounter( module_filename, module_name, module_package, module_kind ) if plugin_decision: return plugin_decision if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." if module_package is None: full_name = module_name else: full_name = module_package + "." + module_name no_case, reason = matchesModuleNameToPatterns( module_name=full_name, patterns=Options.getShallFollowInNoCase() ) if no_case: return (False, "Module %s instructed by user to not recurse to." % reason) any_case, reason = matchesModuleNameToPatterns( module_name=full_name, patterns=Options.getShallFollowModules() ) if any_case: return (True, "Module %s instructed by user to recurse to." % reason) if Options.shallFollowNoImports(): return (False, "Requested to not recurse at all.") if StandardLibrary.isStandardLibraryPath(module_filename): return ( Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ("" if Options.shallFollowStandardLibrary() else "not "), ) if Options.shallFollowAllImports(): return (True, "Requested to recurse to all non-standard library modules.") # Means, we were not given instructions how to handle things. if extra_recursion: return (True, "Lives in plug-in directory.") if Options.shallMakeModule(): return (False, "Making a module, not following any imports by default.") return (None, "Default behavior, not recursing without request.")
def _onEnterNode(self, node): # This has many different things it deals with, so there need to be a # lot of branches and statements, pylint: disable=R0912,R0915 # Also all self specific things have been done on the outside, # pylint: disable=R0201 # Find nodes with only compile time constant children, these are # missing some obvious optimization potentially. if False: # For searching only, pylint: disable=W0125 if not node.isStatementReturn() and \ not node.isExpressionYield() and \ not node.isStatementRaiseException() and \ not node.isExpressionCall() and \ not node.isExpressionBuiltinIter1(): children = node.getVisitableNodes() if children: for child in children: if child.isStatement() or child.isStatementsSequence(): break if not child.isCompileTimeConstant(): break else: assert False, (node, node.parent, children) if node.isExpressionFunctionBody(): if node.isUnoptimized(): node.markAsLocalsDict() if node.needsLocalsDict(): provider = node.getParentVariableProvider() if not provider.isCompiledPythonModule(): provider.markAsLocalsDict() if node.isStatementReturn() or node.isStatementGeneratorReturn(): search = node in_tried_block = False # Search up to the containing function, and check for a try/finally # containing the "return" statement. search = search.getParentReturnConsumer() if search.isExpressionGeneratorObjectBody() or \ search.isExpressionCoroutineObjectBody(): if in_tried_block: search.markAsNeedsGeneratorReturnHandling(2) else: search.markAsNeedsGeneratorReturnHandling(1) if node.isExpressionBuiltinImport() and \ not Options.getShallFollowExtra() and \ not Options.getShallFollowExtraFilePatterns() and \ not Options.shallFollowNoImports() and \ not isWhiteListedImport(node) and \ not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()): warning("""Unresolved '__import__' call at '%s' may require use \ of '--recurse-directory'.""" % ( node.getSourceReference().getAsString() ) ) if node.isExpressionFunctionCreation(): if not node.getParent().isExpressionFunctionCall() or \ node.getParent().getFunction() is not node: node.getFunctionRef().getFunctionBody().markAsNeedsCreation() if node.isExpressionFunctionCall(): node.getFunction().getFunctionRef().getFunctionBody().\ markAsDirectlyCalled() if node.isExpressionFunctionRef(): function_body = node.getFunctionBody() parent_module = function_body.getParentModule() node_module = node.getParentModule() if node_module is not parent_module: function_body.markAsCrossModuleUsed() node_module.addCrossUsedFunction(function_body) if node.isStatementAssignmentVariable(): target_var = node.getTargetVariableRef().getVariable() assign_source = node.getAssignSource() if assign_source.isExpressionOperationBinary(): left_arg = assign_source.getLeft() if left_arg.isExpressionVariableRef(): if assign_source.getLeft().getVariable().isModuleVariable(): assign_source.unmarkAsInplaceSuspect() elif assign_source.getLeft().getVariable() is target_var: if assign_source.isInplaceSuspect(): node.markAsInplaceSuspect() if node.isStatementPublishException(): node.getParentStatementsFrame().markAsFrameExceptionPreserving() if python_version >= 300: if node.isExpressionYield() or node.isExpressionYieldFrom(): search = node.getParent() while not search.isExpressionGeneratorObjectBody(): last_search = search search = search.getParent() if search.isStatementTry() and \ last_search == search.getBlockExceptHandler(): node.markAsExceptionPreserving() break
def decideRecursion(module_filename, module_name, module_package, module_kind): # Many branches, which make decisions immediately, by returning # pylint: disable=R0911,R0912 Plugins.onModuleEncounter( module_filename, module_name, module_package, module_kind ) if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." if module_package is None: full_name = module_name else: full_name = module_package + '.' + module_name no_case_modules = Options.getShallFollowInNoCase() for no_case_module in no_case_modules: if full_name == no_case_module: return ( False, "Module listed explicitly to not recurse to." ) if full_name.startswith(no_case_module + '.'): return ( False, "Module in package listed explicitly to not recurse to." ) any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return ( True, "Module listed explicitly to recurse to." ) if full_name.startswith(any_case_module + '.'): return ( True, "Module in package listed explicitly to recurse to." ) if Options.shallFollowNoImports(): return ( False, "Requested to not recurse at all." ) if StandardLibrary.isStandardLibraryPath(module_filename): return ( Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ( "" if Options.shallFollowStandardLibrary() else "not " ) ) if Options.shallFollowAllImports(): return ( True, "Requested to recurse to all non-standard library modules." ) # Means, we were not given instructions how to handle things. return ( None, "Default behavior, not recursing without request." )
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename = filename, module_name = "_collections_abc", result = result, user_provided = user_provided, technical = technical ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code ) debug( "Freezing module '%s' (from '%s').", module_name, filename ) is_package = Utils.basename(filename) == "__init__.py" source_code = Plugins.onFrozenModuleSourceCode( module_name = module_name, is_package = is_package, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = is_package, bytecode = bytecode ) result.append( makeUncompiledPythonModule( module_name = module_name, bytecode = marshal.dumps( bytecode ), is_package = is_package, filename = filename, user_provided = user_provided, technical = technical ) ) module_names.add(module_name)
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915 assert package is None or type(package) is str assert filename is not None if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename( filename = filename, ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % ( module_name ) ) sys.exit(2) if is_shlib: result = PythonShlibModule( name = module_name, package_name = package, source_ref = source_ref ) elif is_main: result = PythonMainModule( main_added = main_added, source_ref = source_ref ) else: if package is not None: full_name = package + "." + module_name else: full_name = module_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonModule( name = module_name, package_name = package, source_ref = source_ref ) else: source_code = readSourceCodeFromFilename(module_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name = full_name, is_package = False, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = False, bytecode = bytecode ) result = UncompiledPythonModule( name = module_name, package_name = package, bytecode = bytecode, filename = filename, user_provided = True, technical = False, source_ref = source_ref ) # Don't read it anymore. source_filename = None elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): source_ref, result = createNamespacePackage( package_name = package_name, module_relpath = filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename = Utils.abspath(source_filename), ) if package is not None: full_name = package + "." + package_name else: full_name = package_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonPackage( name = package_name, package_name = package, source_ref = source_ref ) else: bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = False, bytecode = bytecode ) result = UncompiledPythonPackage( name = module_name, package_name = package, bytecode = bytecode, filename = filename, user_provided = True, technical = False, source_ref = source_ref ) # Don't read it anymore. source_filename = None else: sys.stderr.write( "%s: can't open file '%s'.\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def main(): """ Main program flow of Nuitka At this point, options will be parsed already, Nuitka will be executing in the desired version of Python with desired flags, and we just get to execute the task assigned. We might be asked to only re-compile generated C, dump only an XML representation of the internal node tree after optimization, etc. """ # Main has to fulfill many options, leading to many branches and statements # to deal with them. pylint: disable=too-many-branches filename = Options.getPositionalArgs()[0] # Inform the importing layer about the main script directory, so it can use # it when attempting to follow imports. Importing.setMainScriptDirectory( main_dir=os.path.dirname(os.path.abspath(filename)) ) # Detect to be frozen modules if any, so we can consider to not recurse # to them. if Options.isStandaloneMode(): for module in detectEarlyImports(): ModuleRegistry.addUncompiledModule(module) # Turn that source code into a node tree structure. try: main_module = createNodeTree(filename=filename) except (SyntaxError, IndentationError) as e: handleSyntaxError(e) if Options.shallDumpBuiltTreeXML(): # XML output only. for module in ModuleRegistry.getDoneModules(): dumpTreeXML(module) else: # Make the actual compilation. result, options = compileTree(main_module=main_module) # Exit if compilation failed. if not result: sys.exit(1) if Options.shallNotDoExecCCompilerCall(): if Options.isShowMemory(): MemoryUsage.showMemoryTrace() sys.exit(0) executePostProcessing(getResultFullpath(main_module)) if Options.isStandaloneMode(): binary_filename = options["result_exe"] standalone_entry_points.insert(0, (binary_filename, binary_filename, None)) dist_dir = getStandaloneDirectoryPath(main_module) for module in ModuleRegistry.getDoneUserModules(): standalone_entry_points.extend( Plugins.considerExtraDlls(dist_dir, module) ) copyUsedDLLs( source_dir=getSourceDirectoryPath(main_module), dist_dir=dist_dir, standalone_entry_points=standalone_entry_points, ) data_files = [] for module in ModuleRegistry.getDoneModules(): data_files.extend(Plugins.considerDataFiles(module)) copyDataFiles(dist_dir=dist_dir, data_files=data_files) Plugins.onStandaloneDistributionFinished(dist_dir) # Remove the source directory (now build directory too) if asked to. if Options.isRemoveBuildDir(): removeDirectory( path=getSourceDirectoryPath(main_module), ignore_errors=False ) if Options.shallMakeModule() and Options.shallCreatePyiFile(): pyi_filename = getResultBasepath(main_module) + ".pyi" with open(pyi_filename, "w") as pyi_file: pyi_file.write( """\ # This file was generated by Nuitka and describes the types of the # created shared library. # At this time it lists only the imports made and can be used by the # tools that bundle libraries, including Nuitka itself. For instance # standalone mode usage of the created library will need it. # In the future, this will also contain type information for values # in the module, so IDEs will use this. Therefore please include it # when you make software releases of the extension module that it # describes. %(imports)s # This is not Python source even if it looks so. Make it clear for # now. This was decided by PEP 484 designers. __name__ = ... """ % { "imports": "\n".join( "import %s" % module_name for module_name in getImportedNames() ) } ) # Execute the module immediately if option was given. if Options.shallExecuteImmediately(): if Options.shallMakeModule(): executeModule( tree=main_module, clean_path=Options.shallClearPythonPathEnvironment(), ) else: executeMain( binary_filename=getResultFullpath(main_module), clean_path=Options.shallClearPythonPathEnvironment(), )
def _onEnterNode(self, node): # This has many different things it deals with, so there need to be a # lot of branches and statements, pylint: disable=R0912,R0915 # Also all self specific things have been done on the outside, # pylint: disable=R0201 # Find nodes with only compile time constant children, these are # missing some obvious optimization potentially. if False: # For searching only, pylint: disable=W0125 if not node.isStatementReturn() and \ not node.isExpressionYield() and \ not node.isStatementRaiseException() and \ not node.isExpressionCall() and \ not node.isExpressionBuiltinIter1(): children = node.getVisitableNodes() if children: for child in children: if child.isStatement() or child.isStatementsSequence(): break if not child.isCompileTimeConstant(): break else: assert False, (node, node.parent, children) if node.isExpressionFunctionBody(): if node.isUnoptimized(): node.markAsLocalsDict() if node.needsLocalsDict(): provider = node.getParentVariableProvider() if not provider.isCompiledPythonModule(): provider.markAsLocalsDict() if node.isStatementReturn() or node.isStatementGeneratorReturn(): search = node in_tried_block = False # Search up to the containing function, and check for a try/finally # containing the "return" statement. search = search.getParentReturnConsumer() if search.isExpressionGeneratorObjectBody() or \ search.isExpressionCoroutineObjectBody(): if in_tried_block: search.markAsNeedsGeneratorReturnHandling(2) else: search.markAsNeedsGeneratorReturnHandling(1) if node.isExpressionBuiltinImport() and \ not Options.getShallFollowExtra() and \ not Options.getShallFollowExtraFilePatterns() and \ not Options.shallFollowNoImports() and \ not isWhiteListedImport(node) and \ not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()): warning("""Unresolved '__import__' call at '%s' may require use \ of '--recurse-directory'.""" % (node.getSourceReference().getAsString())) if node.isExpressionFunctionCreation(): if not node.getParent().isExpressionFunctionCall() or \ node.getParent().getFunction() is not node: node.getFunctionRef().getFunctionBody().markAsNeedsCreation() if node.isExpressionFunctionCall(): node.getFunction().getFunctionRef().getFunctionBody().\ markAsDirectlyCalled() if node.isExpressionFunctionRef(): function_body = node.getFunctionBody() parent_module = function_body.getParentModule() node_module = node.getParentModule() if node_module is not parent_module: function_body.markAsCrossModuleUsed() node_module.addCrossUsedFunction(function_body) if node.isStatementAssignmentVariable(): target_var = node.getTargetVariableRef().getVariable() assign_source = node.getAssignSource() if assign_source.isExpressionOperationBinary(): left_arg = assign_source.getLeft() if left_arg.isExpressionVariableRef(): if assign_source.getLeft().getVariable().isModuleVariable( ): assign_source.unmarkAsInplaceSuspect() elif assign_source.getLeft().getVariable() is target_var: if assign_source.isInplaceSuspect(): node.markAsInplaceSuspect() if node.isStatementPublishException(): node.getParentStatementsFrame().markAsFrameExceptionPreserving() if python_version >= 300: if node.isExpressionYield() or node.isExpressionYieldFrom(): search = node.getParent() while not search.isExpressionGeneratorObjectBody(): last_search = search search = search.getParent() if search.isStatementTry() and \ last_search == search.getBlockExceptHandler(): node.markAsExceptionPreserving() break
def optimizeShlibModule(module): # Pick up parent package if any. _attemptRecursion(module) Plugins.considerImplicitImports(module, signal_change = signalChange)
def main(): """ Main program flow of Nuitka At this point, options will be parsed already, Nuitka will be executing in the desired version of Python with desired flags, and we just get to execute the task assigned. We might be asked to only re-compile generated C++, dump only an XML representation of the internal node tree after optimization, etc. """ # Main has to fullfil many options, leading to many branches and statements # to deal with them. pylint: disable=R0912 positional_args = Options.getPositionalArgs() assert len(positional_args) > 0 filename = Options.getPositionalArgs()[0] # Inform the importing layer about the main script directory, so it can use # it when attempting to follow imports. Importing.setMainScriptDirectory(main_dir=Utils.dirname(Utils.abspath(filename))) # Detect to be frozen modules if any, so we can consider to not recurse # to them. if Options.isStandaloneMode(): for module in detectEarlyImports(): ModuleRegistry.addUncompiledModule(module) if module.getName() == "site": origin_prefix_filename = Utils.joinpath( Utils.dirname(module.getCompileTimeFilename()), "orig-prefix.txt" ) if Utils.isFile(origin_prefix_filename): data_files.append((filename, "orig-prefix.txt")) # Turn that source code into a node tree structure. try: main_module = createNodeTree(filename=filename) except (SyntaxError, IndentationError) as e: # Syntax or indentation errors, output them to the user and abort. sys.exit(SyntaxErrors.formatOutput(e)) if Options.shallDumpBuiltTreeXML(): for module in ModuleRegistry.getDoneModules(): dumpTreeXML(module) elif Options.shallDisplayBuiltTree(): displayTree(main_module) else: result, options = compileTree(main_module=main_module) # Exit if compilation failed. if not result: sys.exit(1) if Options.shallNotDoExecCppCall(): sys.exit(0) # Remove the source directory (now build directory too) if asked to. if Options.isRemoveBuildDir(): shutil.rmtree(getSourceDirectoryPath(main_module)) if Options.isStandaloneMode(): binary_filename = options["result_name"] + ".exe" standalone_entry_points.insert(0, (binary_filename, None)) dist_dir = getStandaloneDirectoryPath(main_module) for module in ModuleRegistry.getDoneUserModules(): standalone_entry_points.extend(Plugins.considerExtraDlls(dist_dir, module)) for module in ModuleRegistry.getUncompiledModules(): standalone_entry_points.extend(Plugins.considerExtraDlls(dist_dir, module)) copyUsedDLLs(dist_dir=dist_dir, standalone_entry_points=standalone_entry_points) for module in ModuleRegistry.getDoneModules(): data_files.extend(Plugins.considerDataFiles(module)) for source_filename, target_filename in data_files: target_filename = Utils.joinpath(getStandaloneDirectoryPath(main_module), target_filename) Utils.makePath(Utils.dirname(target_filename)) shutil.copy2(source_filename, target_filename) # Modules should not be executable, but Scons creates them like it, fix # it up here. if Utils.getOS() != "Windows" and Options.shallMakeModule(): subprocess.call(("chmod", "-x", getResultFullpath(main_module))) # Execute the module immediately if option was given. if Options.shallExecuteImmediately(): if Options.shallMakeModule(): executeModule(tree=main_module, clean_path=Options.shallClearPythonPathEnvironment()) else: executeMain( binary_filename=getResultFullpath(main_module), clean_path=Options.shallClearPythonPathEnvironment() )