def _addSlaveMainModule(root_module): from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name=module_name, package_name=None, mode=mode, source_ref=root_module.getSourceReference()) source_code = readSourceCodeFromFilename("__parents_main__", root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.main()""" createModuleTree(module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False) # This is an alternative entry point of course. addRootModule(slave_main_module)
def _createTriggerLoadedModule(module, trigger_name, code): from nuitka.tree.Building import createModuleTree from nuitka.nodes.ModuleNodes import CompiledPythonModule from nuitka.plugins.Plugins import Plugins module_name = module.getName() + trigger_name source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name) mode = Plugins.decideCompilation(module_name, source_ref) trigger_module = CompiledPythonModule( name = module_name, package_name = module.getPackage(), mode = mode, source_ref = source_ref ) createModuleTree( module = trigger_module, source_ref = module.getSourceReference(), source_code = code, is_main = False ) return trigger_module
def decideCompilationMode(is_top, module_name, source_ref): result = Plugins.decideCompilation(module_name, source_ref) if result == "bytecode" and is_top: plugins_logger.warning("""\ Ignoring plugin decision to compile top level package '%s' as bytecode, the extension module entry point is technically required to compiled.""" % module_name) result = "compiled" return result
def onModuleInitialSet(self): from nuitka.importing.ImportCache import addImportedModule from nuitka.ModuleRegistry import getRootTopModule from nuitka.plugins.Plugins import Plugins from nuitka.tree.Building import ( CompiledPythonModule, createModuleTree, readSourceCodeFromFilename, ) # First, build the module node and then read again from the # source code. root_module = getRootTopModule() module_name = ModuleName("__parents_main__") source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( module_name=module_name, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename(module_name, root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4. if python_version >= 0x340: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) addImportedModule(imported_module=slave_main_module) yield slave_main_module
def _addSlaveMainModule(root_module): from nuitka.tree.Building import ( CompiledPythonModule, readSourceCodeFromFilename, createModuleTree, ) from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins from sys import hexversion # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name=module_name, package_name=None, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename( "__parents_main__", root_module.getFilename() ) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4.0.a2 if hexversion >= 0x030400A2: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) # This is an alternative entry point of course. addRootModule(slave_main_module)
def _addSlaveMainModule(root_module): from nuitka.tree.Building import ( CompiledPythonModule, readSourceCodeFromFilename, createModuleTree, ) from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins from sys import hexversion # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name=module_name, package_name=None, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename("__parents_main__", root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4.0.a2 if hexversion >= 0x030400A2: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) # This is an alternative entry point of course. addRootModule(slave_main_module)
def _createTriggerLoadedModule(module, trigger_name, code): from nuitka.tree.Building import createModuleTree from nuitka.nodes.ModuleNodes import CompiledPythonModule from nuitka.plugins.Plugins import Plugins module_name = module.getName() + trigger_name source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name) mode = Plugins.decideCompilation(module_name, source_ref) trigger_module = CompiledPythonModule(name=module_name, package_name=module.getPackage(), mode=mode, source_ref=source_ref) createModuleTree(module=trigger_module, source_ref=module.getSourceReference(), source_code=code, is_main=False) return trigger_module
def _addSlaveMainModule(root_module): from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree from nuitka.ModuleRegistry import addRootModule from nuitka.plugins.Plugins import Plugins # First, build the module node and then read again from the # source code. module_name = "__parents_main__" source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( name = module_name, package_name = None, mode = mode, source_ref = root_module.getSourceReference() ) source_code = readSourceCodeFromFilename( "__parents_main__", root_module.getFilename() ) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.main()""" createModuleTree( module = slave_main_module, source_ref = root_module.getSourceReference(), source_code = source_code, is_main = False ) # This is an alternative entry point of course. addRootModule(slave_main_module)
def decideCompilationMode(is_top, module_name, for_pgo): """Decide the compilation mode for a module. module_name - The module to decide compilation mode for. for_pgo - consider PGO information or not """ result = Plugins.decideCompilation(module_name) # Cannot change mode of __main__ to bytecode, that is not going # to work currently. if result == "bytecode" and is_top: plugins_logger.warning("""\ Ignoring plugin decision to compile top level package '%s' as bytecode, the extension module entry point is technically required to compiled.""" % module_name) result = "compiled" # Include all of standard library as bytecode, for now. We need to identify # which ones really need that. if not is_top: module_filename = Importing.locateModule(module_name=module_name, parent_package=None, level=0)[1] if module_filename is not None and isStandardLibraryPath( module_filename): result = "bytecode" # Plugins need to win over PGO, as they might know it better if result is None and not for_pgo: result = decideCompilationFromPGO(module_name=module_name) # Default if neither plugins nor PGO have expressed an opinion if result is None: result = "compiled" return result
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements assert package is None or type(package) is str assert filename is not None if is_main and os.path.isdir(filename): source_filename = os.path.join(filename, "__main__.py") if not os.path.isfile(source_filename): sys.stderr.write("%s: can't find '__main__' module in '%s'\n" % (os.path.basename(sys.argv[0]), filename)) sys.exit(2) filename = source_filename main_added = True else: main_added = False if os.path.isfile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename(filename=filename, ) if is_main: module_name = "__main__" else: module_name = os.path.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % (module_name)) sys.exit(2) if is_shlib: result = PythonShlibModule(name=module_name, package_name=package, source_ref=source_ref) elif is_main: result = PythonMainModule(main_added=main_added, mode=Plugins.decideCompilation( "__main__", source_ref), future_spec=None, source_ref=source_ref) else: if package is not None: full_name = package + '.' + module_name else: full_name = module_name result = CompiledPythonModule(name=module_name, package_name=package, mode=Plugins.decideCompilation( full_name, source_ref), future_spec=None, source_ref=source_ref) elif Importing.isPackageDir(filename): if is_top: module_name = splitPath(filename)[-1] else: module_name = os.path.basename(filename) source_filename = os.path.join(filename, "__init__.py") if not os.path.isfile(source_filename): source_ref, result = createNamespacePackage( module_name=module_name, package_name=package, module_relpath=filename) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename=os.path.abspath(source_filename), ) if package is not None: full_name = package + '.' + module_name else: full_name = module_name result = CompiledPythonPackage(name=module_name, package_name=package, mode=Plugins.decideCompilation( full_name, source_ref), future_spec=None, source_ref=source_ref) assert result.getFullName() == full_name, result else: sys.stderr.write("%s: can't open file '%s'.\n" % (os.path.basename(sys.argv[0]), filename)) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915 assert package is None or type(package) is str assert filename is not None if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename( filename = filename, ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % ( module_name ) ) sys.exit(2) if is_shlib: result = PythonShlibModule( name = module_name, package_name = package, source_ref = source_ref ) elif is_main: result = PythonMainModule( main_added = main_added, source_ref = source_ref ) else: if package is not None: full_name = package + "." + module_name else: full_name = module_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonModule( name = module_name, package_name = package, source_ref = source_ref ) else: source_code = readSourceCodeFromFilename(module_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name = full_name, is_package = False, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = False, bytecode = bytecode ) result = UncompiledPythonModule( name = module_name, package_name = package, bytecode = bytecode, filename = filename, user_provided = True, technical = False, source_ref = source_ref ) # Don't read it anymore. source_filename = None elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): source_ref, result = createNamespacePackage( package_name = package_name, module_relpath = filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename = Utils.abspath(source_filename), ) if package is not None: full_name = package + "." + package_name else: full_name = package_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonPackage( name = package_name, package_name = package, source_ref = source_ref ) else: bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = False, bytecode = bytecode ) result = UncompiledPythonPackage( name = module_name, package_name = package, bytecode = bytecode, filename = filename, user_provided = True, technical = False, source_ref = source_ref ) # Don't read it anymore. source_filename = None else: sys.stderr.write( "%s: can't open file '%s'.\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915 assert package is None or type(package) is str assert filename is not None if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write("%s: can't find '__main__' module in '%s'\n" % (Utils.basename(sys.argv[0]), filename)) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename(filename=filename, ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % (module_name)) sys.exit(2) if is_shlib: result = PythonShlibModule(name=module_name, package_name=package, source_ref=source_ref) elif is_main: result = PythonMainModule(main_added=main_added, source_ref=source_ref) else: if package is not None: full_name = package + "." + module_name else: full_name = module_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonModule(name=module_name, package_name=package, source_ref=source_ref) else: source_code = readSourceCodeFromFilename(module_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name=module_name, is_package=False, bytecode=bytecode) result = UncompiledPythonModule(name=module_name, package_name=package, bytecode=bytecode, filename=filename, user_provided=True, technical=False, source_ref=source_ref) # Don't read it anymore. source_filename = None elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): source_ref, result = createNamespacePackage( package_name=package_name, module_relpath=filename) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename=Utils.abspath(source_filename), ) if package is not None: full_name = package + "." + package_name else: full_name = package_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonPackage(name=package_name, package_name=package, source_ref=source_ref) else: bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name=module_name, is_package=False, bytecode=bytecode) result = UncompiledPythonPackage(name=module_name, package_name=package, bytecode=bytecode, filename=filename, user_provided=True, technical=False, source_ref=source_ref) # Don't read it anymore. source_filename = None else: sys.stderr.write("%s: can't open file '%s'.\n" % (Utils.basename(sys.argv[0]), filename)) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename