def getModuleNameAndKindFromFilename(module_filename): """Given a filename, decide the module name and kind. Args: module_name - file path of the module Returns: Tuple with the name of the module basename, and the kind of the module derived from the file suffix. Can be None, None if is is not a known file suffix. Notes: This doesn't concern itself with packages, that needs to be tracked by the using code. It cannot be decided from the filename at all. """ # TODO: This does not handle ".pyw" files it seems. if os.path.isdir(module_filename): module_name = ModuleName(os.path.basename(module_filename)) module_kind = "py" elif module_filename.endswith(".py"): module_name = ModuleName(os.path.basename(module_filename)[:-3]) module_kind = "py" else: for suffix in getSharedLibrarySuffixes(): if module_filename.endswith(suffix): module_name = ModuleName( os.path.basename(module_filename)[: -len(suffix)] ) module_kind = "extension" break else: module_kind = None module_name = None return module_name, module_kind
def buildMainModuleTree(filename, is_main): # Detect to be frozen modules if any, so we can consider to not follow # to them. if is_main: # TODO: Doesn't work for deeply nested packages at all. if Options.hasPythonFlagPackageMode(): module_name = ModuleName(os.path.basename(filename) + ".__main__") else: module_name = ModuleName("__main__") else: module_name = Importing.getModuleNameAndKindFromFilename(filename)[0] module, _added = buildModule( module_name=module_name, module_filename=filename, source_code=None, is_top=True, is_main=is_main, is_extension=False, is_fake=False, hide_syntax_error=False, ) if is_main and Options.isStandaloneMode(): module.setEarlyModules(detectEarlyImports()) # Main modules do not get added to the import cache, but plugins get to see it. if module.isMainModule(): Plugins.onModuleDiscovered(module) else: addImportedModule(imported_module=module) return module
def _find_to_build(self): """ Helper for _build Returns list containing bool (is_package) and module_names Algorithm for finding distinct packages: 1) Take minimum package 2) Find related packages that start with this name 3) Add this to the list to return, then repeat steps 1 & 2 until no more packages exist """ builds = [] # Namespace packages can use / rather than dots. py_packages = [ ModuleName(m.replace("/", ".")) for m in sorted(set(self.compile_packages)) ] py_modules = [ModuleName(m) for m in sorted(set(self.py_modules))] for script_module_name in self.script_module_names: script_module_filename = locateModule( module_name=script_module_name, parent_package=None, level=0)[1] # Decide package or module. ( _main_added, is_package, _is_namespace, _source_ref, _source_filename, ) = decideModuleSourceRef( filename=script_module_filename, module_name=script_module_name, is_main=False, is_fake=False, logger=wheel_logger, ) if is_package: py_packages.append(script_module_name) else: py_modules.append(script_module_name) # Plain modules if they are not in packages to build. builds.extend((False, current_module) for current_module in py_modules if not current_module.hasOneOfNamespaces(py_packages)) while py_packages: current_package = min(py_packages) py_packages = [ p for p in py_packages if not p.hasNamespace(current_package) ] builds.append((True, current_package)) return builds
def considerImplicitImports(self, module, signal_change): """Provide additional modules to import implicitly when encountering the module. Notes: Better do not overload this method. The standard plugin 'ImplicitImports.py' already contains MANY of these. If you do have a new candidate, consider a PR to get it included there. Args: module: the module object signal_change: bool Returns: None """ from nuitka.importing.Importing import getModuleNameAndKindFromFilename for full_name in self.getImplicitImports(module): if type(full_name) in (tuple, list): raise NuitkaPluginError( "Plugin %s needs to be change to only return modules names, not %r" % (self, full_name)) full_name = ModuleName(full_name) try: module_filename = self.locateModule(importing=module, module_name=full_name) except Exception: self.warning("Problem locating '%s' implicit imports '%s'." % (module.getFullName(), full_name)) raise if module_filename is None: if Options.isShowInclusion(): self.info( "Implicit module '%s' suggested by '%s' not found." % (full_name, module.getFullName())) continue _module_name2, module_kind = getModuleNameAndKindFromFilename( module_filename) # TODO: This should get back to plug-ins, they should be allowed to # preempt or override the decision. decision, reason = self.decideRecursion( module_filename=module_filename, module_name=full_name, module_kind=module_kind, ) if decision: self.recurseTo( module_package=full_name.getPackageName(), module_filename=module_filename, module_kind=module_kind, reason=reason, signal_change=signal_change, )
def considerImplicitImports(self, module, signal_change): """ Provide additional modules to import implicitly when encountering the module. Notes: Better do not overload this method. The standard plugin 'ImplicitImports.py' already contains MANY of these. If you do have a new candidate, consider a PR to get it included there. Args: module: the module object signal_change: bool Returns: None """ from nuitka.importing.Importing import getModuleNameAndKindFromFilename for item in self.getImplicitImports(module): # TODO: Temporary, until all plugins are caught up, turn into an error later. if type(item) in (tuple, list): full_name, _required = item else: full_name = item full_name = ModuleName(full_name) module_filename = self.locateModule(importing=module, module_name=full_name) if module_filename is None: if Options.isShowInclusion(): self.info( "Implicit module '%s' suggested by '%s' not found." % (full_name, module.getFullName())) continue _module_name2, module_kind = getModuleNameAndKindFromFilename( module_filename) # TODO: This should get back to plug-ins, they should be allowed to # preempt or override the decision. decision, reason = self.decideRecursion( module_filename=module_filename, module_name=full_name, module_kind=module_kind, ) if decision: self.recurseTo( module_package=full_name.getPackageName(), module_filename=module_filename, module_kind=module_kind, reason=reason, signal_change=signal_change, )
def considerImplicitImports(self, module, signal_change): """ Provide additional modules to import implicitly when encountering the module. Notes: Better do not overload this method. The standard plugin 'ImplicitImports.py' already contains MANY of these. If you do have a new candidate, consider a PR to get it included there. Args: module: the module object signal_change: bool Returns: None """ from nuitka.importing.Importing import getModuleNameAndKindFromFilename for full_name, required in self.getImplicitImports(module): full_name = ModuleName(full_name) module_filename = self.locateModule( importing=module, module_name=full_name, warn=required ) if module_filename is None: if required: sys.exit( "Error, implicit module '%s' expected by '%s' not found." % (full_name, module.getFullName()) ) else: continue _module_name2, module_kind = getModuleNameAndKindFromFilename( module_filename ) # TODO: This should get back to plug-ins, they should be allowed to # preempt or override the decision. decision, reason = self.decideRecursion( module_filename=module_filename, module_name=full_name, module_kind=module_kind, ) if decision: self.recurseTo( module_package=full_name.getPackageName(), module_filename=module_filename, module_kind=module_kind, reason=reason, signal_change=signal_change, )
def locateModules(package_name): """Determine child module names. Return: generator of ModuleName objects """ package_name = ModuleName(package_name) module_filename = locateModule( module_name=ModuleName(package_name), parent_package=None, level=0 )[1] if module_filename is not None: for sub_module in iter_modules([module_filename]): yield package_name.getChildNamed(sub_module.name)
def getCachedImportedModulesNames(module_name, source_code): cache_name = makeCacheName(module_name, source_code) return [ ModuleName(line) for line in getFileContents( _getCacheFilename(cache_name, "txt")).strip().split("\n") ]
def __init__(self, qt_plugins, no_qt_translations): self.qt_plugins = OrderedSet(x.strip().lower() for x in qt_plugins.split(",")) self.no_qt_translations = no_qt_translations self.webengine_done_binaries = False self.webengine_done_data = False self.qt_plugins_dirs = None self.binding_package_name = ModuleName(self.binding_name) # Allow to specify none. if self.qt_plugins == set(["none"]): self.qt_plugins = set() # Prevent the list of binding names from being incomplete, it's used for conflicts. assert self.binding_name in _qt_binding_names, self.binding_name # Also lets have consistency in naming. assert self.plugin_name in getQtPluginNames() active_qt_plugin_name = getActiveQtPlugin() if active_qt_plugin_name is not None: self.sysexit( "Error, confliciting plugin '%s', you can only have one enabled." % active_qt_plugin_name) self.warned_about = set()
def _scanModules(path, prefix): for module_info in pkgutil.walk_packages((path, ), prefix=prefix + "."): result.append(ModuleName(module_info[1])) if module_info[2]: _scanModules(module_info[1], module_name + module_info[1])
def locateModule(module_name, parent_package, level): """Locate a module with given package name as parent. The package name can be None of course. Level is the same as with "__import__" built-in. Returns: Returns a triple of module name the module has considering package containing it, and filename of it which can be a directory for packages, and the location method used. """ module_package, module_filename, finding = findModule( module_name=module_name, parent_package=parent_package, level=level, ) assert module_package is None or ( type(module_package) is ModuleName and module_package != "" ), repr(module_package) if module_filename is not None: module_filename = os.path.normpath(module_filename) module_name, module_kind = getModuleNameAndKindFromFilename(module_filename) assert module_kind is not None, module_filename module_name = ModuleName.makeModuleNameInPackage(module_name, module_package) return module_name, module_filename, finding
def _consider(self, trace_collection, module_filename, module_package): assert module_package is None or ( type(module_package) is ModuleName and module_package != ""), repr(module_package) module_filename = os.path.normpath(module_filename) module_name, module_kind = getModuleNameAndKindFromFilename( module_filename) if module_kind is not None: module_fullpath = ModuleName.makeModuleNameInPackage( module_name, module_package) decision, reason = decideRecursion( module_filename=module_filename, module_name=module_fullpath, module_kind=module_kind, ) if decision: module_relpath = relpath(module_filename) imported_module, added_flag = recurseTo( module_package=module_package, module_filename=module_filename, module_relpath=module_relpath, module_kind=module_kind, reason=reason, ) if added_flag: trace_collection.signalChange( "new_code", imported_module.getSourceReference(), "Recursed to module.", ) return imported_module elif decision is False and module_kind == "py": uncompiled_module = getUncompiledModule( module_fullpath, module_filename) if uncompiled_module is not None: return uncompiled_module elif decision is None and module_kind == "py": if (module_filename not in self._warned_about and module_fullpath not in getModuleIgnoreList()): self._warned_about.add(module_filename) inclusion_logger.warning("""\ Not recursing to '%(full_path)s' (%(filename)s), please specify \ --nofollow-imports (do not warn), \ --follow-imports (recurse to all), \ --nofollow-import-to=%(full_path)s (ignore it), \ --follow-import-to=%(full_path)s (recurse to it) to change.""" % { "full_path": module_fullpath, "filename": module_filename })
def considerFailedImportReferrals(module_name): for plugin in active_plugin_list: new_module_name = plugin.considerFailedImportReferrals(module_name) if new_module_name is not None: return ModuleName(new_module_name) return None
def considerFailedImportReferrals(module_name): for plugin in getActivePlugins(): new_module_name = plugin.considerFailedImportReferrals(module_name) if new_module_name is not None: return ModuleName(new_module_name) return None
def normalizePackageName(module_name): # The "os.path" is strangely hacked into the "os" module, dispatching per # platform, we either cannot look into it, or we require that we resolve it # here correctly. if module_name == "os.path": module_name = ModuleName(os.path.basename(os.path.__name__)) return module_name
def makeIncludedEntryPoint(kind, source_path, dest_path, package_name, executable): if package_name is not None: package_name = ModuleName(package_name) assert type(executable) is bool return IncludedEntryPoint(kind, source_path, dest_path, package_name, executable)
def __init__(self): CompiledPythonModule.__init__( self, module_name=ModuleName("__internal__"), is_top=False, mode="compiled", source_ref=SourceCodeReference.fromFilenameAndLine( filename="internal", line=0), future_spec=FutureSpec(), )
def makeIncludedEntryPoint( kind, source_path, dest_path, package_name, ): if package_name is not None: package_name = ModuleName(package_name) return IncludedEntryPoint(kind, source_path, dest_path, package_name)
def getImplicitImports(self, module): if module.getFullName() == "OpenGL": opengl_infos = self.queryRuntimeInformationSingle( setup_codes="import OpenGL.plugins", value= "[(f.name, f.import_path) for f in OpenGL.plugins.FormatHandler.all()]", ) # TODO: Filter by name. for _name, import_path in opengl_infos: yield ModuleName(import_path).getPackageName()
def __init__( self, noinclude_setuptools_mode, noinclude_pytest_mode, noinclude_ipython_mode, noinclude_default_mode, custom_choices, ): # Default manually to default argument value: if noinclude_setuptools_mode is None: noinclude_setuptools_mode = noinclude_default_mode if noinclude_pytest_mode is None: noinclude_pytest_mode = noinclude_default_mode if noinclude_ipython_mode is None: noinclude_ipython_mode = noinclude_default_mode self.config = parsePackageYaml(__package__, "anti-bloat.yml") self.handled_modules = OrderedDict() # These should be checked, to allow disabling anti-bloat contents. self.control_tags = set() if noinclude_setuptools_mode != "allow": self.handled_modules["setuptools"] = noinclude_setuptools_mode else: self.control_tags.add("allow_setuptools") if noinclude_pytest_mode != "allow": self.handled_modules["pytest"] = noinclude_pytest_mode else: self.control_tags.add("allow_pytest") if noinclude_ipython_mode != "allow": self.handled_modules["IPython"] = noinclude_ipython_mode else: self.control_tags.add("allow_ipython") for custom_choice in custom_choices: if ":" not in custom_choice: self.sysexit( "Error, malformed value '%s' for '--noinclude-custom-mode' used." % custom_choice ) module_name, mode = custom_choice.rsplit(":", 1) if mode not in ("error", "warning", "nofollow", "allow", "bytecode"): self.sysexit( "Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'" % (mode, custom_choice) ) self.handled_modules[ModuleName(module_name)] = mode
def fromXML(cls, provider, source_ref, **args): if "code_flags" in args: future_spec = fromFlags(args["code_flags"]) result = cls( main_added=args["main_added"] == "True", mode=args["mode"], module_name=ModuleName(args["module_name"]), future_spec=future_spec, source_ref=source_ref, ) from nuitka.ModuleRegistry import addRootModule addRootModule(result) function_work = [] for xml in args["functions"]: _kind, node_class, func_args, source_ref = extractKindAndArgsFromXML( xml, source_ref) if "provider" in func_args: func_args["provider"] = getOwnerFromCodeName( func_args["provider"]) else: func_args["provider"] = result if "flags" in args: func_args["flags"] = set(func_args["flags"].split(",")) if "doc" not in args: func_args["doc"] = None function = node_class.fromXML(source_ref=source_ref, **func_args) # Could do more checks for look up of body here, but so what... function_work.append((function, iter(iter(xml).next()).next())) for function, xml in function_work: function.setChild( "body", fromXML(provider=function, xml=xml, source_ref=function.getSourceReference()), ) result.setChild( "body", fromXML(provider=result, xml=args["body"][0], source_ref=source_ref)) return result
def __init__(self, main_added, mode, future_spec, source_ref): self.main_added = main_added CompiledPythonModule.__init__( self, module_name=ModuleName("__main__"), is_top=True, mode=mode, future_spec=future_spec, source_ref=source_ref, )
def _createTriggerLoadedModule(module, trigger_name, code): """Create a "trigger" for a module to be imported. Notes: The trigger will incorpaorate the code to be prepended / appended. Called by @onModuleDiscovered. Args: module: the module object (serves as dict key) trigger_name: string ("-preload"/"-postload") code: the code string Returns trigger_module """ from nuitka.nodes.ModuleNodes import CompiledPythonModule from nuitka.tree.Building import createModuleTree from .Plugins import Plugins module_name = ModuleName(module.getFullName() + trigger_name) source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name) mode = Plugins.decideCompilation(module_name, source_ref) trigger_module = CompiledPythonModule( module_name=module_name, is_top=False, mode=mode, future_spec=None, source_ref=source_ref, ) createModuleTree( module=trigger_module, source_ref=module.getSourceReference(), source_code=code, is_main=False, ) if mode == "bytecode": trigger_module.setSourceCode(code) if Options.isDebug(): source_path = os.path.join( OutputDirectories.getSourceDirectoryPath(), module_name + ".py") with open(source_path, "w") as output: output.write(code) return trigger_module
def onModuleInitialSet(self): from nuitka.importing.ImportCache import addImportedModule from nuitka.ModuleRegistry import getRootTopModule from nuitka.plugins.Plugins import Plugins from nuitka.tree.Building import ( CompiledPythonModule, createModuleTree, readSourceCodeFromFilename, ) # First, build the module node and then read again from the # source code. root_module = getRootTopModule() module_name = ModuleName("__parents_main__") source_ref = root_module.getSourceReference() mode = Plugins.decideCompilation(module_name, source_ref) slave_main_module = CompiledPythonModule( module_name=module_name, is_top=False, mode=mode, future_spec=None, source_ref=root_module.getSourceReference(), ) source_code = readSourceCodeFromFilename(module_name, root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4. if python_version >= 0x340: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" createModuleTree( module=slave_main_module, source_ref=root_module.getSourceReference(), source_code=source_code, is_main=False, ) addImportedModule(imported_module=slave_main_module) yield slave_main_module
def _createTriggerLoadedModule(module, trigger_name, code, flags): """Create a "trigger" for a module to be imported. Notes: The trigger will incorpaorate the code to be prepended / appended. Called by @onModuleDiscovered. Args: module: the module object (serves as dict key) trigger_name: string ("-preload"/"-postload") code: the code string Returns trigger_module """ from nuitka.nodes.ModuleNodes import CompiledPythonModule from nuitka.tree.Building import createModuleTree from .Plugins import Plugins module_name = ModuleName(module.getFullName() + trigger_name) source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name) mode = Plugins.decideCompilation(module_name, source_ref) trigger_module = CompiledPythonModule( module_name=module_name, is_top=False, mode=mode, future_spec=None, source_ref=source_ref, ) createModuleTree( module=trigger_module, source_ref=module.getSourceReference(), source_code=code, is_main=False, ) if mode == "bytecode": trigger_module.setSourceCode(code) # In debug mode, put the files in the build folder, so they can be looked up easily. if Options.is_debug and "HIDE_SOURCE" not in flags: source_path = os.path.join( OutputDirectories.getSourceDirectoryPath(), module_name + ".py") putTextFileContents(filename=source_path, contents=code) return trigger_module
def _considerImplicitImports(plugin, module): from nuitka.importing import Importing result = [] for full_name in plugin.getImplicitImports(module): if type(full_name) in (tuple, list): raise NuitkaPluginError( "Plugin %r needs to be change to only return modules names, not %r" % (plugin, full_name) ) full_name = ModuleName(full_name) try: _module_package, module_filename, _finding = Importing.findModule( importing=module, module_name=full_name, parent_package=None, level=-1, warn=False, ) module_filename = plugin.locateModule( importing=module, module_name=full_name ) except Exception: plugin.warning( "Problem locating '%s' for implicit imports of '%s'." % (module.getFullName(), full_name) ) raise if module_filename is None: if Options.isShowInclusion(): plugin.info( "Implicit module '%s' suggested for '%s' not found." % (full_name, module.getFullName()) ) continue result.append((full_name, module_filename)) if result: plugin.info( "Implicit dependencies of module '%s' added '%s'." % (module.getFullName(), ",".join(r[0] for r in result)) ) return result
def __init__(self, main_added, mode, future_spec, source_ref): # Is this one from a "__main__.py" file. self.main_added = main_added CompiledPythonModule.__init__( self, module_name=ModuleName("__main__"), is_top=True, mode=mode, future_spec=future_spec, source_ref=source_ref, ) self.early_modules = ()
def locateModule(module_name): """Provide a filename / -path for a to-be-imported module. Args: importing: module object that asked for it (tracing only) module_name: (str or ModuleName) full name of module Returns: filename for module """ from nuitka.importing.Importing import locateModule _module_name, module_filename, _finding = locateModule( module_name=ModuleName(module_name), parent_package=None, level=0) return module_filename
def __init__(self, module_name, source_ref): ExpressionBase.__init__(self, source_ref=source_ref) self.module_name = ModuleName(module_name) self.finding = None self.module_filename = None _module_name, self.module_filename, self.finding = locateModule( module_name=self.module_name, parent_package=None, level=0, ) # Expect to find them and to match the name of course. assert self.finding != "not-found", self.module_name assert _module_name == self.module_name
def getCachedImportedModulesNames(module_name, source_code): cache_name = makeCacheName(module_name, source_code) cache_filename = _getCacheFilename(cache_name, "json") if not os.path.exists(cache_filename): return None data = loadJsonFromFilename(cache_filename) if data is None: return None if data.get("file_format_version") != _cache_format_version: return None if data["module_name"] != module_name: return None return [(ModuleName(used_module_name), line_number) for (used_module_name, line_number) in data["modules_used"]]