def locateModule(module_name, parent_package, level): """Locate a module with given package name as parent. The package name can be None of course. Level is the same as with "__import__" built-in. Returns: Returns a triple of module name the module has considering package containing it, and filename of it which can be a directory for packages, and the location method used. """ module_package, module_filename, finding = findModule( module_name=module_name, parent_package=parent_package, level=level, ) assert module_package is None or ( type(module_package) is ModuleName and module_package != "" ), repr(module_package) if module_filename is not None: module_filename = os.path.normpath(module_filename) module_name, module_kind = getModuleNameAndKindFromFilename(module_filename) assert module_kind is not None, module_filename module_name = ModuleName.makeModuleNameInPackage(module_name, module_package) return module_name, module_filename, finding
def _consider(self, trace_collection, module_filename, module_package): assert module_package is None or ( type(module_package) is ModuleName and module_package != ""), repr(module_package) module_filename = os.path.normpath(module_filename) module_name, module_kind = getModuleNameAndKindFromFilename( module_filename) if module_kind is not None: module_fullpath = ModuleName.makeModuleNameInPackage( module_name, module_package) decision, reason = decideRecursion( module_filename=module_filename, module_name=module_fullpath, module_kind=module_kind, ) if decision: module_relpath = relpath(module_filename) imported_module, added_flag = recurseTo( module_package=module_package, module_filename=module_filename, module_relpath=module_relpath, module_kind=module_kind, reason=reason, ) if added_flag: trace_collection.signalChange( "new_code", imported_module.getSourceReference(), "Recursed to module.", ) return imported_module elif decision is False and module_kind == "py": uncompiled_module = getUncompiledModule( module_fullpath, module_filename) if uncompiled_module is not None: return uncompiled_module elif decision is None and module_kind == "py": if (module_filename not in self._warned_about and module_fullpath not in getModuleIgnoreList()): self._warned_about.add(module_filename) inclusion_logger.warning("""\ Not recursing to '%(full_path)s' (%(filename)s), please specify \ --nofollow-imports (do not warn), \ --follow-imports (recurse to all), \ --nofollow-import-to=%(full_path)s (ignore it), \ --follow-import-to=%(full_path)s (recurse to it) to change.""" % { "full_path": module_fullpath, "filename": module_filename })
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=too-many-branches assert package is None or type(package) is ModuleName assert filename is not None if is_main and os.path.isdir(filename): source_filename = os.path.join(filename, "__main__.py") if not os.path.isfile(source_filename): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % (os.path.basename(sys.argv[0]), filename) ) sys.exit(2) filename = source_filename main_added = True else: main_added = False if os.path.isfile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename(filename=filename) if is_main: module_name = ModuleName("__main__") else: # Derive module name from filename. module_name = os.path.basename(filename) if is_shlib: module_name = module_name.split(".")[0] elif module_name.endswith(".py"): module_name = module_name[:-3] if "." in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % (module_name) ) sys.exit(2) module_name = ModuleName.makeModuleNameInPackage(module_name, package) if is_shlib: result = PythonShlibModule(module_name=module_name, source_ref=source_ref) elif is_main: result = PythonMainModule( main_added=main_added, mode=decideCompilationMode(False, module_name, source_ref), future_spec=None, source_ref=source_ref, ) else: result = CompiledPythonModule( module_name=module_name, is_top=is_top, mode=decideCompilationMode(is_top, module_name, source_ref), future_spec=None, source_ref=source_ref, ) elif Importing.isPackageDir(filename): if is_top: module_name = splitPath(filename)[-1] else: module_name = os.path.basename(filename) module_name = ModuleName.makeModuleNameInPackage(module_name, package) source_filename = os.path.join(filename, "__init__.py") if not os.path.isfile(source_filename): source_ref, result = createNamespacePackage( module_name=module_name, is_top=is_top, module_relpath=filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename=os.path.abspath(source_filename) ) result = CompiledPythonPackage( module_name=module_name, is_top=is_top, mode=decideCompilationMode(is_top, module_name, source_ref), future_spec=None, source_ref=source_ref, ) else: sys.stderr.write( "%s: can't open file '%s'.\n" % (os.path.basename(sys.argv[0]), filename) ) sys.exit(2) return result, source_ref, source_filename
def checkPluginSinglePath(plugin_filename, module_package): # Many branches, for the decision is very complex, pylint: disable=too-many-branches debug("Checking detail plug-in path '%s' '%s':", plugin_filename, module_package) module_name, module_kind = Importing.getModuleNameAndKindFromFilename( plugin_filename) module_name = ModuleName.makeModuleNameInPackage(module_name, module_package) if module_kind is not None: decision, reason = decideRecursion( module_filename=plugin_filename, module_name=module_name, module_kind=module_kind, extra_recursion=True, ) if decision: module_relpath = relpath(plugin_filename) module, is_added = recurseTo( module_filename=plugin_filename, module_relpath=module_relpath, module_package=module_package, module_kind=module_kind, reason=reason, ) if module: if not is_added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isCompiledPythonPackage() else "module", module.getName(), plugin_filename, ) if not isSameModulePath(module.getFilename(), plugin_filename): warning( "Duplicate '%s' of '%s' ignored .", plugin_filename, module.getFilename(), ) return debug( "Recursed to %s %s %s", module.getName(), module.getFullName().getPackageName(), module, ) ImportCache.addImportedModule(module) if module.isCompiledPythonPackage(): package_filename = module.getFilename() if os.path.isdir(package_filename): # Must be a namespace package. assert python_version >= 300 package_dir = package_filename # Only include it, if it contains actual modules, which will # recurse to this one and find it again. else: package_dir = os.path.dirname(package_filename) # Real packages will always be included. ModuleRegistry.addRootModule(module) debug("Package directory %s", package_dir) for sub_path, sub_filename in listDir(package_dir): if sub_filename in ("__init__.py", "__pycache__"): continue assert sub_path != plugin_filename if Importing.isPackageDir( sub_path) or sub_path.endswith(".py"): checkPluginSinglePath(sub_path, module.getFullName()) elif module.isCompiledPythonModule(): ModuleRegistry.addRootModule(module) else: warning("Failed to include module from '%s'.", plugin_filename)
def checkPluginSinglePath(plugin_filename, module_package): # Many branches, for the decision is very complex, pylint: disable=too-many-branches # The importing wants these to be unique. plugin_filename = os.path.abspath(plugin_filename) if Options.isShowInclusion(): recursion_logger.info("Checking detail plug-in path '%s' '%s':" % (plugin_filename, module_package)) module_name, module_kind = Importing.getModuleNameAndKindFromFilename( plugin_filename) module_name = ModuleName.makeModuleNameInPackage(module_name, module_package) if module_kind == "extension" and not Options.isStandaloneMode(): recursion_logger.warning( "Cannot include '%s' unless using at least standalone mode." % module_name.asString()) if module_kind is not None: decision, reason = decideRecursion( module_filename=plugin_filename, module_name=module_name, module_kind=module_kind, extra_recursion=True, ) if decision: module = recurseTo( signal_change=None, module_filename=plugin_filename, module_name=module_name, module_kind=module_kind, reason=reason, ) if module: if Options.isShowInclusion(): recursion_logger.info("Included '%s' as '%s'." % ( module.getFullName(), module, )) ImportCache.addImportedModule(module) if module.isCompiledPythonPackage(): package_filename = module.getFilename() if os.path.isdir(package_filename): # Must be a namespace package. assert python_version >= 0x300 package_dir = package_filename # Only include it, if it contains actual modules, which will # recurse to this one and find it again. else: package_dir = os.path.dirname(package_filename) # Real packages will always be included. ModuleRegistry.addRootModule(module) if Options.isShowInclusion(): recursion_logger.info("Package directory '%s'." % package_dir) for sub_path, sub_filename in listDir(package_dir): if sub_filename in ("__init__.py", "__pycache__"): continue assert sub_path != plugin_filename if Importing.isPackageDir( sub_path) and not os.path.exists(sub_path + ".py"): checkPluginSinglePath( sub_path, module_package=module.getFullName()) elif sub_path.endswith(".py"): checkPluginSinglePath( sub_path, module_package=module.getFullName()) elif module.isCompiledPythonModule(): ModuleRegistry.addRootModule(module) elif module.isPythonExtensionModule(): if Options.isStandaloneMode(): ModuleRegistry.addRootModule(module) else: recursion_logger.warning( "Failed to include module from '%s'." % plugin_filename)
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases # pylint: disable=too-many-branches,too-many-locals,too-many-statements assert package is None or type(package) is ModuleName assert filename is not None if is_main and os.path.isdir(filename): source_filename = os.path.join(filename, "__main__.py") if not os.path.isfile(source_filename): sys.stderr.write("%s: can't find '__main__' module in '%s'\n" % (os.path.basename(sys.argv[0]), filename)) sys.exit(2) filename = source_filename main_added = True else: main_added = False if os.path.isfile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename(filename=filename) if is_main: module_name = ModuleName("__main__") else: # Derive module name from filename. module_name = os.path.basename(filename) if is_shlib: module_name = module_name.split(".")[0] elif module_name.endswith(".py"): module_name = module_name[:-3] if "." in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % (module_name)) sys.exit(2) module_name = ModuleName.makeModuleNameInPackage( module_name, package) if is_shlib: result = PythonShlibModule(module_name=module_name, source_ref=source_ref) source_code = None else: source_code = readSourceCodeFromFilename( module_name=module_name, source_filename=source_filename) if is_main: result = PythonMainModule( main_added=main_added, mode=decideCompilationMode(False, module_name, source_ref), future_spec=None, source_ref=source_ref, ) checkPythonVersionFromCode(source_code) else: mode = decideCompilationMode(is_top, module_name, source_ref) if (mode == "bytecode" and not is_top and hasCachedImportedModulesNames( module_name, source_code)): optimization_logger.info("%r is included as bytecode." % (module_name.asString())) result = UncompiledPythonModule( module_name=module_name, filename=filename, bytecode=demoteSourceCodeToBytecode( module_name=module_name, source_code=source_code, filename=filename, ), source_ref=source_ref, user_provided=False, technical=False, ) used_modules = OrderedSet() for used_module_name in getCachedImportedModulesNames( module_name=module_name, source_code=source_code): ( _module_package, module_filename, _finding, ) = Importing.findModule( importing=result, module_name=used_module_name, parent_package=None, level=-1, warn=False, ) used_modules.add((used_module_name, os.path.relpath(module_filename))) result.setUsedModules(used_modules) # Not used anymore source_code = None else: result = CompiledPythonModule( module_name=module_name, is_top=is_top, mode=mode, future_spec=None, source_ref=source_ref, ) elif Importing.isPackageDir(filename): if is_top: module_name = splitPath(filename)[-1] else: module_name = os.path.basename(filename) module_name = ModuleName.makeModuleNameInPackage(module_name, package) source_filename = os.path.join(filename, "__init__.py") if not os.path.isfile(source_filename): source_ref, result = createNamespacePackage( module_name=module_name, is_top=is_top, module_relpath=filename) source_filename = None source_code = None else: source_ref = SourceCodeReferences.fromFilename( filename=os.path.abspath(source_filename)) result = CompiledPythonPackage( module_name=module_name, is_top=is_top, mode=decideCompilationMode(is_top, module_name, source_ref), future_spec=None, source_ref=source_ref, ) source_code = readSourceCodeFromFilename( module_name=module_name, source_filename=source_filename) else: sys.stderr.write("%s: can't open file '%s'.\n" % (os.path.basename(sys.argv[0]), filename)) sys.exit(2) return result, source_ref, source_code