def checkPluginPath(plugin_filename, module_package): debug( "Checking top level plugin path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: # File or package makes a difference, handle that if Utils.isFile(plugin_info[0]) or \ Importing.isPackageDir(plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0]) else: warning("Failed to recurse to directory '%s'.", plugin_filename)
def checkPluginPath(plugin_filename, module_package): debug( "Checking top level plugin path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: # File or package makes a difference, handle that if Utils.isFile(plugin_info[0]) or \ Importing.isPackageDir(plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0]) else: warning("Failed to recurse to directory '%s'.", plugin_filename)
def _attemptRecursion(self, constraint_collection): assert self.getModule() is None parent_module = self.getParentModule() if parent_module.isPythonPackage(): parent_package = parent_module.getFullName() else: parent_package = self.getParentModule().getPackage() module_package, _module_name, module_filename = Importing.findModule( source_ref = self.source_ref, module_name = self.getModuleName(), parent_package = parent_package, level = self.getLevel(), warn = True ) # That would be an illegal package name, catch it. assert module_package != "" if module_filename is not None: imported_module = self._consider( constraint_collection = constraint_collection, module_filename = module_filename, module_package = module_package ) if imported_module is not None: self.setModule(imported_module) self.found_modules = [] import_list = self.getImportList() if import_list and imported_module.isPythonPackage(): for import_item in import_list: module_package, _module_name, module_filename = \ Importing.findModule( source_ref = self.source_ref, module_name = import_item, parent_package = imported_module.getFullName(), level = -1, warn = False ) if module_filename is not None: sub_imported_module = self._consider( constraint_collection = constraint_collection, module_filename = module_filename, module_package = module_package ) if sub_imported_module is not None: self.found_modules.append(sub_imported_module)
def _attemptRecursion(self, constraint_collection): assert self.getModule() is None parent_module = self.getParentModule() if parent_module.isPythonPackage(): parent_package = parent_module.getFullName() else: parent_package = self.getParentModule().getPackage() module_package, _module_name, module_filename = Importing.findModule( source_ref = self.source_ref, module_name = self.getModuleName(), parent_package = parent_package, level = self.getLevel(), warn = True ) # That would be an illegal package name, catch it. assert module_package != "" if module_filename is not None: imported_module = self._consider( constraint_collection = constraint_collection, module_filename = module_filename, module_package = module_package ) if imported_module is not None: self.setModule(imported_module) self.found_modules = [] import_list = self.getImportList() if import_list and imported_module.isPythonPackage(): for import_item in import_list: module_package, _module_name, module_filename = \ Importing.findModule( source_ref = self.source_ref, module_name = import_item, parent_package = imported_module.getFullName(), level = -1, warn = False ) if module_filename is not None: sub_imported_module = self._consider( constraint_collection = constraint_collection, module_filename = module_filename, module_package = module_package ) if sub_imported_module is not None: self.found_modules.append(sub_imported_module)
def _checkPluginPath(plugin_filename, module_package): plugin_info = considerFilename(module_package=module_package, module_filename=plugin_filename) if plugin_info is not None: module, added = recurseTo(module_filename=plugin_info[0], module_relpath=plugin_info[1], module_package=module_package) if module: if not added: warning("Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0]) if module.isPythonPackage(): package_dir = Utils.dirname(module.getFilename()) for sub_path, sub_filename in Utils.listDir(package_dir): if sub_filename == "__init__.py": continue assert sub_path != plugin_filename, package_dir if Importing.isPackageDir(sub_path) or sub_path.endswith( ".py"): _checkPluginPath(sub_path, module.getFullName()) else: warning("Failed to include module from '%s'.", plugin_info[0])
def considerImplicitImports(self, signal_change): for module_name, module_package in self.getImplicitImports(): _module_package, _module_name, module_filename = \ Importing.findModule( source_ref = self.source_ref, module_name = module_name, parent_package = module_package, level = -1, warn = True ) if module_filename is None: sys.exit( "Error, implicit module '%s' expected by '%s' not found" % ( module_name, self.getFullName() ) ) elif Utils.isDir(module_filename): module_kind = "py" elif module_filename.endswith(".py"): module_kind = "py" elif module_filename.endswith(".so"): module_kind = "shlib" elif module_filename.endswith(".pyd"): module_kind = "shlib" else: assert False, module_filename from nuitka.tree import Recursion decision, reason = Recursion.decideRecursion( module_filename = module_filename, module_name = module_name, module_package = module_package, module_kind = module_kind ) assert decision or reason == "Module is frozen." if decision: module_relpath = Utils.relpath(module_filename) imported_module, added_flag = Recursion.recurseTo( module_package = module_package, module_filename = module_filename, module_relpath = module_relpath, module_kind = module_kind, reason = reason ) from nuitka.ModuleRegistry import addUsedModule addUsedModule(imported_module) if added_flag: signal_change( "new_code", imported_module.getSourceReference(), "Recursed to module." )
def attemptRecursion( self, module ): if not Options.shallMakeModule(): # Make sure the package is recursed to. module_package = module.getPackage() if module_package is not None: package_package, _package_module_name, package_filename = Importing.findModule( source_ref = module.getSourceReference(), module_name = module_package, parent_package = None, level = 1 ) imported_module, added_flag = Recursion.recurseTo( module_package = package_package, module_filename = package_filename, module_relpath = Utils.relpath( package_filename ) ) if added_flag: self.signalChange( "new_code", imported_module.getSourceReference(), "Recursed to module package." )
def considerImplicitImports(self, signal_change): for module_name, module_package in self.getImplicitImports(): _module_package, _module_name, module_filename = \ Importing.findModule( source_ref = self.source_ref, module_name = module_name, parent_package = module_package, level = -1, warn = True ) if module_filename is None: sys.exit( "Error, implicit module '%s' expected by '%s' not found" % ( module_name, self.getFullName() ) ) elif Utils.isDir(module_filename): module_kind = "py" elif module_filename.endswith(".py"): module_kind = "py" elif module_filename.endswith(".so"): module_kind = "shlib" elif module_filename.endswith(".pyd"): module_kind = "shlib" else: assert False, module_filename from nuitka.tree import Recursion decision, reason = Recursion.decideRecursion( module_filename = module_filename, module_name = module_name, module_package = module_package, module_kind = module_kind ) assert decision or reason == "Module is frozen." if decision: module_relpath = Utils.relpath(module_filename) imported_module, added_flag = Recursion.recurseTo( module_package = module_package, module_filename = module_filename, module_relpath = module_relpath, module_kind = module_kind, reason = reason ) from nuitka.ModuleRegistry import addUsedModule addUsedModule(imported_module) if added_flag: signal_change( "new_code", imported_module.getSourceReference(), "Recursed to module." )
def checkPluginPath(plugin_filename, module_package): plugin_info = considerFilename(module_package=module_package, module_filename=plugin_filename) if plugin_info is not None: # File or package, handle that. if Utils.isFile(plugin_info[0]) or Importing.isPackageDir( plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or sub_path.endswith( ".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0])
def checkPluginPath( plugin_filename, module_package ): plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: # File or package, handle that. if Utils.isFile( plugin_info[0] ) or Importing.isPackageDir( plugin_info[0] ): _checkPluginPath( plugin_filename, module_package ) elif Utils.isDir( plugin_info[0] ): for sub_path, sub_filename in Utils.listDir( plugin_info[0] ): assert sub_filename != "__init__.py" if Importing.isPackageDir( sub_path ) or sub_path.endswith( ".py" ): _checkPluginPath( sub_path, None ) else: warning( "Failed to include module from '%s'.", plugin_info[0] )
def decideRecursion(module_filename, module_name, module_package, module_kind): # Many branches, which make decisions immediately, pylint: disable=R0911 if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." if module_package is None: full_name = module_name else: full_name = module_package + "." + module_name if isFrozenModule(full_name): return False, "Module is frozen." no_case_modules = Options.getShallFollowInNoCase() for no_case_module in no_case_modules: if full_name == no_case_module: return (False, "Module listed explicitely to not recurse to.") if full_name.startswith(no_case_module + "."): return (False, "Module in package listed explicitely to not recurse to.") any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return (True, "Module listed explicitely to recurse to.") if full_name.startswith(any_case_module + "."): return (True, "Module in package listed explicitely to recurse to.") if Options.shallFollowNoImports(): return (False, "Requested to not recurse at all.") if Importing.isStandardLibraryPath(module_filename): return (Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ("" if Options.shallFollowStandardLibrary() else "not ")) if Options.shallFollowAllImports(): return (True, "Requested to recurse to all non-standard library modules.") # Means, we were not given instructions how to handle things. return (None, "Default behaviour, not recursing without request.")
def attemptRecursion(self): # Make sure the package is recursed to. from nuitka.tree import Recursion from nuitka import Importing # Return the list of newly added modules. result = [] if self.package_name is not None and self.package is None: package_package, _package_module_name, package_filename = \ Importing.findModule( source_ref = self.getSourceReference(), module_name = self.package_name, parent_package = None, level = 1, warn = Utils.python_version < 330 ) # TODO: Temporary, if we can't find the package for Python3.3 that # is semi-OK, maybe. if Utils.python_version >= 330 and not package_filename: return [] imported_module, is_added = Recursion.recurseTo( module_package = package_package, module_filename = package_filename, module_relpath = Utils.relpath(package_filename), module_kind = "py", reason = "Containing package of recursed module.", ) self.package = imported_module if is_added: result.append(imported_module) if self.package: from nuitka.ModuleRegistry import addUsedModule addUsedModule(self.package) # print "Recursed to package", self.package_name result.extend(self.package.attemptRecursion()) return result
def attemptRecursion(self): # Make sure the package is recursed to. from nuitka.tree import Recursion from nuitka import Importing # Return the list of newly added modules. result = [] if self.package_name is not None and self.package is None: package_package, _package_module_name, package_filename = \ Importing.findModule( source_ref = self.getSourceReference(), module_name = self.package_name, parent_package = None, level = 1, warn = Utils.python_version < 330 ) # TODO: Temporary, if we can't find the package for Python3.3 that # is semi-OK, maybe. if Utils.python_version >= 330 and not package_filename: return [] imported_module, is_added = Recursion.recurseTo( module_package=package_package, module_filename=package_filename, module_relpath=Utils.relpath(package_filename), module_kind="py", reason="Containing package of recursed module.", ) self.package = imported_module if is_added: result.append(imported_module) if self.package: from nuitka.ModuleRegistry import addUsedModule addUsedModule(self.package) # print "Recursed to package", self.package_name result.extend(self.package.attemptRecursion()) return result
def attemptRecursion(self, module): if not Options.shallMakeModule(): # Make sure the package is recursed to. module_package = module.getPackage() if module_package is not None: package_package, _package_module_name, package_filename = Importing.findModule( source_ref=module.getSourceReference(), module_name=module_package, parent_package=None, level=1) imported_module, added_flag = Recursion.recurseTo( module_package=package_package, module_filename=package_filename, module_relpath=Utils.relpath(package_filename)) if added_flag: self.signalChange("new_code", imported_module.getSourceReference(), "Recursed to module package.")
def _decide( module_filename, module_name, module_package ): # Many branches, which make decisions immediately, pylint: disable=R0911 no_case_modules = Options.getShallFollowInNoCase() if module_package is None: full_name = module_name else: full_name = module_package + "." + module_name for no_case_module in no_case_modules: if full_name == no_case_module: return False if full_name.startswith( no_case_module + "." ): return False any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return True if full_name.startswith( any_case_module + "." ): return True if Options.shallFollowNoImports(): return False if Importing.isStandardLibraryPath( module_filename ): return Options.shallFollowStandardLibrary() if Options.shallFollowAllImports(): return True # Means, I don't know. return None
def _decide(module_filename, module_name, module_package): # Many branches, which make decisions immediately, pylint: disable=R0911 no_case_modules = Options.getShallFollowInNoCase() if module_package is None: full_name = module_name else: full_name = module_package + "." + module_name for no_case_module in no_case_modules: if full_name == no_case_module: return False if full_name.startswith(no_case_module + "."): return False any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return True if full_name.startswith(any_case_module + "."): return True if Options.shallFollowNoImports(): return False if Importing.isStandardLibraryPath(module_filename): return Options.shallFollowStandardLibrary() if Options.shallFollowAllImports(): return True # Means, I don't know. return None
def _checkPluginPath( plugin_filename, module_package ): plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: module, added = recurseTo( module_filename = plugin_info[0], module_relpath = plugin_info[1], module_package = module_package ) if module: if not added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0] ) if module.isPythonPackage(): package_dir = Utils.dirname( module.getFilename() ) for sub_path, sub_filename in Utils.listDir( package_dir ): if sub_filename == "__init__.py": continue assert sub_path != plugin_filename, package_dir if Importing.isPackageDir( sub_path ) or sub_path.endswith( ".py" ): _checkPluginPath( sub_path, module.getFullName() ) else: warning( "Failed to include module from '%s'.", plugin_info[0] )
def decideRecursion(module_filename, module_name, module_package, module_kind ): # Many branches, which make decisions immediately, pylint: disable=R0911 if module_kind == "shlib": if Options.isStandaloneMode(): return True, "Shared library for inclusion." else: return False, "Shared library cannot be inspected." if module_package is None: full_name = module_name else: full_name = module_package + "." + module_name if isFrozenModule(full_name): return False, "Module is frozen." no_case_modules = Options.getShallFollowInNoCase() for no_case_module in no_case_modules: if full_name == no_case_module: return ( False, "Module listed explicitely to not recurse to." ) if full_name.startswith(no_case_module + "."): return ( False, "Module in package listed explicitely to not recurse to." ) any_case_modules = Options.getShallFollowModules() for any_case_module in any_case_modules: if full_name == any_case_module: return ( True, "Module listed explicitely to recurse to." ) if full_name.startswith( any_case_module + "." ): return ( True, "Module in package listed explicitely to recurse to." ) if Options.shallFollowNoImports(): return ( False, "Requested to not recurse at all." ) if Importing.isStandardLibraryPath(module_filename): return ( Options.shallFollowStandardLibrary(), "Requested to %srecurse to standard library." % ( "" if Options.shallFollowStandardLibrary() else "not " ) ) if Options.shallFollowAllImports(): return ( True, "Requested to recurse to all non-standard library modules." ) # Means, we were not given instructions how to handle things. return ( None, "Default behaviour, not recursing without request." )
def _checkPluginPath(plugin_filename, module_package): debug( "Checking detail plugin path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: module, is_added = recurseTo( module_filename = plugin_info[0], module_relpath = plugin_info[1], module_package = module_package, module_kind = "py", reason = "Lives in plugin directory." ) if module: if not is_added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0] ) if not isSameModulePath(module.getFilename(), plugin_info[0]): warning( "Duplicate ignored '%s'.", plugin_info[1] ) return debug( "Recursed to %s %s %s", module.getName(), module.getPackage(), module ) if module.isPythonPackage(): package_filename = module.getFilename() if Utils.isDir(package_filename): # Must be a namespace package. assert Utils.python_version >= 330 package_dir = package_filename # Only include it, if it contains actual modules, which will # recurse to this one and find it again. useful = False else: package_dir = Utils.dirname(package_filename) # Real packages will always be included. useful = True debug( "Package directory %s", package_dir ) for sub_path, sub_filename in Utils.listDir(package_dir): if sub_filename in ("__init__.py", "__pycache__"): continue assert sub_path != plugin_filename if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, module.getFullName()) else: # Modules should always be included. useful = True if useful: ModuleRegistry.addRootModule(module) else: warning( "Failed to include module from '%s'.", plugin_info[0] )
def isWhiteListedImport(node): module = node.getParentModule() return Importing.isStandardLibraryPath(module.getFilename())
def isWhiteListedImport(node): module = node.getParentModule() return Importing.isStandardLibraryPath(module.getFilename())
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912 assert package is None or type( package ) is str if is_main and Utils.isDir( filename ): source_filename = Utils.joinpath( filename, "__main__.py" ) if not Utils.isFile( source_filename ): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % ( Utils.basename( sys.argv[0] ), filename ) ) sys.exit( 2 ) filename = source_filename main_added = True else: main_added = False if Utils.isFile( filename ): source_filename = filename source_ref = SourceCodeReferences.fromFilename( filename = filename, future_spec = FutureSpec() ) if is_main: module_name = "__main__" else: module_name = Utils.basename( filename ) if module_name.endswith( ".py" ): module_name = module_name[:-3] if is_shlib: module_name = module_name.split(".")[0] if "." in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % ( module_name ) ) sys.exit( 2 ) if is_shlib: result = PythonShlibModule( name = module_name, source_ref = source_ref, package_name = package, ) elif is_main: result = PythonMainModule( source_ref = source_ref, main_added = main_added ) else: result = PythonModule( name = module_name, package_name = package, source_ref = source_ref ) elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile( source_filename ): assert Utils.python_version >= 330, source_filename source_ref, result = createNamespacePackage( package_name = package_name, module_relpath = filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename = Utils.abspath( source_filename ), future_spec = FutureSpec() ) result = PythonPackage( name = package_name, package_name = package, source_ref = source_ref ) else: sys.stderr.write( "%s: can't open file '%s'.\n" % ( Utils.basename( sys.argv[0] ), filename ) ) sys.exit( 2 ) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def _checkPluginPath(plugin_filename, module_package): debug( "Checking detail plugin path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: module, is_added = recurseTo( module_filename = plugin_info[0], module_relpath = plugin_info[1], module_package = module_package, module_kind = "py", reason = "Lives in plugin directory." ) if module: if not is_added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0] ) if not isSameModulePath(module.getFilename(), plugin_info[0]): warning( "Duplicate ignored '%s'.", plugin_info[1] ) return debug( "Recursed to %s %s %s", module.getName(), module.getPackage(), module ) if module.isPythonPackage(): package_filename = module.getFilename() if Utils.isDir(package_filename): # Must be a namespace package. assert Utils.python_version >= 330 package_dir = package_filename # Only include it, if it contains actual modules, which will # recurse to this one and find it again. useful = False else: package_dir = Utils.dirname(package_filename) # Real packages will always be included. useful = True debug( "Package directory %s", package_dir ) for sub_path, sub_filename in Utils.listDir(package_dir): if sub_filename in ("__init__.py", "__pycache__"): continue assert sub_path != plugin_filename if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, module.getFullName()) else: # Modules should always be included. useful = True if useful: ModuleRegistry.addRootModule(module) else: warning( "Failed to include module from '%s'.", plugin_info[0] )
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912 assert package is None or type(package) is str if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename( filename = filename, future_spec = FutureSpec() ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % ( module_name ) ) sys.exit(2) if is_shlib: result = PythonShlibModule( name = module_name, source_ref = source_ref, package_name = package, ) elif is_main: result = PythonMainModule( source_ref = source_ref, main_added = main_added ) else: result = PythonModule( name = module_name, package_name = package, source_ref = source_ref ) elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): assert Utils.python_version >= 330, source_filename source_ref, result = createNamespacePackage( package_name = package_name, module_relpath = filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename = Utils.abspath(source_filename), future_spec = FutureSpec() ) result = PythonPackage( name = package_name, package_name = package, source_ref = source_ref ) else: sys.stderr.write( "%s: can't open file '%s'.\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename