def removeDllDependencies(dll_filename, dll_filenames): """Create list of removable shared libraries by scanning through the plugins. Args: dll_filename: shared library filename dll_filenames: list of shared library filenames Returns: list of removable files """ dll_filenames = tuple(sorted(dll_filenames)) to_remove = OrderedSet() for plugin in getActivePlugins(): removed_dlls = tuple( plugin.removeDllDependencies(dll_filename, dll_filenames)) if removed_dlls and Options.isShowInclusion(): plugin.info("Removing DLLs %s of %s by plugin decision." % (dll_filename, removed_dlls)) for removed_dll in removed_dlls: to_remove.add(removed_dll) for removed in to_remove: dll_filenames.discard(removed)
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase): def __init__(self, module): assert module.isCompiledPythonModule(), module CollectionStartpointMixin.__init__(self) TraceCollectionBase.__init__(self, owner=module, name="module:" + module.getFullName(), parent=None) self.used_modules = OrderedSet() def onUsedModule(self, module_name, module_relpath): assert type(module_name) is str, module_name # TODO: Make users provide this through a method that has already # done this. module_relpath = relpath(module_relpath) self.used_modules.add((module_name, module_relpath)) module = getImportedModuleByNameAndPath(module_name, module_relpath) addUsedModule(module) def getUsedModules(self): return self.used_modules
def _parseDependsExeOutput2(lines): result = OrderedSet() inside = False first = False for line in lines: if "| Module Dependency Tree |" in line: inside = True first = True continue if not inside: continue if "| Module List |" in line: break if "]" not in line: continue dll_filename = line[line.find("]") + 2 :].rstrip() dll_filename = os.path.normcase(dll_filename) # Skip DLLs that failed to load, apparently not needed anyway. if "E" in line[: line.find("]")]: continue # Skip missing DLLs, apparently not needed anyway. if "?" in line[: line.find("]")]: # One exception are PythonXY.DLL if dll_filename.startswith("python") and dll_filename.endswith(".dll"): dll_filename = os.path.join( os.environ["SYSTEMROOT"], "SysWOW64" if getArchitecture() == "x86_64" else "System32", dll_filename, ) dll_filename = os.path.normcase(dll_filename) else: continue dll_filename = os.path.abspath(dll_filename) dll_name = os.path.basename(dll_filename) # Ignore this runtime DLL of Python2, will be coming via manifest. if dll_name in ("msvcr90.dll",): continue # The executable itself is of course exempted. We cannot check its path # because depends.exe mistreats unicode paths. if first: first = False continue assert os.path.isfile(dll_filename), (dll_filename, line) result.add(os.path.normcase(os.path.abspath(dll_filename))) return result
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase): def __init__(self, module): CollectionStartpointMixin.__init__(self) TraceCollectionBase.__init__( self, owner = module, name = "module", parent = None ) self.used_modules = OrderedSet() def onUsedModule(self, module_name): assert type(module_name) is str, module_name self.used_modules.add(module_name) if isImportedModuleByName(module_name): module = getImportedModuleByName(module_name) addUsedModule(module) def getUsedModules(self): return self.used_modules
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase): def __init__(self, module): assert module.isCompiledPythonModule(), module CollectionStartpointMixin.__init__(self) TraceCollectionBase.__init__( self, owner=module, name="module:" + module.getFullName(), parent=None ) self.used_modules = OrderedSet() def onUsedModule(self, module_name, module_relpath): assert type(module_name) is str, module_name # TODO: Make users provide this through a method that has already # done this. module_relpath = relpath(module_relpath) self.used_modules.add((module_name, module_relpath)) module = getImportedModuleByNameAndPath(module_name, module_relpath) addUsedModule(module) def getUsedModules(self): return self.used_modules
def getPackageFiles(module, packages, folders_only): """Yield all (!) filenames in given package(s). Notes: This should be required in rare occasions only. The one example I know is 'dns' when used by package 'eventlet'. Eventlet imports dns modules only to replace them with 'green' (i.e. non-blocking) counterparts. Args: module: module object packages: package name(s) - str or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest), if folders_only is False, else tuples (_createEmptyDirNone, dest). """ # TODO: Maybe use isinstance(basestring) for this if not hasattr(packages, "__getitem__"): # so should be a string type packages = (packages, ) file_list = [] item_set = OrderedSet() file_dirs = [] for package in packages: pkg_base, pkg_dir = get_package_paths(package) # read package folders if pkg_dir: filename_start = len(pkg_base) # position of package name in dir # read out the filenames pkg_files = getFileList(pkg_dir, ignore_dirs=("__pycache__", ), ignore_suffixes=(".pyc", )) file_dirs.append(pkg_dir) for f in pkg_files: file_list.append((filename_start, f)) # append to file list if not file_list: # safeguard for unexpected cases msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % ( module.getFullName(), packages, file_dirs, ) NuitkaPluginDataFileCollector.warning(msg) for filename_start, f in file_list: # re-read the collected filenames target = f[filename_start:] # make part of name if folders_only is False: # normal case: indeed copy the files item_set.add((f, target)) else: # just create the empty folder structure item_set.add((_createEmptyDirNone, target)) for f in item_set: yield f
def replaceRootModule(old, new): # Using global here, as this is really a singleton, in the form of a module, # pylint: disable=global-statement global root_modules new_root_modules = OrderedSet() for module in root_modules: new_root_modules.add(module if module is not old else new) root_modules = new_root_modules
def replaceRootModule(old, new): # Using global here, as this is really a singleton, in the form of a module, # pylint: disable=global-statement global root_modules new_root_modules = OrderedSet() for module in root_modules: new_root_modules.add(module if module is not old else new) root_modules = new_root_modules
def getSubDirectoryFiles(module, subdirs, folders_only): """Yield filenames in given subdirs of the module. Notes: All filenames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - str or None or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest) are yielded if folders_only is False, else tuples (_createEmptyDirNone, dest) are yielded. """ module_folder = module.getCompileTimeDirectory() elements = module.getFullName().split(".") filename_start = module_folder.find(elements[0]) file_list = [] item_set = OrderedSet() if subdirs is None: data_dirs = [module_folder] elif isinstance(subdirs, basestring): data_dirs = [os.path.join(module_folder, subdirs)] else: data_dirs = [os.path.join(module_folder, subdir) for subdir in subdirs] # Gather the full file list, probably makes no sense to include bytecode files file_list = sum( (getFileList(data_dir, ignore_dirs=("__pycache__", ), ignore_suffixes=(".pyc", )) for data_dir in data_dirs), [], ) if not file_list: msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % ( module.getFullName(), subdirs, data_dirs, ) NuitkaPluginDataFileCollector.warning(msg) for f in file_list: target = f[filename_start:] if folders_only is False: item_set.add((f, target)) else: item_set.add((_createEmptyDirNone, target)) for f in item_set: yield f
class VariableWriteExtractor(VisitorNoopMixin): """Extract variables written to.""" def __init__(self): self.written_to = OrderedSet() def onEnterNode(self, node): if node.isStatementAssignmentVariable() or node.isStatementDelVariable(): self.written_to.add(node.getVariable()) def getResult(self): return self.written_to
def _get_subdir_files(module, subdirs, folders_only): """Yield filenames in given subdirs of the module. Notes: All filenames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - str or None or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest) are yielded if folders_only is False, else tuples (_createEmptyDirNone, dest) are yielded. """ module_folder = module.getCompileTimeDirectory() elements = module.getFullName().split(".") filename_start = module_folder.find(elements[0]) file_list = [] item_set = OrderedSet() if subdirs is None: file_list = getFileList(module_folder) elif type(subdirs) is str: data_dir = os.path.join(module_folder, subdirs) file_list = getFileList(data_dir) else: for subdir in subdirs: data_dir = os.path.join(module_folder, subdir) file_list.extend(getFileList(data_dir)) if file_list == []: msg = "No files or folders found for '%s' in subfolder(s) '%s'." % ( module.getFullName(), str(subdirs), ) warning(msg) yield () for f in file_list: if "__pycache__" in f or f.endswith(".pyc"): # probably makes no sense continue target = f[filename_start:] if folders_only is False: item_set.add((f, target)) else: item_set.add((_createEmptyDirNone, target)) for f in item_set: yield f
def getModuleSpecificDllPaths(module_name): """Provide a list of directories, where DLLs should be searched for this package (or module). Args: module_name: name of a package or module, for which the DLL path addition applies. """ result = OrderedSet() for plugin in getActivePlugins(): for dll_path in plugin.getModuleSpecificDllPaths(module_name): result.add(dll_path) return result
def getExtraLinkLibraries(): result = OrderedSet() for plugin in getActivePlugins(): value = plugin.getExtraLinkLibraries() if value is not None: if isinstance(value, basestring): result.add(value) else: for library_name in value: result.add(library_name) return result
class TraceCollectionPureFunction(TraceCollectionFunction): """Pure functions don't feed their parent.""" __slots__ = ("used_functions",) def __init__(self, function_body): TraceCollectionFunction.__init__(self, parent=None, function_body=function_body) self.used_functions = OrderedSet() def getUsedFunctions(self): return self.used_functions def onUsedFunction(self, function_body): self.used_functions.add(function_body)
def _getLdLibraryPath(package_name, python_rpath, original_dir): key = package_name, python_rpath, original_dir if key not in _ld_library_cache: ld_library_path = OrderedSet() if python_rpath: ld_library_path.add(python_rpath) ld_library_path.update(_getPackageSpecificDLLDirectories(package_name)) if original_dir is not None: ld_library_path.add(original_dir) _ld_library_cache[key] = ld_library_path return _ld_library_cache[key]
def getPackageSpecificDLLDirectories(package_name): scan_dirs = OrderedSet() if package_name is not None: from nuitka.importing.Importing import findModule package_dir = findModule(None, package_name, None, 0, False)[1] if os.path.isdir(package_dir): scan_dirs.add(package_dir) scan_dirs.update( getSubDirectories(package_dir, ignore_dirs=("__pycache__", ))) scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name)) return scan_dirs
def _getPackageSpecificDLLDirectories(package_name): scan_dirs = OrderedSet() if package_name is not None: package_dir = locateModule(module_name=package_name, parent_package=None, level=0)[1] if os.path.isdir(package_dir): scan_dirs.add(package_dir) scan_dirs.update( getSubDirectories(package_dir, ignore_dirs=("__pycache__", ))) scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name)) return scan_dirs
def _getPackageFiles(module, *packages): """Yield all (!) filenames in given package(s). Notes: This should be required in rare occasions only. The one example I know is 'dns' when used by package 'eventlet'. Eventlet imports dns modules only to replace them with 'green' (i.e. non-blocking) counterparts. Args: module: module object packages: package name(s) - str or tuple Yields: Tuples of paths (source, dest) """ file_list = [] item_set = OrderedSet() file_dirs = [] for package in packages: pkg_base, pkg_dir = get_package_paths(package) # read package folders if pkg_dir: filename_start = len(pkg_base) # position of package name in dir # read out the filenames pkg_files = getFileList( pkg_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",) ) file_dirs.append(pkg_dir) for f in pkg_files: file_list.append((filename_start, f)) # append to file list if not file_list: # safeguard for unexpected cases msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % ( module.getFullName(), packages, file_dirs, ) NuitkaPluginDataFileCollector.warning(msg) for filename_start, f in file_list: # re-read the collected filenames target = f[filename_start:] # make part of name item_set.add((f, target)) for f in item_set: yield f
def detectBinaryPathDLLsWindowsDependencyWalker( is_main_executable, source_dir, original_dir, binary_filename, package_name, use_cache, update_cache, ): # This is the caching mechanism and plugin handling for DLL imports. if use_cache or update_cache: cache_filename = _getCacheFilename( dependency_tool="depends.exe", is_main_executable=is_main_executable, source_dir=source_dir, original_dir=original_dir, binary_filename=binary_filename, ) if use_cache: with withFileLock(): if not os.path.exists(cache_filename): use_cache = False if use_cache: result = OrderedSet() for line in getFileContentByLine(cache_filename): line = line.strip() result.add(line) return result if Options.isShowProgress(): general.info("Analysing dependencies of '%s'." % binary_filename) scan_dirs = getScanDirectories(package_name, original_dir) result = detectDLLsWithDependencyWalker(binary_filename, scan_dirs) if update_cache: putTextFileContents(filename=cache_filename, contents=result) return result
class DetectUsedModules(VisitorNoopMixin): def __init__(self): self.used_modules = OrderedSet() def onEnterNode(self, node): try: self._onEnterNode(node) except Exception: general.my_print( "Problem with %r at %s" % (node, node.getSourceReference().getAsString()) ) raise def _onEnterNode(self, node): if node.isExpressionBuiltinImport(): for ( used_module_name, used_module_filename, finding, level, ) in node.getUsedModules(): self.used_modules.add( ( used_module_name, used_module_filename, finding, level, node.source_ref, ) ) elif ( node.isExpressionImportModuleHard() or node.isExpressionImportModuleNameHard() or node.isExpressionImportModuleFixed() ): used_module_name, used_module_filename, finding = node.getUsedModule() self.used_modules.add( (used_module_name, used_module_filename, finding, 0, node.source_ref) ) def getUsedModules(self): return self.used_modules
def getModifiedPaths(): result = OrderedSet() output = check_output(["git", "diff", "--name-only"]) for line in output.splitlines(): if str is not bytes: line = line.decode("utf8") result.add(line) output = check_output(["git", "diff", "--cached", "--name-only"]) for line in output.splitlines(): if str is not bytes: line = line.decode("utf8") result.add(line) return tuple(sorted(result))
def _findPythons(python_version): result = OrderedSet() if python_version == python_version_str: result.add( InstalledPython(python_exe=sys.executable, python_version=python_version)) if isWin32Windows(): result.update( InstalledPython(python_exe=python_exe, python_version=python_version) for python_exe in _getPythonInstallPathsWindows(python_version)) candidate = getExecutablePath("python" + python_version) if candidate is not None: result.add( InstalledPython(python_exe=candidate, python_version=python_version)) return result
def _loadUncompiledModuleFromCache(module_name, is_package, source_code, source_ref): result = makeUncompiledPythonModule( module_name=module_name, filename=source_ref.getFilename(), bytecode=demoteSourceCodeToBytecode( module_name=module_name, source_code=source_code, filename=source_ref.getFilename(), ), user_provided=False, technical=False, is_package=is_package, ) used_modules = OrderedSet() for used_module_name, line_number in getCachedImportedModulesNames( module_name=module_name, source_code=source_code): _module_name, module_filename, finding = Importing.locateModule( module_name=used_module_name, parent_package=None, level=0, ) assert _module_name == used_module_name used_modules.add(( used_module_name, module_filename, finding, 0, source_ref.atLineNumber(line_number), )) # assert not is_package, (module_name, used_modules, result, result.getCompileTimeFilename()) result.setUsedModules(used_modules) return result
class ConstraintCollectionModule(CollectionStartpointMixin, ConstraintCollectionBase, VariableUsageTrackingMixin): def __init__(self, module): CollectionStartpointMixin.__init__(self) ConstraintCollectionBase.__init__(self, owner=module, name="module", parent=None) self.used_modules = OrderedSet() def onUsedModule(self, module_name): self.used_modules.add(module_name) if isImportedModuleByName(module_name): module = getImportedModuleByName(module_name) addUsedModule(module) def getUsedModules(self): return self.used_modules
def _locateStaticLinkLibrary(dll_name): # singleton, pylint: disable=global-statement # global _ldconf_paths if _ldconf_paths is None: _ldconf_paths = OrderedSet() for conf_filemame in getFileList("/etc/ld.so.conf.d", only_suffixes=".conf"): for conf_line in getFileContentByLine(conf_filemame): conf_line = conf_line.split("#", 1)[0] conf_line = conf_line.strip() if os.path.exists(conf_line): _ldconf_paths.add(conf_line) for ld_config_path in _ldconf_paths: candidate = os.path.join(ld_config_path, "lib%s.a" % dll_name) if os.path.exists(candidate): return candidate return None
def _detectBinaryRPathsMacOS(original_dir, binary_filename): stdout = getOtoolListing(binary_filename) lines = stdout.split(b"\n") result = OrderedSet() for i, line in enumerate(lines): if line.endswith(b"cmd LC_RPATH"): line = lines[i + 2] if str is not bytes: line = line.decode("utf8") line = line.split("path ", 1)[1] line = line.split(" (offset", 1)[0] if line.startswith("@loader_path"): line = os.path.join(original_dir, line[13:]) elif line.startswith("@executable_path"): continue result.add(line) return result
class ConstraintCollectionModule(CollectionStartpointMixin, ConstraintCollectionBase, VariableUsageTrackingMixin): def __init__(self, module): CollectionStartpointMixin.__init__(self) ConstraintCollectionBase.__init__( self, owner = module, name = "module", parent = None ) self.used_modules = OrderedSet() def onUsedModule(self, module_name): self.used_modules.add(module_name) if isImportedModuleByName(module_name): module = getImportedModuleByName(module_name) addUsedModule(module) def getUsedModules(self): return self.used_modules
def _parseOtoolListingOutput(output): paths = OrderedSet() for line in output.split(b"\n")[1:]: if str is not bytes: line = line.decode("utf8") if not line: continue filename = line.split(" (", 1)[0].strip() # Ignore dependency from system paths. if not isPathBelow( path=( "/usr/lib/", "/System/Library/Frameworks/", "/System/Library/PrivateFrameworks/", ), filename=filename, ): paths.add(filename) return paths
def _readPyPIFile(self): """ Read the .pyi file if present and scan for dependencies. """ if self.used_modules is None: pyi_filename = self.getPyIFilename() if os.path.exists(pyi_filename): pyi_deps = OrderedSet() for line in open(pyi_filename): line = line.strip() if line.startswith("import "): imported = line[7:] pyi_deps.add(imported) elif line.startswith("from "): parts = line.split(None, 3) assert parts[0] == "from" assert parts[2] == "import" if parts[1] == "typing": continue pyi_deps.add(parts[1]) imported = parts[3] if imported.startswith('('): # No multiline imports please assert imported.endswith(')') imported = imported[1:-1] assert imported if imported == '*': continue for name in imported.split(','): name = name.strip() pyi_deps.add(parts[1] + '.' + name) if "typing" in pyi_deps: pyi_deps.discard("typing") self.used_modules = tuple( (pyi_dep, None) for pyi_dep in pyi_deps ) else: self.used_modules = ()
def _readPyPIFile(self): """ Read the .pyi file if present and scan for dependencies. """ if self.used_modules is None: pyi_filename = self.getPyIFilename() if os.path.exists(pyi_filename): pyi_deps = OrderedSet() for line in getFileContentByLine(pyi_filename): line = line.strip() if line.startswith("import "): imported = line[7:] pyi_deps.add(imported) elif line.startswith("from "): parts = line.split(None, 3) assert parts[0] == "from" assert parts[2] == "import" if parts[1] == "typing": continue pyi_deps.add(parts[1]) imported = parts[3] if imported.startswith("("): # No multiline imports please assert imported.endswith(")") imported = imported[1:-1] assert imported if imported == "*": continue for name in imported.split(","): name = name.strip() pyi_deps.add(parts[1] + "." + name) if "typing" in pyi_deps: pyi_deps.discard("typing") self.used_modules = tuple((pyi_dep, None) for pyi_dep in pyi_deps) else: self.used_modules = ()
def _getSubDirectoryFolders(module, subdirs): """Get dirnames in given subdirs of the module. Notes: All dirnames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - tuple Returns: makeIncludedEmptyDirectories of found dirnames. """ module_dir = module.getCompileTimeDirectory() file_list = [] data_dirs = [os.path.join(module_dir, subdir) for subdir in subdirs] # Gather the full file list, probably makes no sense to include bytecode files file_list = sum( ( getFileList( data_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",) ) for data_dir in data_dirs ), [], ) if not file_list: msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % ( module.getFullName(), subdirs, data_dirs, ) NuitkaPluginDataFileCollector.warning(msg) is_package = module.isCompiledPythonPackage() or module.isUncompiledPythonPackage() # We need to preserve the package target path in the dist folder. if is_package: package_part = module.getFullName().asPath() else: package = module.getFullName().getPackageName() if package is None: package_part = "" else: package_part = package.asPath() item_set = OrderedSet() for f in file_list: target = os.path.join(package_part, os.path.relpath(f, module_dir)) dir_name = os.path.dirname(target) item_set.add(dir_name) return makeIncludedEmptyDirectories( source_path=module_dir, dest_paths=item_set, reason="Subdirectories of module %s" % module.getFullName(), )
def _findModuleInPath2(module_name, search_path): """ This is out own module finding low level implementation. Just the full module name and search path are given. This is then tasked to raise "ImportError" or return a path if it finds it, or None, if it is a built-in. """ # We have many branches here, because there are a lot of cases to try. # pylint: disable=too-many-branches,too-many-locals # We may have to decide between package and module, therefore build # a list of candidates. candidates = OrderedSet() considered = set() for entry in search_path: # Don't try again, just with an entry of different casing or complete # duplicate. if os.path.normcase(entry) in considered: continue considered.add(os.path.normcase(entry)) package_directory = os.path.join(entry, module_name) # First, check for a package with an init file, that would be the # first choice. if os.path.isdir(package_directory): for suffix, _mode, mtype in imp.get_suffixes(): if mtype == imp.C_EXTENSION: continue package_file_name = "__init__" + suffix file_path = os.path.join(package_directory, package_file_name) if os.path.isfile(file_path): candidates.add((entry, 1, package_directory)) break else: if python_version >= 300: candidates.add((entry, 2, package_directory)) # Then, check out suffixes of all kinds. for suffix, _mode, _type in imp.get_suffixes(): file_path = os.path.join(entry, module_name + suffix) if os.path.isfile(file_path): candidates.add((entry, 1, file_path)) break if _debug_module_finding: print("Candidates", candidates) if candidates: # Ignore lower priority matches from package directories without # "__init__.py" file. min_prio = min(candidate[1] for candidate in candidates) candidates = [candidate for candidate in candidates if candidate[1] == min_prio] # On case sensitive systems, no resolution needed. if case_sensitive: return candidates[0][2] else: for candidate in candidates: for fullname, _filename in listDir(candidate[0]): if fullname == candidate[2]: return candidate[2] # Only exact case matches matter, all candidates were ignored, # lets just fall through to raising the import error. # Nothing found. raise ImportError
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin, ClosureGiverNodeBase): """ Compiled Python Module """ kind = "COMPILED_PYTHON_MODULE" named_children = ( "body", ) checkers = { "body": checkStatementsSequenceOrNone } def __init__(self, name, package_name, source_ref): ClosureGiverNodeBase.__init__( self, name = name, code_prefix = "module", source_ref = source_ref ) PythonModuleMixin.__init__( self, name = name, package_name = package_name ) ChildrenHavingMixin.__init__( self, values = { "body" : None # delayed }, ) self.variables = {} # The list functions contained in that module. self.functions = OrderedSet() self.active_functions = OrderedSet() self.cross_used_functions = OrderedSet() # SSA trace based information about the module. self.constraint_collection = None self.code_object = CodeObjectSpec( code_name = "<module>" if self.isMainModule() else self.getName(), code_kind = "Module", arg_names = (), kw_only_count = 0, has_stardict = False, has_starlist = False, ) def getCodeObject(self): return self.code_object def getDetails(self): return { "filename" : self.source_ref.getFilename(), "package" : self.package_name, "name" : self.name } def asXml(self): result = super(CompiledPythonModule, self).asXml() for function_body in self.active_functions: result.append(function_body.asXml()) return result def asGraph(self, computation_counter): from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021 graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName()) graph.body.append("style=filled") graph.body.append("color=lightgrey") graph.body.append("label=Iteration_%d" % computation_counter) def makeTraceNodeName(variable_trace): return "%d/ %s %s %s" % ( computation_counter, variable_trace.getVariable(), variable_trace.getVersion(), variable_trace.__class__.__name__ ) for function_body in self.active_functions: constraint_collection = function_body.constraint_collection for (_variable, _version), variable_trace in constraint_collection.getVariableTracesAll().items(): node = makeTraceNodeName(variable_trace) previous = variable_trace.getPrevious() if variable_trace.hasDefiniteUsages(): graph.attr("node", style = "filled", color = "blue") elif variable_trace.hasPotentialUsages(): graph.attr("node", style = "filled", color = "yellow") else: graph.attr("node", style = "filled", color = "red") graph.node(node) if type(previous) is tuple: for previous in previous: graph.edge(makeTraceNodeName(previous), node) elif previous is not None: graph.edge(makeTraceNodeName(previous), node) return graph getBody = ChildrenHavingMixin.childGetter("body") setBody = ChildrenHavingMixin.childSetter("body") @staticmethod def isCompiledPythonModule(): return True def getParent(self): assert False def getParentVariableProvider(self): return None def hasVariableName(self, variable_name): return variable_name in self.variables or variable_name in self.temp_variables def getVariables(self): return self.variables.values() def getFilename(self): return self.source_ref.getFilename() def getVariableForAssignment(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForReference(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForClosure(self, variable_name): return self.getProvidedVariable( variable_name = variable_name ) def createProvidedVariable(self, variable_name): assert variable_name not in self.variables result = Variables.ModuleVariable( module = self, variable_name = variable_name ) self.variables[variable_name] = result return result @staticmethod def getContainingClassDictCreation(): return None def isEarlyClosure(self): # Modules should immediately closure variables on use. # pylint: disable=R0201 return True def getCodeName(self): def r(match): c = match.group() if c == '.': return '$' else: return "$$%d$" % ord(c) return "".join( re.sub("[^a-zA-Z0-9_]", r ,c) for c in self.getFullName() ) def addFunction(self, function_body): assert function_body not in self.functions self.functions.add(function_body) def getFunctions(self): return self.functions def startTraversal(self): self.active_functions = OrderedSet() def addUsedFunction(self, function_body): assert function_body in self.functions assert function_body.isExpressionFunctionBody() or \ function_body.isExpressionClassBody() or \ function_body.isExpressionGeneratorObjectBody() or \ function_body.isExpressionCoroutineObjectBody() if function_body not in self.active_functions: self.active_functions.add(function_body) def getUsedFunctions(self): return self.active_functions def getUnusedFunctions(self): for function in self.functions: if function not in self.active_functions: yield function def addCrossUsedFunction(self, function_body): if function_body not in self.cross_used_functions: self.cross_used_functions.add(function_body) def getCrossUsedFunctions(self): return self.cross_used_functions def getOutputFilename(self): main_filename = self.getFilename() if main_filename.endswith(".py"): result = main_filename[:-3] else: result = main_filename # There are some characters that somehow are passed to shell, by # Scons or unknown, so lets avoid them for now. return result.replace(')',"").replace('(',"") # TODO: Can't really use locals for modules, this should probably be made # sure to not be used. @staticmethod def getLocalsMode(): return "copy" def computeModule(self): old_collection = self.constraint_collection self.constraint_collection = ConstraintCollectionModule(self) module_body = self.getBody() if module_body is not None: result = module_body.computeStatementsSequence( constraint_collection = self.constraint_collection ) if result is not module_body: self.setBody(result) new_modules = self.attemptRecursion() for new_module in new_modules: self.constraint_collection.signalChange( source_ref = new_module.getSourceReference(), tags = "new_code", message = "Recursed to module package." ) self.constraint_collection.updateFromCollection(old_collection) def getTraceCollections(self): yield self.constraint_collection for function in self.getUsedFunctions(): yield function.constraint_collection
class PythonModule(PythonModuleMixin, ChildrenHavingMixin, ClosureGiverNodeBase): """ Module The module is the only possible root of a tree. When there are many modules they form a forest. """ kind = "PYTHON_MODULE" named_children = ( "body", ) checkers = { "body": checkStatementsSequenceOrNone } def __init__(self, name, package_name, source_ref): ClosureGiverNodeBase.__init__( self, name = name, code_prefix = "module", source_ref = source_ref ) PythonModuleMixin.__init__( self, name = name, package_name = package_name ) ChildrenHavingMixin.__init__( self, values = { "body" : None # delayed }, ) self.variables = set() # The list functions contained in that module. self.functions = OrderedSet() self.active_functions = OrderedSet() self.cross_used_functions = OrderedSet() # SSA trace based information about the module. self.constraint_collection = None def getDetails(self): return { "filename" : self.source_ref.getFilename(), "package" : self.package_name, "name" : self.name } def asXml(self): result = super(PythonModule, self).asXml() for function_body in self.active_functions: result.append(function_body.asXml()) return result getBody = ChildrenHavingMixin.childGetter("body") setBody = ChildrenHavingMixin.childSetter("body") @staticmethod def isPythonModule(): return True def getParent(self): assert False def getParentVariableProvider(self): return None def getVariables(self): return self.variables def getFilename(self): return self.source_ref.getFilename() def getVariableForAssignment(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForReference(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForClosure(self, variable_name): return self.getProvidedVariable( variable_name = variable_name ) def createProvidedVariable(self, variable_name): result = Variables.ModuleVariable( module = self, variable_name = variable_name ) assert result not in self.variables self.variables.add(result) return result @staticmethod def getContainingClassDictCreation(): return None def isEarlyClosure(self): # Modules should immediately closure variables on use. # pylint: disable=R0201 return True def getCodeName(self): def r(match): c = match.group() if c == '.': return '$' else: return "$$%d$" % ord(c) return "".join( re.sub("[^a-zA-Z0-9_]", r ,c) for c in self.getFullName() ) def addFunction(self, function_body): assert function_body not in self.functions self.functions.add(function_body) def getFunctions(self): return self.functions def startTraversal(self): self.active_functions = OrderedSet() def addUsedFunction(self, function_body): assert function_body in self.functions assert function_body.isExpressionFunctionBody() if function_body not in self.active_functions: self.active_functions.add(function_body) def getUsedFunctions(self): return self.active_functions def getUnusedFunctions(self): for function in self.functions: if function not in self.active_functions: yield function def addCrossUsedFunction(self, function_body): if function_body not in self.cross_used_functions: self.cross_used_functions.add(function_body) def getCrossUsedFunctions(self): return self.cross_used_functions def getOutputFilename(self): main_filename = self.getFilename() if main_filename.endswith(".py"): result = main_filename[:-3] else: result = main_filename # There are some characters that somehow are passed to shell, by # Scons or unknown, so lets avoid them for now. return result.replace(')',"").replace('(',"") # TODO: Can't really use locals for modules, this should probably be made # sure to not be used. @staticmethod def getLocalsMode(): return "copy" def computeModule(self): old_collection = self.constraint_collection self.constraint_collection = ConstraintCollectionModule() module_body = self.getBody() if module_body is not None: result = module_body.computeStatementsSequence( constraint_collection = self.constraint_collection ) if result is not module_body: self.setBody(result) new_modules = self.attemptRecursion() for new_module in new_modules: self.constraint_collection.signalChange( source_ref = new_module.getSourceReference(), tags = "new_code", message = "Recursed to module package." ) self.constraint_collection.updateFromCollection(old_collection) def getTraceCollections(self): yield self.constraint_collection for function in self.getUsedFunctions(): yield function.constraint_collection def hasUnclearLocals(self): for collection in self.getTraceCollections(): if collection.hasUnclearLocals(): return True return False
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin, ClosureGiverNodeBase): """ Compiled Python Module """ kind = "COMPILED_PYTHON_MODULE" named_children = ( "body", "functions" ) checkers = { "body": checkStatementsSequenceOrNone } def __init__(self, name, package_name, mode, source_ref): ClosureGiverNodeBase.__init__( self, name = name, code_prefix = "module", source_ref = source_ref ) PythonModuleMixin.__init__( self, name = name, package_name = package_name ) ChildrenHavingMixin.__init__( self, values = { "body" : None, # delayed "functions" : (), }, ) self.mode = mode self.variables = {} self.active_functions = OrderedSet() self.cross_used_functions = OrderedSet() # SSA trace based information about the module. self.trace_collection = None def getDetails(self): return { "filename" : self.source_ref.getFilename(), "package" : self.package_name, "name" : self.name } def getDetailsForDisplay(self): result = self.getDetails() result["code_flags"] = ','.join(self.source_ref.getFutureSpec().asFlags()) return result @classmethod def fromXML(cls, provider, source_ref, **args): # Modules are not having any provider. assert provider is None assert False def asGraph(self, computation_counter): from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021 graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName()) graph.body.append("style=filled") graph.body.append("color=lightgrey") graph.body.append("label=Iteration_%d" % computation_counter) def makeTraceNodeName(variable_trace): return "%d/ %s %s %s" % ( computation_counter, variable_trace.getVariable(), variable_trace.getVersion(), variable_trace.__class__.__name__ ) for function_body in self.active_functions: trace_collection = function_body.trace_collection for (_variable, _version), variable_trace in trace_collection.getVariableTracesAll().items(): node = makeTraceNodeName(variable_trace) previous = variable_trace.getPrevious() if variable_trace.hasDefiniteUsages(): graph.attr("node", style = "filled", color = "blue") elif variable_trace.hasPotentialUsages(): graph.attr("node", style = "filled", color = "yellow") else: graph.attr("node", style = "filled", color = "red") graph.node(node) if type(previous) is tuple: for previous in previous: graph.edge(makeTraceNodeName(previous), node) elif previous is not None: graph.edge(makeTraceNodeName(previous), node) return graph getBody = ChildrenHavingMixin.childGetter("body") setBody = ChildrenHavingMixin.childSetter("body") getFunctions = ChildrenHavingMixin.childGetter("functions") setFunctions = ChildrenHavingMixin.childSetter("functions") @staticmethod def isCompiledPythonModule(): return True def getParent(self): assert False def getParentVariableProvider(self): return None def hasVariableName(self, variable_name): return variable_name in self.variables or variable_name in self.temp_variables def getVariables(self): return self.variables.values() def getFilename(self): return self.source_ref.getFilename() def getVariableForAssignment(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForReference(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForClosure(self, variable_name): return self.getProvidedVariable( variable_name = variable_name ) def createProvidedVariable(self, variable_name): assert variable_name not in self.variables result = Variables.ModuleVariable( module = self, variable_name = variable_name ) self.variables[variable_name] = result return result @staticmethod def getContainingClassDictCreation(): return None def isEarlyClosure(self): # Modules should immediately closure variables on use. # pylint: disable=R0201 return True def getCodeName(self): # For code name of modules, we need to translate to C identifiers, # removing characters illegal for that. return encodePythonIdentifierToC(self.getFullName()) def addFunction(self, function_body): functions = self.getFunctions() assert function_body not in functions functions += (function_body,) self.setFunctions(functions) def startTraversal(self): self.active_functions = OrderedSet() def addUsedFunction(self, function_body): assert function_body in self.getFunctions() assert function_body.isExpressionFunctionBody() or \ function_body.isExpressionClassBody() or \ function_body.isExpressionGeneratorObjectBody() or \ function_body.isExpressionCoroutineObjectBody() if function_body not in self.active_functions: self.active_functions.add(function_body) def getUsedFunctions(self): return self.active_functions def getUnusedFunctions(self): for function in self.getFunctions(): if function not in self.active_functions: yield function def addCrossUsedFunction(self, function_body): if function_body not in self.cross_used_functions: self.cross_used_functions.add(function_body) def getCrossUsedFunctions(self): return self.cross_used_functions def getFunctionFromCodeName(self, code_name): for function in self.getFunctions(): if function.getCodeName() == code_name: return function def getOutputFilename(self): main_filename = self.getFilename() if main_filename.endswith(".py"): result = main_filename[:-3] else: result = main_filename # There are some characters that somehow are passed to shell, by # Scons or unknown, so lets avoid them for now. return result.replace(')',"").replace('(',"") # TODO: Can't really use locals for modules, this should probably be made # sure to not be used. @staticmethod def getLocalsMode(): return "copy" def computeModule(self): old_collection = self.trace_collection self.trace_collection = TraceCollectionModule(self) module_body = self.getBody() if module_body is not None: result = module_body.computeStatementsSequence( trace_collection = self.trace_collection ) if result is not module_body: self.setBody(result) new_modules = self.attemptRecursion() for new_module in new_modules: self.trace_collection.signalChange( source_ref = new_module.getSourceReference(), tags = "new_code", message = "Recursed to module package." ) self.trace_collection.updateFromCollection(old_collection) def getTraceCollections(self): yield self.trace_collection for function in self.getUsedFunctions(): yield function.trace_collection
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None and not Utils.isPosixWindows(): _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( "$ORIGIN", os.path.dirname(sys.executable)) ld_library_path = OrderedSet() if _detected_python_rpath: ld_library_path.add(_detected_python_rpath) ld_library_path.update(getPackageSpecificDLLDirectories(package_name)) if original_dir is not None: ld_library_path.add(original_dir) # ld_library_path.update(getSubDirectories(original_dir, ignore_dirs=("__pycache__",))) with withEnvironmentPathAdded("LD_LIBRARY_PATH", *ld_library_path): process = subprocess.Popen( args=["ldd", dll_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() stderr = b"\n".join( line for line in stderr.splitlines() if not line.startswith( b"ldd: warning: you do not have execution permission for")) inclusion_logger.debug("ldd output for %s is:\n%s" % (dll_filename, stdout)) if stderr: inclusion_logger.debug("ldd error for %s is:\n%s" % (dll_filename, stderr)) for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 0x300: filename = filename.decode("utf-8") # Sometimes might use stuff not found or supplied by ldd itself. if filename in ("not found", "ldd"): continue # Do not include kernel / glibc specific libraries. This list has been # assembled by looking what are the most common .so files provided by # glibc packages from ArchLinux, Debian Stretch and CentOS. # # Online sources: # - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html # - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html # - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html # - https://packages.debian.org/stretch/amd64/libc6/filelist # # Note: This list may still be incomplete. Some additional libraries # might be provided by glibc - it may vary between the package versions # and between Linux distros. It might or might not be a problem in the # future, but it should be enough for now. if os.path.basename(filename).startswith(( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", )): continue result.add(filename) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsPosix( dll_filename=sub_dll_filename, package_name=package_name, original_dir=original_dir, )) return sub_result
class CompiledPythonModule( ChildrenHavingMixin, ClosureGiverNodeMixin, MarkNeedsAnnotationsMixin, EntryPointMixin, PythonModuleBase, ): """Compiled Python Module""" # This one has a few indicators, pylint: disable=too-many-instance-attributes kind = "COMPILED_PYTHON_MODULE" __slots__ = ( "is_top", "name", "code_prefix", "code_name", "uids", "temp_variables", "temp_scopes", "preserver_id", "needs_annotations_dict", "trace_collection", "mode", "variables", "active_functions", "visited_functions", "cross_used_functions", "used_modules", "future_spec", "source_code", "module_dict_name", "locals_scope", ) named_children = ("body", "functions") checkers = {"body": checkStatementsSequenceOrNone} def __init__(self, module_name, is_top, mode, future_spec, source_ref): PythonModuleBase.__init__(self, module_name=module_name, source_ref=source_ref) ClosureGiverNodeMixin.__init__(self, name=module_name.getBasename(), code_prefix="module") ChildrenHavingMixin.__init__( self, values={ "body": None, "functions": () } # delayed ) MarkNeedsAnnotationsMixin.__init__(self) EntryPointMixin.__init__(self) self.is_top = is_top self.mode = mode self.variables = {} # Functions that have been used. self.active_functions = OrderedSet() # Functions that should be visited again. self.visited_functions = set() self.cross_used_functions = OrderedSet() self.used_modules = OrderedSet() # Often "None" until tree building finishes its part. self.future_spec = future_spec # The source code of the module if changed or not from disk. self.source_code = None self.module_dict_name = "globals_%s" % (self.getCodeName(), ) self.locals_scope = getLocalsDictHandle(self.module_dict_name, "module_dict", self) @staticmethod def isCompiledPythonModule(): return True def getDetails(self): return { "filename": self.source_ref.getFilename(), "module_name": self.module_name, } def getDetailsForDisplay(self): result = self.getDetails() if self.future_spec is not None: result["code_flags"] = ",".join(self.future_spec.asFlags()) return result def getCompilationMode(self): return self.mode @classmethod def fromXML(cls, provider, source_ref, **args): # Modules are not having any provider, must not be used, assert False def getFutureSpec(self): return self.future_spec def setFutureSpec(self, future_spec): self.future_spec = future_spec def isTopModule(self): return self.is_top def asGraph(self, graph, desc): graph = graph.add_subgraph(name="cluster_%s" % desc, comment="Graph for %s" % self.getName()) # graph.body.append("style=filled") # graph.body.append("color=lightgrey") # graph.body.append("label=Iteration_%d" % desc) def makeTraceNodeName(variable, version, variable_trace): return "%s/ %s %s %s" % ( desc, variable.getName(), version, variable_trace.__class__.__name__, ) for function_body in self.active_functions: trace_collection = function_body.trace_collection node_names = {} for ( (variable, version), variable_trace, ) in trace_collection.getVariableTracesAll().items(): node_name = makeTraceNodeName(variable, version, variable_trace) node_names[variable_trace] = node_name for ( (variable, version), variable_trace, ) in trace_collection.getVariableTracesAll().items(): node_name = node_names[variable_trace] previous = variable_trace.getPrevious() attrs = {"style": "filled"} if variable_trace.getUsageCount(): attrs["color"] = "blue" else: attrs["color"] = "red" graph.add_node(node_name, **attrs) if type(previous) is tuple: for prev_trace in previous: graph.add_edge(node_names[prev_trace], node_name) assert prev_trace is not variable_trace elif previous is not None: assert previous is not variable_trace graph.add_edge(node_names[previous], node_name) return graph def getSourceCode(self): if self.source_code is not None: return self.source_code else: return readSourceCodeFromFilename( module_name=self.getFullName(), source_filename=self.getCompileTimeFilename(), ) def setSourceCode(self, code): self.source_code = code def getParent(self): # We have never have a parent return None def getParentVariableProvider(self): # We have never have a provider return None def hasVariableName(self, variable_name): return variable_name in self.variables or variable_name in self.temp_variables def getProvidedVariables(self): return self.variables.values() def getFilename(self): return self.source_ref.getFilename() def getVariableForAssignment(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForReference(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForClosure(self, variable_name): return self.getProvidedVariable(variable_name=variable_name) def createProvidedVariable(self, variable_name): assert variable_name not in self.variables result = Variables.ModuleVariable(module=self, variable_name=variable_name) self.variables[variable_name] = result return result @staticmethod def getContainingClassDictCreation(): return None @staticmethod def isEarlyClosure(): # Modules should immediately closure variables on use. return True def getEntryPoint(self): return self def getCodeName(self): # For code name of modules, we need to translate to C identifiers, # removing characters illegal for that. return encodePythonIdentifierToC(self.getFullName()) def addFunction(self, function_body): functions = self.subnode_functions assert function_body not in functions functions += (function_body, ) self.setChild("functions", functions) def startTraversal(self): self.used_modules = OrderedSet() self.active_functions = OrderedSet() def restartTraversal(self): self.visited_functions = set() def addUsedModule(self, key): self.used_modules.add(key) def getUsedModules(self): return self.used_modules def addUsedFunction(self, function_body): assert function_body in self.subnode_functions, function_body assert (function_body.isExpressionFunctionBody() or function_body.isExpressionClassBody() or function_body.isExpressionGeneratorObjectBody() or function_body.isExpressionCoroutineObjectBody() or function_body.isExpressionAsyncgenObjectBody()) self.active_functions.add(function_body) result = function_body not in self.visited_functions self.visited_functions.add(function_body) return result def getUsedFunctions(self): return self.active_functions def getUnusedFunctions(self): for function in self.subnode_functions: if function not in self.active_functions: yield function def addCrossUsedFunction(self, function_body): if function_body not in self.cross_used_functions: self.cross_used_functions.add(function_body) def getCrossUsedFunctions(self): return self.cross_used_functions def getFunctionFromCodeName(self, code_name): for function in self.subnode_functions: if function.getCodeName() == code_name: return function def getOutputFilename(self): main_filename = self.getFilename() if main_filename.endswith(".py"): result = main_filename[:-3] elif main_filename.endswith(".pyw"): result = main_filename[:-4] else: result = main_filename # There are some characters that somehow are passed to shell, by # Scons or unknown, so lets avoid them for now. return result.replace(")", "").replace("(", "") def computeModule(self): self.restartTraversal() old_collection = self.trace_collection self.trace_collection = TraceCollectionModule(self) module_body = self.subnode_body if module_body is not None: result = module_body.computeStatementsSequence( trace_collection=self.trace_collection) if result is not module_body: self.setChild("body", result) new_modules = self.attemptRecursion() for new_module in new_modules: self.trace_collection.signalChange( source_ref=new_module.getSourceReference(), tags="new_code", message="Recursed to module package.", ) # Finalize locals scopes previously determined for removal in last pass. self.trace_collection.updateVariablesFromCollection( old_collection, self.source_ref) # Indicate if this is pass 1 for the module as return value. was_complete = not self.locals_scope.complete def markAsComplete(body, trace_collection): if (body.locals_scope is not None and body.locals_scope.isMarkedForPropagation()): body.locals_scope = None if body.locals_scope is not None: body.locals_scope.markAsComplete(trace_collection) def markEntryPointAsComplete(body): markAsComplete(body, body.trace_collection) outline_bodies = body.trace_collection.getOutlineFunctions() if outline_bodies is not None: for outline_body in outline_bodies: markAsComplete(outline_body, body.trace_collection) body.optimizeUnusedTempVariables() markEntryPointAsComplete(self) for function_body in self.getUsedFunctions(): markEntryPointAsComplete(function_body) function_body.optimizeUnusedClosureVariables() function_body.optimizeVariableReleases() return was_complete def getTraceCollections(self): yield self.trace_collection for function in self.getUsedFunctions(): yield function.trace_collection def isUnoptimized(self): # Modules don't do this, pylint: disable=no-self-use return False def getLocalVariables(self): # Modules don't do this, pylint: disable=no-self-use return () def getUserLocalVariables(self): # Modules don't do this, pylint: disable=no-self-use return () @staticmethod def getFunctionVariablesWithAutoReleases(): """Return the list of function variables that should be released at exit.""" return () def getOutlineLocalVariables(self): outlines = self.getTraceCollection().getOutlineFunctions() if outlines is None: return () result = [] for outline in outlines: result.extend(outline.getUserLocalVariables()) return result def hasClosureVariable(self, variable): # Modules don't do this, pylint: disable=no-self-use,unused-argument return False def removeUserVariable(self, variable): outlines = self.getTraceCollection().getOutlineFunctions() for outline in outlines: user_locals = outline.getUserLocalVariables() if variable in user_locals: outline.removeUserVariable(variable) break def getLocalsScope(self): return self.locals_scope
class CompiledPythonModule( ChildrenHavingMixin, ClosureGiverNodeMixin, MarkNeedsAnnotationsMixin, EntryPointMixin, PythonModuleBase, ): """ Compiled Python Module """ # This one has a few indicators, pylint: disable=too-many-instance-attributes kind = "COMPILED_PYTHON_MODULE" named_children = ("body", "functions") checkers = {"body": checkStatementsSequenceOrNone} def __init__(self, name, package_name, is_top, mode, future_spec, source_ref): PythonModuleBase.__init__( self, name=name, package_name=package_name, source_ref=source_ref ) ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module") ChildrenHavingMixin.__init__( self, values={"body": None, "functions": ()} # delayed ) MarkNeedsAnnotationsMixin.__init__(self) EntryPointMixin.__init__(self) self.is_top = is_top self.mode = mode self.variables = {} self.active_functions = OrderedSet() self.cross_used_functions = OrderedSet() # Often "None" until tree building finishes its part. self.future_spec = future_spec self.module_dict_name = "globals_%s" % (self.getCodeName(),) setLocalsDictType(self.module_dict_name, "module_dict") def getDetails(self): return { "filename": self.source_ref.getFilename(), "package": self.package_name, "name": self.name, } def getDetailsForDisplay(self): result = self.getDetails() if self.future_spec is not None: result["code_flags"] = ",".join(self.future_spec.asFlags()) return result @classmethod def fromXML(cls, provider, source_ref, **args): # Modules are not having any provider, must not be used, assert False def getFutureSpec(self): return self.future_spec def isTopModule(self): return self.is_top def asGraph(self, graph, desc): graph = graph.add_subgraph( name="cluster_%s" % desc, comment="Graph for %s" % self.getName() ) # graph.body.append("style=filled") # graph.body.append("color=lightgrey") # graph.body.append("label=Iteration_%d" % desc) def makeTraceNodeName(variable, version, variable_trace): return "%s/ %s %s %s" % ( desc, variable.getName(), version, variable_trace.__class__.__name__, ) for function_body in self.active_functions: trace_collection = function_body.trace_collection node_names = {} for ( (variable, version), variable_trace, ) in trace_collection.getVariableTracesAll().items(): node_name = makeTraceNodeName(variable, version, variable_trace) node_names[variable_trace] = node_name for ( (variable, version), variable_trace, ) in trace_collection.getVariableTracesAll().items(): node_name = node_names[variable_trace] previous = variable_trace.getPrevious() attrs = {"style": "filled"} if variable_trace.hasDefiniteUsages(): attrs["color"] = "blue" elif variable_trace.hasPotentialUsages(): attrs["color"] = "yellow" else: attrs["color"] = "red" graph.add_node(node_name, **attrs) if type(previous) is tuple: for prev_trace in previous: graph.add_edge(node_names[prev_trace], node_name) assert prev_trace is not variable_trace elif previous is not None: assert previous is not variable_trace graph.add_edge(node_names[previous], node_name) return graph getBody = ChildrenHavingMixin.childGetter("body") setBody = ChildrenHavingMixin.childSetter("body") getFunctions = ChildrenHavingMixin.childGetter("functions") setFunctions = ChildrenHavingMixin.childSetter("functions") @staticmethod def isCompiledPythonModule(): return True def getParent(self): # We have never have a parent return None def getParentVariableProvider(self): # We have never have a provider return None def hasVariableName(self, variable_name): return variable_name in self.variables or variable_name in self.temp_variables def getVariables(self): return self.variables.values() def getFilename(self): return self.source_ref.getFilename() def getVariableForAssignment(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForReference(self, variable_name): return self.getProvidedVariable(variable_name) def getVariableForClosure(self, variable_name): return self.getProvidedVariable(variable_name=variable_name) def createProvidedVariable(self, variable_name): assert variable_name not in self.variables result = Variables.ModuleVariable(module=self, variable_name=variable_name) self.variables[variable_name] = result return result @staticmethod def getContainingClassDictCreation(): return None @staticmethod def isEarlyClosure(): # Modules should immediately closure variables on use. return True def getEntryPoint(self): return self def getCodeName(self): # For code name of modules, we need to translate to C identifiers, # removing characters illegal for that. return encodePythonIdentifierToC(self.getFullName()) def addFunction(self, function_body): functions = self.getFunctions() assert function_body not in functions functions += (function_body,) self.setFunctions(functions) def startTraversal(self): self.active_functions = OrderedSet() def addUsedFunction(self, function_body): assert function_body in self.getFunctions() assert ( function_body.isExpressionFunctionBody() or function_body.isExpressionClassBody() or function_body.isExpressionGeneratorObjectBody() or function_body.isExpressionCoroutineObjectBody() or function_body.isExpressionAsyncgenObjectBody() ) result = function_body not in self.active_functions if result: self.active_functions.add(function_body) return result def getUsedFunctions(self): return self.active_functions def getUnusedFunctions(self): for function in self.getFunctions(): if function not in self.active_functions: yield function def addCrossUsedFunction(self, function_body): if function_body not in self.cross_used_functions: self.cross_used_functions.add(function_body) def getCrossUsedFunctions(self): return self.cross_used_functions def getFunctionFromCodeName(self, code_name): for function in self.getFunctions(): if function.getCodeName() == code_name: return function def getOutputFilename(self): main_filename = self.getFilename() if main_filename.endswith(".py"): result = main_filename[:-3] elif main_filename.endswith(".pyw"): result = main_filename[:-4] else: result = main_filename # There are some characters that somehow are passed to shell, by # Scons or unknown, so lets avoid them for now. return result.replace(")", "").replace("(", "") def computeModule(self): old_collection = self.trace_collection self.trace_collection = TraceCollectionModule(self) module_body = self.getBody() if module_body is not None: result = module_body.computeStatementsSequence( trace_collection=self.trace_collection ) if result is not module_body: self.setBody(result) new_modules = self.attemptRecursion() for new_module in new_modules: self.trace_collection.signalChange( source_ref=new_module.getSourceReference(), tags="new_code", message="Recursed to module package.", ) self.trace_collection.updateVariablesFromCollection(old_collection) def getTraceCollections(self): yield self.trace_collection for function in self.getUsedFunctions(): yield function.trace_collection def isUnoptimized(self): # Modules don't do this, pylint: disable=no-self-use return False def getLocalVariables(self): # Modules don't do this, pylint: disable=no-self-use return () def getUserLocalVariables(self): # Modules don't do this, pylint: disable=no-self-use return () def getOutlineLocalVariables(self): outlines = self.getTraceCollection().getOutlineFunctions() if outlines is None: return () result = [] for outline in outlines: result.extend(outline.getUserLocalVariables()) return tuple(result) def hasClosureVariable(self, variable): # Modules don't do this, pylint: disable=no-self-use,unused-argument return False def removeUserVariable(self, variable): outlines = self.getTraceCollection().getOutlineFunctions() for outline in outlines: user_locals = outline.getUserLocalVariables() if variable in user_locals: outline.removeUserVariable(variable) break @staticmethod def getFunctionLocalsScope(): """ Modules have no locals scope. """ return None def getModuleDictScope(self): return getLocalsDictHandle(self.module_dict_name)
def _readPyPIFile(self): """ Read the .pyi file if present and scan for dependencies. """ # Complex stuff, pylint: disable=too-many-branches,too-many-statements if self.used_modules is None: pyi_filename = self.getPyIFilename() if os.path.exists(pyi_filename): pyi_deps = OrderedSet() # Flag signalling multiline import handling in_import = False in_import_part = "" for line in getFileContentByLine(pyi_filename): line = line.strip() if not in_import: if line.startswith("import "): imported = line[7:] pyi_deps.add(imported) elif line.startswith("from "): parts = line.split(None, 3) assert parts[0] == "from" assert parts[2] == "import" origin_name = parts[1] if origin_name == "typing": continue if origin_name == ".": origin_name = self.getFullName() # TODO: Might want to add full relative import handling. if origin_name != self.getFullName(): pyi_deps.add(origin_name) imported = parts[3] if imported.startswith("("): # Handle multiline imports if not imported.endswith(")"): in_import = True imported = imported[1:] in_import_part = origin_name assert in_import_part, ( "Multiline part in file %s cannot be empty" % pyi_filename) else: in_import = False imported = imported[1:-1] assert imported if imported == "*": continue for name in imported.split(","): if name: name = name.strip() pyi_deps.add(origin_name + "." + name) else: # In import imported = line if imported.endswith(")"): imported = imported[0:-1] in_import = False for name in imported.split(","): name = name.strip() if name: pyi_deps.add(in_import_part + "." + name) if "typing" in pyi_deps: pyi_deps.discard("typing") if "__future__" in pyi_deps: pyi_deps.discard("__future__") if self.getFullName() in pyi_deps: pyi_deps.discard(self.getFullName()) if self.getFullName().getPackageName() in pyi_deps: pyi_deps.discard(self.getFullName().getPackageName()) self.used_modules = tuple( (pyi_dep, None) for pyi_dep in pyi_deps) else: self.used_modules = ()