def removeDllDependencies(dll_filename, dll_filenames):
"""Create list of removable shared libraries by scanning through the plugins.
Args:
dll_filename: shared library filename
dll_filenames: list of shared library filenames
Returns:
list of removable files
"""
dll_filenames = tuple(sorted(dll_filenames))
to_remove = OrderedSet()
for plugin in getActivePlugins():
removed_dlls = tuple(
plugin.removeDllDependencies(dll_filename, dll_filenames))
if removed_dlls and Options.isShowInclusion():
plugin.info("Removing DLLs %s of %s by plugin decision." %
(dll_filename, removed_dlls))
for removed_dll in removed_dlls:
to_remove.add(removed_dll)
for removed in to_remove:
dll_filenames.discard(removed)
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase):
def __init__(self, module):
assert module.isCompiledPythonModule(), module
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(self,
owner=module,
name="module:" + module.getFullName(),
parent=None)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name, module_relpath):
assert type(module_name) is str, module_name
# TODO: Make users provide this through a method that has already
# done this.
module_relpath = relpath(module_relpath)
self.used_modules.add((module_name, module_relpath))
module = getImportedModuleByNameAndPath(module_name, module_relpath)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def _parseDependsExeOutput2(lines):
result = OrderedSet()
inside = False
first = False
for line in lines:
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
dll_filename = line[line.find("]") + 2 :].rstrip()
dll_filename = os.path.normcase(dll_filename)
# Skip DLLs that failed to load, apparently not needed anyway.
if "E" in line[: line.find("]")]:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[: line.find("]")]:
# One exception are PythonXY.DLL
if dll_filename.startswith("python") and dll_filename.endswith(".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64" if getArchitecture() == "x86_64" else "System32",
dll_filename,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
dll_filename = os.path.abspath(dll_filename)
dll_name = os.path.basename(dll_filename)
# Ignore this runtime DLL of Python2, will be coming via manifest.
if dll_name in ("msvcr90.dll",):
continue
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), (dll_filename, line)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
return result
class TraceCollectionModule(CollectionStartpointMixin,
TraceCollectionBase):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(
self,
owner = module,
name = "module",
parent = None
)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
assert type(module_name) is str, module_name
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase):
def __init__(self, module):
assert module.isCompiledPythonModule(), module
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(
self, owner=module, name="module:" + module.getFullName(), parent=None
)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name, module_relpath):
assert type(module_name) is str, module_name
# TODO: Make users provide this through a method that has already
# done this.
module_relpath = relpath(module_relpath)
self.used_modules.add((module_name, module_relpath))
module = getImportedModuleByNameAndPath(module_name, module_relpath)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def getPackageFiles(module, packages, folders_only):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest), if folders_only is False,
else tuples (_createEmptyDirNone, dest).
"""
# TODO: Maybe use isinstance(basestring) for this
if not hasattr(packages, "__getitem__"): # so should be a string type
packages = (packages, )
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(pkg_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", ))
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
if folders_only is False: # normal case: indeed copy the files
item_set.add((f, target))
else: # just create the empty folder structure
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def replaceRootModule(old, new):
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=global-statement
global root_modules
new_root_modules = OrderedSet()
for module in root_modules:
new_root_modules.add(module if module is not old else new)
root_modules = new_root_modules
def replaceRootModule(old, new):
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=global-statement
global root_modules
new_root_modules = OrderedSet()
for module in root_modules:
new_root_modules.add(module if module is not old else new)
root_modules = new_root_modules
def getSubDirectoryFiles(module, subdirs, folders_only):
"""Yield filenames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else tuples (_createEmptyDirNone, dest) are yielded.
"""
module_folder = module.getCompileTimeDirectory()
elements = module.getFullName().split(".")
filename_start = module_folder.find(elements[0])
file_list = []
item_set = OrderedSet()
if subdirs is None:
data_dirs = [module_folder]
elif isinstance(subdirs, basestring):
data_dirs = [os.path.join(module_folder, subdirs)]
else:
data_dirs = [os.path.join(module_folder, subdir) for subdir in subdirs]
# Gather the full file list, probably makes no sense to include bytecode files
file_list = sum(
(getFileList(data_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", )) for data_dir in data_dirs),
[],
)
if not file_list:
msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % (
module.getFullName(),
subdirs,
data_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for f in file_list:
target = f[filename_start:]
if folders_only is False:
item_set.add((f, target))
else:
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
class VariableWriteExtractor(VisitorNoopMixin):
"""Extract variables written to."""
def __init__(self):
self.written_to = OrderedSet()
def onEnterNode(self, node):
if node.isStatementAssignmentVariable() or node.isStatementDelVariable():
self.written_to.add(node.getVariable())
def getResult(self):
return self.written_to
def _get_subdir_files(module, subdirs, folders_only):
"""Yield filenames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else tuples (_createEmptyDirNone, dest) are yielded.
"""
module_folder = module.getCompileTimeDirectory()
elements = module.getFullName().split(".")
filename_start = module_folder.find(elements[0])
file_list = []
item_set = OrderedSet()
if subdirs is None:
file_list = getFileList(module_folder)
elif type(subdirs) is str:
data_dir = os.path.join(module_folder, subdirs)
file_list = getFileList(data_dir)
else:
for subdir in subdirs:
data_dir = os.path.join(module_folder, subdir)
file_list.extend(getFileList(data_dir))
if file_list == []:
msg = "No files or folders found for '%s' in subfolder(s) '%s'." % (
module.getFullName(),
str(subdirs),
)
warning(msg)
yield ()
for f in file_list:
if "__pycache__" in f or f.endswith(".pyc"): # probably makes no sense
continue
target = f[filename_start:]
if folders_only is False:
item_set.add((f, target))
else:
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def getModuleSpecificDllPaths(module_name):
"""Provide a list of directories, where DLLs should be searched for this package (or module).
Args:
module_name: name of a package or module, for which the DLL path addition applies.
"""
result = OrderedSet()
for plugin in getActivePlugins():
for dll_path in plugin.getModuleSpecificDllPaths(module_name):
result.add(dll_path)
return result
def getExtraLinkLibraries():
result = OrderedSet()
for plugin in getActivePlugins():
value = plugin.getExtraLinkLibraries()
if value is not None:
if isinstance(value, basestring):
result.add(value)
else:
for library_name in value:
result.add(library_name)
return result
class TraceCollectionPureFunction(TraceCollectionFunction):
"""Pure functions don't feed their parent."""
__slots__ = ("used_functions",)
def __init__(self, function_body):
TraceCollectionFunction.__init__(self, parent=None, function_body=function_body)
self.used_functions = OrderedSet()
def getUsedFunctions(self):
return self.used_functions
def onUsedFunction(self, function_body):
self.used_functions.add(function_body)
def _getLdLibraryPath(package_name, python_rpath, original_dir):
key = package_name, python_rpath, original_dir
if key not in _ld_library_cache:
ld_library_path = OrderedSet()
if python_rpath:
ld_library_path.add(python_rpath)
ld_library_path.update(_getPackageSpecificDLLDirectories(package_name))
if original_dir is not None:
ld_library_path.add(original_dir)
_ld_library_cache[key] = ld_library_path
return _ld_library_cache[key]
def getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def _getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
package_dir = locateModule(module_name=package_name,
parent_package=None,
level=0)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def _getPackageFiles(module, *packages):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
Yields:
Tuples of paths (source, dest)
"""
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(
pkg_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",)
)
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
item_set.add((f, target))
for f in item_set:
yield f
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is the caching mechanism and plugin handling for DLL imports.
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
result = OrderedSet()
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
general.info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
result = detectDLLsWithDependencyWalker(binary_filename, scan_dirs)
if update_cache:
putTextFileContents(filename=cache_filename, contents=result)
return result
class DetectUsedModules(VisitorNoopMixin):
def __init__(self):
self.used_modules = OrderedSet()
def onEnterNode(self, node):
try:
self._onEnterNode(node)
except Exception:
general.my_print(
"Problem with %r at %s"
% (node, node.getSourceReference().getAsString())
)
raise
def _onEnterNode(self, node):
if node.isExpressionBuiltinImport():
for (
used_module_name,
used_module_filename,
finding,
level,
) in node.getUsedModules():
self.used_modules.add(
(
used_module_name,
used_module_filename,
finding,
level,
node.source_ref,
)
)
elif (
node.isExpressionImportModuleHard()
or node.isExpressionImportModuleNameHard()
or node.isExpressionImportModuleFixed()
):
used_module_name, used_module_filename, finding = node.getUsedModule()
self.used_modules.add(
(used_module_name, used_module_filename, finding, 0, node.source_ref)
)
def getUsedModules(self):
return self.used_modules
def getModifiedPaths():
result = OrderedSet()
output = check_output(["git", "diff", "--name-only"])
for line in output.splitlines():
if str is not bytes:
line = line.decode("utf8")
result.add(line)
output = check_output(["git", "diff", "--cached", "--name-only"])
for line in output.splitlines():
if str is not bytes:
line = line.decode("utf8")
result.add(line)
return tuple(sorted(result))
def _findPythons(python_version):
result = OrderedSet()
if python_version == python_version_str:
result.add(
InstalledPython(python_exe=sys.executable,
python_version=python_version))
if isWin32Windows():
result.update(
InstalledPython(python_exe=python_exe,
python_version=python_version)
for python_exe in _getPythonInstallPathsWindows(python_version))
candidate = getExecutablePath("python" + python_version)
if candidate is not None:
result.add(
InstalledPython(python_exe=candidate,
python_version=python_version))
return result
def _loadUncompiledModuleFromCache(module_name, is_package, source_code,
source_ref):
result = makeUncompiledPythonModule(
module_name=module_name,
filename=source_ref.getFilename(),
bytecode=demoteSourceCodeToBytecode(
module_name=module_name,
source_code=source_code,
filename=source_ref.getFilename(),
),
user_provided=False,
technical=False,
is_package=is_package,
)
used_modules = OrderedSet()
for used_module_name, line_number in getCachedImportedModulesNames(
module_name=module_name, source_code=source_code):
_module_name, module_filename, finding = Importing.locateModule(
module_name=used_module_name,
parent_package=None,
level=0,
)
assert _module_name == used_module_name
used_modules.add((
used_module_name,
module_filename,
finding,
0,
source_ref.atLineNumber(line_number),
))
# assert not is_package, (module_name, used_modules, result, result.getCompileTimeFilename())
result.setUsedModules(used_modules)
return result
class ConstraintCollectionModule(CollectionStartpointMixin,
ConstraintCollectionBase,
VariableUsageTrackingMixin):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
ConstraintCollectionBase.__init__(self,
owner=module,
name="module",
parent=None)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def _locateStaticLinkLibrary(dll_name):
# singleton, pylint: disable=global-statement
#
global _ldconf_paths
if _ldconf_paths is None:
_ldconf_paths = OrderedSet()
for conf_filemame in getFileList("/etc/ld.so.conf.d", only_suffixes=".conf"):
for conf_line in getFileContentByLine(conf_filemame):
conf_line = conf_line.split("#", 1)[0]
conf_line = conf_line.strip()
if os.path.exists(conf_line):
_ldconf_paths.add(conf_line)
for ld_config_path in _ldconf_paths:
candidate = os.path.join(ld_config_path, "lib%s.a" % dll_name)
if os.path.exists(candidate):
return candidate
return None
def _detectBinaryRPathsMacOS(original_dir, binary_filename):
stdout = getOtoolListing(binary_filename)
lines = stdout.split(b"\n")
result = OrderedSet()
for i, line in enumerate(lines):
if line.endswith(b"cmd LC_RPATH"):
line = lines[i + 2]
if str is not bytes:
line = line.decode("utf8")
line = line.split("path ", 1)[1]
line = line.split(" (offset", 1)[0]
if line.startswith("@loader_path"):
line = os.path.join(original_dir, line[13:])
elif line.startswith("@executable_path"):
continue
result.add(line)
return result
class ConstraintCollectionModule(CollectionStartpointMixin,
ConstraintCollectionBase,
VariableUsageTrackingMixin):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
ConstraintCollectionBase.__init__(
self,
owner = module,
name = "module",
parent = None
)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def _parseOtoolListingOutput(output):
paths = OrderedSet()
for line in output.split(b"\n")[1:]:
if str is not bytes:
line = line.decode("utf8")
if not line:
continue
filename = line.split(" (", 1)[0].strip()
# Ignore dependency from system paths.
if not isPathBelow(
path=(
"/usr/lib/",
"/System/Library/Frameworks/",
"/System/Library/PrivateFrameworks/",
),
filename=filename,
):
paths.add(filename)
return paths
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in open(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith('('):
# No multiline imports please
assert imported.endswith(')')
imported = imported[1:-1]
assert imported
if imported == '*':
continue
for name in imported.split(','):
name = name.strip()
pyi_deps.add(parts[1] + '.' + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple(
(pyi_dep, None)
for pyi_dep in
pyi_deps
)
else:
self.used_modules = ()
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in getFileContentByLine(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith("("):
# No multiline imports please
assert imported.endswith(")")
imported = imported[1:-1]
assert imported
if imported == "*":
continue
for name in imported.split(","):
name = name.strip()
pyi_deps.add(parts[1] + "." + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple((pyi_dep, None) for pyi_dep in pyi_deps)
else:
self.used_modules = ()
def _getSubDirectoryFolders(module, subdirs):
"""Get dirnames in given subdirs of the module.
Notes:
All dirnames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - tuple
Returns:
makeIncludedEmptyDirectories of found dirnames.
"""
module_dir = module.getCompileTimeDirectory()
file_list = []
data_dirs = [os.path.join(module_dir, subdir) for subdir in subdirs]
# Gather the full file list, probably makes no sense to include bytecode files
file_list = sum(
(
getFileList(
data_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",)
)
for data_dir in data_dirs
),
[],
)
if not file_list:
msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % (
module.getFullName(),
subdirs,
data_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
is_package = module.isCompiledPythonPackage() or module.isUncompiledPythonPackage()
# We need to preserve the package target path in the dist folder.
if is_package:
package_part = module.getFullName().asPath()
else:
package = module.getFullName().getPackageName()
if package is None:
package_part = ""
else:
package_part = package.asPath()
item_set = OrderedSet()
for f in file_list:
target = os.path.join(package_part, os.path.relpath(f, module_dir))
dir_name = os.path.dirname(target)
item_set.add(dir_name)
return makeIncludedEmptyDirectories(
source_path=module_dir,
dest_paths=item_set,
reason="Subdirectories of module %s" % module.getFullName(),
)
def _findModuleInPath2(module_name, search_path):
""" This is out own module finding low level implementation.
Just the full module name and search path are given. This is then
tasked to raise "ImportError" or return a path if it finds it, or
None, if it is a built-in.
"""
# We have many branches here, because there are a lot of cases to try.
# pylint: disable=too-many-branches,too-many-locals
# We may have to decide between package and module, therefore build
# a list of candidates.
candidates = OrderedSet()
considered = set()
for entry in search_path:
# Don't try again, just with an entry of different casing or complete
# duplicate.
if os.path.normcase(entry) in considered:
continue
considered.add(os.path.normcase(entry))
package_directory = os.path.join(entry, module_name)
# First, check for a package with an init file, that would be the
# first choice.
if os.path.isdir(package_directory):
for suffix, _mode, mtype in imp.get_suffixes():
if mtype == imp.C_EXTENSION:
continue
package_file_name = "__init__" + suffix
file_path = os.path.join(package_directory, package_file_name)
if os.path.isfile(file_path):
candidates.add((entry, 1, package_directory))
break
else:
if python_version >= 300:
candidates.add((entry, 2, package_directory))
# Then, check out suffixes of all kinds.
for suffix, _mode, _type in imp.get_suffixes():
file_path = os.path.join(entry, module_name + suffix)
if os.path.isfile(file_path):
candidates.add((entry, 1, file_path))
break
if _debug_module_finding:
print("Candidates", candidates)
if candidates:
# Ignore lower priority matches from package directories without
# "__init__.py" file.
min_prio = min(candidate[1] for candidate in candidates)
candidates = [candidate for candidate in candidates if candidate[1] == min_prio]
# On case sensitive systems, no resolution needed.
if case_sensitive:
return candidates[0][2]
else:
for candidate in candidates:
for fullname, _filename in listDir(candidate[0]):
if fullname == candidate[2]:
return candidate[2]
# Only exact case matches matter, all candidates were ignored,
# lets just fall through to raising the import error.
# Nothing found.
raise ImportError
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name = "<module>" if self.isMainModule() else self.getName(),
code_kind = "Module",
arg_names = (),
kw_only_count = 0,
has_stardict = False,
has_starlist = False,
)
def getCodeObject(self):
return self.code_object
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(CompiledPythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021
graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter,
variable_trace.getVariable(),
variable_trace.getVersion(),
variable_trace.__class__.__name__
)
for function_body in self.active_functions:
constraint_collection = function_body.constraint_collection
for (_variable, _version), variable_trace in constraint_collection.getVariableTracesAll().items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style = "filled", color = "blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style = "filled", color = "yellow")
else:
graph.attr("node", style = "filled", color = "red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
class PythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Module
The module is the only possible root of a tree. When there are many
modules they form a forest.
"""
kind = "PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(PythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def getVariables(self):
return self.variables
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
assert result not in self.variables
self.variables.add(result)
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule()
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
def hasUnclearLocals(self):
for collection in self.getTraceCollections():
if collection.hasUnclearLocals():
return True
return False
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = (
"body",
"functions"
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, mode, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None, # delayed
"functions" : (),
},
)
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.trace_collection = None
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def getDetailsForDisplay(self):
result = self.getDetails()
result["code_flags"] = ','.join(self.source_ref.getFutureSpec().asFlags())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider.
assert provider is None
assert False
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021
graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter,
variable_trace.getVariable(),
variable_trace.getVersion(),
variable_trace.__class__.__name__
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
for (_variable, _version), variable_trace in trace_collection.getVariableTracesAll().items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style = "filled", color = "blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style = "filled", color = "yellow")
else:
graph.attr("node", style = "filled", color = "red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
getFunctions = ChildrenHavingMixin.childGetter("functions")
setFunctions = ChildrenHavingMixin.childSetter("functions")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.getFunctions()
assert function_body not in functions
functions += (function_body,)
self.setFunctions(functions)
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.getFunctions()
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.getFunctions():
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.getFunctions():
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection = self.trace_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.trace_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
"$ORIGIN", os.path.dirname(sys.executable))
ld_library_path = OrderedSet()
if _detected_python_rpath:
ld_library_path.add(_detected_python_rpath)
ld_library_path.update(getPackageSpecificDLLDirectories(package_name))
if original_dir is not None:
ld_library_path.add(original_dir)
# ld_library_path.update(getSubDirectories(original_dir, ignore_dirs=("__pycache__",)))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", *ld_library_path):
process = subprocess.Popen(
args=["ldd", dll_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
stderr = b"\n".join(
line for line in stderr.splitlines() if not line.startswith(
b"ldd: warning: you do not have execution permission for"))
inclusion_logger.debug("ldd output for %s is:\n%s" %
(dll_filename, stdout))
if stderr:
inclusion_logger.debug("ldd error for %s is:\n%s" %
(dll_filename, stderr))
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Do not include kernel / glibc specific libraries. This list has been
# assembled by looking what are the most common .so files provided by
# glibc packages from ArchLinux, Debian Stretch and CentOS.
#
# Online sources:
# - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html
# - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html
# - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html
# - https://packages.debian.org/stretch/amd64/libc6/filelist
#
# Note: This list may still be incomplete. Some additional libraries
# might be provided by glibc - it may vary between the package versions
# and between Linux distros. It might or might not be a problem in the
# future, but it should be enough for now.
if os.path.basename(filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
result.add(filename)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsPosix(
dll_filename=sub_dll_filename,
package_name=package_name,
original_dir=original_dir,
))
return sub_result
class CompiledPythonModule(
ChildrenHavingMixin,
ClosureGiverNodeMixin,
MarkNeedsAnnotationsMixin,
EntryPointMixin,
PythonModuleBase,
):
"""Compiled Python Module"""
# This one has a few indicators, pylint: disable=too-many-instance-attributes
kind = "COMPILED_PYTHON_MODULE"
__slots__ = (
"is_top",
"name",
"code_prefix",
"code_name",
"uids",
"temp_variables",
"temp_scopes",
"preserver_id",
"needs_annotations_dict",
"trace_collection",
"mode",
"variables",
"active_functions",
"visited_functions",
"cross_used_functions",
"used_modules",
"future_spec",
"source_code",
"module_dict_name",
"locals_scope",
)
named_children = ("body", "functions")
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, module_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(self,
module_name=module_name,
source_ref=source_ref)
ClosureGiverNodeMixin.__init__(self,
name=module_name.getBasename(),
code_prefix="module")
ChildrenHavingMixin.__init__(
self,
values={
"body": None,
"functions": ()
} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
# Functions that have been used.
self.active_functions = OrderedSet()
# Functions that should be visited again.
self.visited_functions = set()
self.cross_used_functions = OrderedSet()
self.used_modules = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
# The source code of the module if changed or not from disk.
self.source_code = None
self.module_dict_name = "globals_%s" % (self.getCodeName(), )
self.locals_scope = getLocalsDictHandle(self.module_dict_name,
"module_dict", self)
@staticmethod
def isCompiledPythonModule():
return True
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"module_name": self.module_name,
}
def getDetailsForDisplay(self):
result = self.getDetails()
if self.future_spec is not None:
result["code_flags"] = ",".join(self.future_spec.asFlags())
return result
def getCompilationMode(self):
return self.mode
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider, must not be used,
assert False
def getFutureSpec(self):
return self.future_spec
def setFutureSpec(self, future_spec):
self.future_spec = future_spec
def isTopModule(self):
return self.is_top
def asGraph(self, graph, desc):
graph = graph.add_subgraph(name="cluster_%s" % desc,
comment="Graph for %s" % self.getName())
# graph.body.append("style=filled")
# graph.body.append("color=lightgrey")
# graph.body.append("label=Iteration_%d" % desc)
def makeTraceNodeName(variable, version, variable_trace):
return "%s/ %s %s %s" % (
desc,
variable.getName(),
version,
variable_trace.__class__.__name__,
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
node_names = {}
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = makeTraceNodeName(variable, version,
variable_trace)
node_names[variable_trace] = node_name
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = node_names[variable_trace]
previous = variable_trace.getPrevious()
attrs = {"style": "filled"}
if variable_trace.getUsageCount():
attrs["color"] = "blue"
else:
attrs["color"] = "red"
graph.add_node(node_name, **attrs)
if type(previous) is tuple:
for prev_trace in previous:
graph.add_edge(node_names[prev_trace], node_name)
assert prev_trace is not variable_trace
elif previous is not None:
assert previous is not variable_trace
graph.add_edge(node_names[previous], node_name)
return graph
def getSourceCode(self):
if self.source_code is not None:
return self.source_code
else:
return readSourceCodeFromFilename(
module_name=self.getFullName(),
source_filename=self.getCompileTimeFilename(),
)
def setSourceCode(self, code):
self.source_code = code
def getParent(self):
# We have never have a parent
return None
def getParentVariableProvider(self):
# We have never have a provider
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getProvidedVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(module=self,
variable_name=variable_name)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
@staticmethod
def isEarlyClosure():
# Modules should immediately closure variables on use.
return True
def getEntryPoint(self):
return self
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.subnode_functions
assert function_body not in functions
functions += (function_body, )
self.setChild("functions", functions)
def startTraversal(self):
self.used_modules = OrderedSet()
self.active_functions = OrderedSet()
def restartTraversal(self):
self.visited_functions = set()
def addUsedModule(self, key):
self.used_modules.add(key)
def getUsedModules(self):
return self.used_modules
def addUsedFunction(self, function_body):
assert function_body in self.subnode_functions, function_body
assert (function_body.isExpressionFunctionBody()
or function_body.isExpressionClassBody()
or function_body.isExpressionGeneratorObjectBody()
or function_body.isExpressionCoroutineObjectBody()
or function_body.isExpressionAsyncgenObjectBody())
self.active_functions.add(function_body)
result = function_body not in self.visited_functions
self.visited_functions.add(function_body)
return result
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.subnode_functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.subnode_functions:
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
elif main_filename.endswith(".pyw"):
result = main_filename[:-4]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(")", "").replace("(", "")
def computeModule(self):
self.restartTraversal()
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.subnode_body
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection=self.trace_collection)
if result is not module_body:
self.setChild("body", result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.",
)
# Finalize locals scopes previously determined for removal in last pass.
self.trace_collection.updateVariablesFromCollection(
old_collection, self.source_ref)
# Indicate if this is pass 1 for the module as return value.
was_complete = not self.locals_scope.complete
def markAsComplete(body, trace_collection):
if (body.locals_scope is not None
and body.locals_scope.isMarkedForPropagation()):
body.locals_scope = None
if body.locals_scope is not None:
body.locals_scope.markAsComplete(trace_collection)
def markEntryPointAsComplete(body):
markAsComplete(body, body.trace_collection)
outline_bodies = body.trace_collection.getOutlineFunctions()
if outline_bodies is not None:
for outline_body in outline_bodies:
markAsComplete(outline_body, body.trace_collection)
body.optimizeUnusedTempVariables()
markEntryPointAsComplete(self)
for function_body in self.getUsedFunctions():
markEntryPointAsComplete(function_body)
function_body.optimizeUnusedClosureVariables()
function_body.optimizeVariableReleases()
return was_complete
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def isUnoptimized(self):
# Modules don't do this, pylint: disable=no-self-use
return False
def getLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getUserLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
@staticmethod
def getFunctionVariablesWithAutoReleases():
"""Return the list of function variables that should be released at exit."""
return ()
def getOutlineLocalVariables(self):
outlines = self.getTraceCollection().getOutlineFunctions()
if outlines is None:
return ()
result = []
for outline in outlines:
result.extend(outline.getUserLocalVariables())
return result
def hasClosureVariable(self, variable):
# Modules don't do this, pylint: disable=no-self-use,unused-argument
return False
def removeUserVariable(self, variable):
outlines = self.getTraceCollection().getOutlineFunctions()
for outline in outlines:
user_locals = outline.getUserLocalVariables()
if variable in user_locals:
outline.removeUserVariable(variable)
break
def getLocalsScope(self):
return self.locals_scope
class CompiledPythonModule(
ChildrenHavingMixin,
ClosureGiverNodeMixin,
MarkNeedsAnnotationsMixin,
EntryPointMixin,
PythonModuleBase,
):
""" Compiled Python Module
"""
# This one has a few indicators, pylint: disable=too-many-instance-attributes
kind = "COMPILED_PYTHON_MODULE"
named_children = ("body", "functions")
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, name, package_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(
self, name=name, package_name=package_name, source_ref=source_ref
)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self, values={"body": None, "functions": ()} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
self.module_dict_name = "globals_%s" % (self.getCodeName(),)
setLocalsDictType(self.module_dict_name, "module_dict")
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"package": self.package_name,
"name": self.name,
}
def getDetailsForDisplay(self):
result = self.getDetails()
if self.future_spec is not None:
result["code_flags"] = ",".join(self.future_spec.asFlags())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider, must not be used,
assert False
def getFutureSpec(self):
return self.future_spec
def isTopModule(self):
return self.is_top
def asGraph(self, graph, desc):
graph = graph.add_subgraph(
name="cluster_%s" % desc, comment="Graph for %s" % self.getName()
)
# graph.body.append("style=filled")
# graph.body.append("color=lightgrey")
# graph.body.append("label=Iteration_%d" % desc)
def makeTraceNodeName(variable, version, variable_trace):
return "%s/ %s %s %s" % (
desc,
variable.getName(),
version,
variable_trace.__class__.__name__,
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
node_names = {}
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = makeTraceNodeName(variable, version, variable_trace)
node_names[variable_trace] = node_name
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = node_names[variable_trace]
previous = variable_trace.getPrevious()
attrs = {"style": "filled"}
if variable_trace.hasDefiniteUsages():
attrs["color"] = "blue"
elif variable_trace.hasPotentialUsages():
attrs["color"] = "yellow"
else:
attrs["color"] = "red"
graph.add_node(node_name, **attrs)
if type(previous) is tuple:
for prev_trace in previous:
graph.add_edge(node_names[prev_trace], node_name)
assert prev_trace is not variable_trace
elif previous is not None:
assert previous is not variable_trace
graph.add_edge(node_names[previous], node_name)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
getFunctions = ChildrenHavingMixin.childGetter("functions")
setFunctions = ChildrenHavingMixin.childSetter("functions")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
# We have never have a parent
return None
def getParentVariableProvider(self):
# We have never have a provider
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(module=self, variable_name=variable_name)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
@staticmethod
def isEarlyClosure():
# Modules should immediately closure variables on use.
return True
def getEntryPoint(self):
return self
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.getFunctions()
assert function_body not in functions
functions += (function_body,)
self.setFunctions(functions)
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.getFunctions()
assert (
function_body.isExpressionFunctionBody()
or function_body.isExpressionClassBody()
or function_body.isExpressionGeneratorObjectBody()
or function_body.isExpressionCoroutineObjectBody()
or function_body.isExpressionAsyncgenObjectBody()
)
result = function_body not in self.active_functions
if result:
self.active_functions.add(function_body)
return result
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.getFunctions():
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.getFunctions():
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
elif main_filename.endswith(".pyw"):
result = main_filename[:-4]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(")", "").replace("(", "")
def computeModule(self):
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection=self.trace_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.",
)
self.trace_collection.updateVariablesFromCollection(old_collection)
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def isUnoptimized(self):
# Modules don't do this, pylint: disable=no-self-use
return False
def getLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getUserLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getOutlineLocalVariables(self):
outlines = self.getTraceCollection().getOutlineFunctions()
if outlines is None:
return ()
result = []
for outline in outlines:
result.extend(outline.getUserLocalVariables())
return tuple(result)
def hasClosureVariable(self, variable):
# Modules don't do this, pylint: disable=no-self-use,unused-argument
return False
def removeUserVariable(self, variable):
outlines = self.getTraceCollection().getOutlineFunctions()
for outline in outlines:
user_locals = outline.getUserLocalVariables()
if variable in user_locals:
outline.removeUserVariable(variable)
break
@staticmethod
def getFunctionLocalsScope():
""" Modules have no locals scope. """
return None
def getModuleDictScope(self):
return getLocalsDictHandle(self.module_dict_name)
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
# Complex stuff, pylint: disable=too-many-branches,too-many-statements
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
# Flag signalling multiline import handling
in_import = False
in_import_part = ""
for line in getFileContentByLine(pyi_filename):
line = line.strip()
if not in_import:
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
origin_name = parts[1]
if origin_name == "typing":
continue
if origin_name == ".":
origin_name = self.getFullName()
# TODO: Might want to add full relative import handling.
if origin_name != self.getFullName():
pyi_deps.add(origin_name)
imported = parts[3]
if imported.startswith("("):
# Handle multiline imports
if not imported.endswith(")"):
in_import = True
imported = imported[1:]
in_import_part = origin_name
assert in_import_part, (
"Multiline part in file %s cannot be empty"
% pyi_filename)
else:
in_import = False
imported = imported[1:-1]
assert imported
if imported == "*":
continue
for name in imported.split(","):
if name:
name = name.strip()
pyi_deps.add(origin_name + "." + name)
else: # In import
imported = line
if imported.endswith(")"):
imported = imported[0:-1]
in_import = False
for name in imported.split(","):
name = name.strip()
if name:
pyi_deps.add(in_import_part + "." + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
if "__future__" in pyi_deps:
pyi_deps.discard("__future__")
if self.getFullName() in pyi_deps:
pyi_deps.discard(self.getFullName())
if self.getFullName().getPackageName() in pyi_deps:
pyi_deps.discard(self.getFullName().getPackageName())
self.used_modules = tuple(
(pyi_dep, None) for pyi_dep in pyi_deps)
else:
self.used_modules = ()
