def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def __init__(self, name, package_name, mode, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None, # delayed
"functions" : (),
},
)
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.trace_collection = None
class TraceCollectionModule(CollectionStartpointMixin,
TraceCollectionBase):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(
self,
owner = module,
name = "module",
parent = None
)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
assert type(module_name) is str, module_name
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def __init__(self, name, package_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(
self, name=name, package_name=package_name, source_ref=source_ref
)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self, values={"body": None, "functions": ()} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
self.module_dict_name = "globals_%s" % (self.getCodeName(),)
setLocalsDictType(self.module_dict_name, "module_dict")
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase):
def __init__(self, module):
assert module.isCompiledPythonModule(), module
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(
self, owner=module, name="module:" + module.getFullName(), parent=None
)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name, module_relpath):
assert type(module_name) is str, module_name
# TODO: Make users provide this through a method that has already
# done this.
module_relpath = relpath(module_relpath)
self.used_modules.add((module_name, module_relpath))
module = getImportedModuleByNameAndPath(module_name, module_relpath)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def __init__(self, qt_plugins):
self.webengine_done = False
self.qt_plugins_dirs = None
self.qt_plugins = OrderedSet(x.strip().lower()
for x in qt_plugins.split(","))
# Allow to specify none.
if self.qt_plugins == set(["none"]):
self.qt_plugins = set()
def replaceRootModule(old, new):
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=global-statement
global root_modules
new_root_modules = OrderedSet()
for module in root_modules:
new_root_modules.add(module if module is not old else new)
root_modules = new_root_modules
def startTraversal():
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=W0603
global active_modules, done_modules
active_modules = OrderedSet(root_modules)
done_modules = OrderedSet()
for active_module in active_modules:
active_module.startTraversal()
def replaceRootModule(old, new):
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=global-statement
global root_modules
new_root_modules = OrderedSet()
for module in root_modules:
new_root_modules.add(module if module is not old else new)
root_modules = new_root_modules
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(
self,
owner = module,
name = "module",
parent = None
)
self.used_modules = OrderedSet()
def __init__(self, module):
assert module.isCompiledPythonModule(), module
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(self,
owner=module,
name="module:" + module.getFullName(),
parent=None)
self.used_modules = OrderedSet()
class VariableWriteExtractor(VisitorNoopMixin):
"""Extract variables written to."""
def __init__(self):
self.written_to = OrderedSet()
def onEnterNode(self, node):
if node.isStatementAssignmentVariable() or node.isStatementDelVariable():
self.written_to.add(node.getVariable())
def getResult(self):
return self.written_to
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in open(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith('('):
# No multiline imports please
assert imported.endswith(')')
imported = imported[1:-1]
assert imported
if imported == '*':
continue
for name in imported.split(','):
name = name.strip()
pyi_deps.add(parts[1] + '.' + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple(
(pyi_dep, None)
for pyi_dep in
pyi_deps
)
else:
self.used_modules = ()
def getModuleSpecificDllPaths(module_name):
"""Provide a list of directories, where DLLs should be searched for this package (or module).
Args:
module_name: name of a package or module, for which the DLL path addition applies.
"""
result = OrderedSet()
for plugin in getActivePlugins():
for dll_path in plugin.getModuleSpecificDllPaths(module_name):
result.add(dll_path)
return result
def getPluginsEnabled():
"""*tuple*, user enabled (standard) plugins (not including user plugins)
Note:
Do not use this outside of main binary, as other plugins, e.g.
hinted compilation will activate plugins themselves and this
will not be visible here.
"""
result = OrderedSet()
if options:
for plugin_enabled in options.plugins_enabled:
result.update(plugin_enabled.split(","))
return tuple(result)
class TraceCollectionPureFunction(TraceCollectionFunction):
"""Pure functions don't feed their parent."""
__slots__ = ("used_functions",)
def __init__(self, function_body):
TraceCollectionFunction.__init__(self, parent=None, function_body=function_body)
self.used_functions = OrderedSet()
def getUsedFunctions(self):
return self.used_functions
def onUsedFunction(self, function_body):
self.used_functions.add(function_body)
def onMissingTrust(operation, source_ref, *args):
key = (operation, ) + args
if key not in _missing_trust:
_missing_trust[key] = OrderedSet()
_missing_trust[key].add(source_ref)
def buildOrderedSet(*producers):
values = []
for producer in producers:
values.extend(producer)
return OrderedSet(values)
def getPluginsEnabled():
"""*tuple*, user enabled (standard) plugins (not including user plugins)
Note:
Do not use this outside of main binary, as plugins are allowed
to activate plugins themselves and that will not be visible here.
"""
result = OrderedSet()
if options:
for plugin_enabled in options.plugins_enabled:
result.update(
getPluginNameConsideringRenames(plugin_name)
for plugin_name in plugin_enabled.split(","))
return tuple(result)
def getPluginsDisabled():
"""*tuple*, user disabled (standard) plugins.
Note:
Do not use this outside of main binary, as other plugins, e.g.
hinted compilation will activate plugins themselves and this
will not be visible here.
"""
result = OrderedSet()
if options:
for plugin_disabled in options.plugins_disabled:
result.update(
getPluginNameConsideringRenames(plugin_name)
for plugin_name in plugin_disabled.split(","))
return tuple(result)
def _getPackageFiles(module, *packages):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
Yields:
Tuples of paths (source, dest)
"""
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(
pkg_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",)
)
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
item_set.add((f, target))
for f in item_set:
yield f
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is the caching mechanism and plugin handling for DLL imports.
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
result = OrderedSet()
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
general.info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
result = detectDLLsWithDependencyWalker(binary_filename, scan_dirs)
if update_cache:
putTextFileContents(filename=cache_filename, contents=result)
return result
class DetectUsedModules(VisitorNoopMixin):
def __init__(self):
self.used_modules = OrderedSet()
def onEnterNode(self, node):
try:
self._onEnterNode(node)
except Exception:
general.my_print(
"Problem with %r at %s"
% (node, node.getSourceReference().getAsString())
)
raise
def _onEnterNode(self, node):
if node.isExpressionBuiltinImport():
for (
used_module_name,
used_module_filename,
finding,
level,
) in node.getUsedModules():
self.used_modules.add(
(
used_module_name,
used_module_filename,
finding,
level,
node.source_ref,
)
)
elif (
node.isExpressionImportModuleHard()
or node.isExpressionImportModuleNameHard()
or node.isExpressionImportModuleFixed()
):
used_module_name, used_module_filename, finding = node.getUsedModule()
self.used_modules.add(
(used_module_name, used_module_filename, finding, 0, node.source_ref)
)
def getUsedModules(self):
return self.used_modules
def __init__(self, module):
assert module.isCompiledPythonModule(), module
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(
self, owner=module, name="module:" + module.getFullName(), parent=None
)
self.used_modules = OrderedSet()
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in getFileContentByLine(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith("("):
# No multiline imports please
assert imported.endswith(")")
imported = imported[1:-1]
assert imported
if imported == "*":
continue
for name in imported.split(","):
name = name.strip()
pyi_deps.add(parts[1] + "." + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple((pyi_dep, None) for pyi_dep in pyi_deps)
else:
self.used_modules = ()
def __init__(self, name, package_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(
self,
name = name,
package_name = package_name,
source_ref = source_ref
)
ClosureGiverNodeMixin.__init__(
self,
name = name,
code_prefix = "module"
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None, # delayed
"functions" : (),
},
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
self.module_dict_name = "globals_%s" % (
self.getCodeName(),
)
setLocalsDictType(self.module_dict_name, "module_dict")
def __init__(self, name, package_name, mode, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.mode = mode
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name = "<module>" if self.isMainModule() else self.getName(),
code_kind = "Module",
arg_names = (),
kw_only_count = 0,
has_stardict = False,
has_starlist = False,
)
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(self,
name=name,
code_prefix="module",
source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
ChildrenHavingMixin.__init__(
self,
values={
"body": None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def _loadUncompiledModuleFromCache(module_name, is_package, source_code,
source_ref):
result = makeUncompiledPythonModule(
module_name=module_name,
filename=source_ref.getFilename(),
bytecode=demoteSourceCodeToBytecode(
module_name=module_name,
source_code=source_code,
filename=source_ref.getFilename(),
),
user_provided=False,
technical=False,
is_package=is_package,
)
used_modules = OrderedSet()
for used_module_name, line_number in getCachedImportedModulesNames(
module_name=module_name, source_code=source_code):
_module_name, module_filename, finding = Importing.locateModule(
module_name=used_module_name,
parent_package=None,
level=0,
)
assert _module_name == used_module_name
used_modules.add((
used_module_name,
module_filename,
finding,
0,
source_ref.atLineNumber(line_number),
))
# assert not is_package, (module_name, used_modules, result, result.getCompileTimeFilename())
result.setUsedModules(used_modules)
return result
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(self,
name=name,
code_prefix="module",
source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
ChildrenHavingMixin.__init__(
self,
values={
"body": None # delayed
},
)
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name="<module>" if self.isMainModule() else self.getName(),
code_kind="Module",
arg_names=(),
kw_only_count=0,
has_stardict=False,
has_starlist=False,
)
class ConstraintCollectionModule(CollectionStartpointMixin,
ConstraintCollectionBase,
VariableUsageTrackingMixin):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
ConstraintCollectionBase.__init__(self,
owner=module,
name="module",
parent=None)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
class TraceCollectionModule(CollectionStartpointMixin, TraceCollectionBase):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(self,
owner=module,
name="module",
parent=None)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
assert type(module_name) is str, module_name
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def _getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
package_dir = locateModule(module_name=package_name,
parent_package=None,
level=0)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def _getLdLibraryPath(package_name, python_rpath, original_dir):
key = package_name, python_rpath, original_dir
if key not in _ld_library_cache:
ld_library_path = OrderedSet()
if python_rpath:
ld_library_path.add(python_rpath)
ld_library_path.update(_getPackageSpecificDLLDirectories(package_name))
if original_dir is not None:
ld_library_path.add(original_dir)
_ld_library_cache[key] = ld_library_path
return _ld_library_cache[key]
def _locateStaticLinkLibrary(dll_name):
# singleton, pylint: disable=global-statement
#
global _ldconf_paths
if _ldconf_paths is None:
_ldconf_paths = OrderedSet()
for conf_filemame in getFileList("/etc/ld.so.conf.d", only_suffixes=".conf"):
for conf_line in getFileContentByLine(conf_filemame):
conf_line = conf_line.split("#", 1)[0]
conf_line = conf_line.strip()
if os.path.exists(conf_line):
_ldconf_paths.add(conf_line)
for ld_config_path in _ldconf_paths:
candidate = os.path.join(ld_config_path, "lib%s.a" % dll_name)
if os.path.exists(candidate):
return candidate
return None
def getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
class ConstraintCollectionModule(CollectionStartpointMixin,
ConstraintCollectionBase,
VariableUsageTrackingMixin):
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
ConstraintCollectionBase.__init__(
self,
owner = module,
name = "module",
parent = None
)
self.used_modules = OrderedSet()
def onUsedModule(self, module_name):
self.used_modules.add(module_name)
if isImportedModuleByName(module_name):
module = getImportedModuleByName(module_name)
addUsedModule(module)
def getUsedModules(self):
return self.used_modules
def run(self):
wheel_logger.info("Specified packages: %s." %
self.distribution.packages,
style="blue")
wheel_logger.info("Specified modules: %s." %
self.distribution.py_modules,
style="blue")
self.compile_packages = self.distribution.packages or []
self.py_modules = self.distribution.py_modules or []
# Determine
self.script_module_names = OrderedSet()
if self.distribution.entry_points is not None:
for group, script_specs in self.distribution.entry_points.items():
for script_spec in script_specs:
try:
script_module_name = (script_spec.split(
"=", 1)[1].strip().split(":")[0])
except Exception as e: # Catch all the things, pylint: disable=broad-except
wheel_logger.info(
"Problem parsing '%s' script specification in '%s' due to %s"
% (script_spec, group, e))
self.script_module_names.add(
ModuleName(script_module_name))
if not self.compile_packages and not self.py_modules:
wheel_logger.sysexit(
"No modules or packages specified, aborting. Did you provide packages in 'setup.cfg' or 'setup.py'?"
)
# Python2 does not allow super on this old style class.
distutils.command.build.build.run(self)
self._build(os.path.abspath(self.build_lib))
def getExtraLinkLibraries(cls):
if cls.extra_link_libraries is None:
cls.extra_link_libraries = OrderedSet()
for plugin in getActivePlugins():
value = plugin.getExtraLinkLibraries()
if value is not None:
if isinstance(value, basestring):
cls.extra_link_libraries.add(value)
else:
for library_name in value:
cls.extra_link_libraries.add(library_name)
return cls.extra_link_libraries
def getPackageFiles(module, packages, folders_only):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest), if folders_only is False,
else tuples (_createEmptyDirNone, dest).
"""
# TODO: Maybe use isinstance(basestring) for this
if not hasattr(packages, "__getitem__"): # so should be a string type
packages = (packages, )
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(pkg_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", ))
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
if folders_only is False: # normal case: indeed copy the files
item_set.add((f, target))
else: # just create the empty folder structure
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
class PythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Module
The module is the only possible root of a tree. When there are many
modules they form a forest.
"""
kind = "PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(PythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def getVariables(self):
return self.variables
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
assert result not in self.variables
self.variables.add(result)
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule()
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection
def hasUnclearLocals(self):
for collection in self.getTraceCollections():
if collection.hasUnclearLocals():
return True
return False
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = (
"body",
"functions"
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, mode, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None, # delayed
"functions" : (),
},
)
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.trace_collection = None
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def getDetailsForDisplay(self):
result = self.getDetails()
result["code_flags"] = ','.join(self.source_ref.getFutureSpec().asFlags())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider.
assert provider is None
assert False
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021
graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter,
variable_trace.getVariable(),
variable_trace.getVersion(),
variable_trace.__class__.__name__
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
for (_variable, _version), variable_trace in trace_collection.getVariableTracesAll().items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style = "filled", color = "blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style = "filled", color = "yellow")
else:
graph.attr("node", style = "filled", color = "red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
getFunctions = ChildrenHavingMixin.childGetter("functions")
setFunctions = ChildrenHavingMixin.childSetter("functions")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.getFunctions()
assert function_body not in functions
functions += (function_body,)
self.setFunctions(functions)
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.getFunctions()
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.getFunctions():
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.getFunctions():
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection = self.trace_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.trace_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def _findModuleInPath2(module_name, search_path):
""" This is out own module finding low level implementation.
Just the full module name and search path are given. This is then
tasked to raise "ImportError" or return a path if it finds it, or
None, if it is a built-in.
"""
# We have many branches here, because there are a lot of cases to try.
# pylint: disable=too-many-branches,too-many-locals
# We may have to decide between package and module, therefore build
# a list of candidates.
candidates = OrderedSet()
considered = set()
for entry in search_path:
# Don't try again, just with an entry of different casing or complete
# duplicate.
if os.path.normcase(entry) in considered:
continue
considered.add(os.path.normcase(entry))
package_directory = os.path.join(entry, module_name)
# First, check for a package with an init file, that would be the
# first choice.
if os.path.isdir(package_directory):
for suffix, _mode, mtype in imp.get_suffixes():
if mtype == imp.C_EXTENSION:
continue
package_file_name = "__init__" + suffix
file_path = os.path.join(package_directory, package_file_name)
if os.path.isfile(file_path):
candidates.add((entry, 1, package_directory))
break
else:
if python_version >= 300:
candidates.add((entry, 2, package_directory))
# Then, check out suffixes of all kinds.
for suffix, _mode, _type in imp.get_suffixes():
file_path = os.path.join(entry, module_name + suffix)
if os.path.isfile(file_path):
candidates.add((entry, 1, file_path))
break
if _debug_module_finding:
print("Candidates", candidates)
if candidates:
# Ignore lower priority matches from package directories without
# "__init__.py" file.
min_prio = min(candidate[1] for candidate in candidates)
candidates = [candidate for candidate in candidates if candidate[1] == min_prio]
# On case sensitive systems, no resolution needed.
if case_sensitive:
return candidates[0][2]
else:
for candidate in candidates:
for fullname, _filename in listDir(candidate[0]):
if fullname == candidate[2]:
return candidate[2]
# Only exact case matches matter, all candidates were ignored,
# lets just fall through to raising the import error.
# Nothing found.
raise ImportError
class CompiledPythonModule(
ChildrenHavingMixin,
ClosureGiverNodeMixin,
MarkNeedsAnnotationsMixin,
EntryPointMixin,
PythonModuleBase,
):
""" Compiled Python Module
"""
# This one has a few indicators, pylint: disable=too-many-instance-attributes
kind = "COMPILED_PYTHON_MODULE"
named_children = ("body", "functions")
checkers = {"body": checkStatementsSequenceOrNone}
def __init__(self, name, package_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(
self, name=name, package_name=package_name, source_ref=source_ref
)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self, values={"body": None, "functions": ()} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
self.module_dict_name = "globals_%s" % (self.getCodeName(),)
setLocalsDictType(self.module_dict_name, "module_dict")
def getDetails(self):
return {
"filename": self.source_ref.getFilename(),
"package": self.package_name,
"name": self.name,
}
def getDetailsForDisplay(self):
result = self.getDetails()
if self.future_spec is not None:
result["code_flags"] = ",".join(self.future_spec.asFlags())
return result
@classmethod
def fromXML(cls, provider, source_ref, **args):
# Modules are not having any provider, must not be used,
assert False
def getFutureSpec(self):
return self.future_spec
def isTopModule(self):
return self.is_top
def asGraph(self, graph, desc):
graph = graph.add_subgraph(
name="cluster_%s" % desc, comment="Graph for %s" % self.getName()
)
# graph.body.append("style=filled")
# graph.body.append("color=lightgrey")
# graph.body.append("label=Iteration_%d" % desc)
def makeTraceNodeName(variable, version, variable_trace):
return "%s/ %s %s %s" % (
desc,
variable.getName(),
version,
variable_trace.__class__.__name__,
)
for function_body in self.active_functions:
trace_collection = function_body.trace_collection
node_names = {}
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = makeTraceNodeName(variable, version, variable_trace)
node_names[variable_trace] = node_name
for (
(variable, version),
variable_trace,
) in trace_collection.getVariableTracesAll().items():
node_name = node_names[variable_trace]
previous = variable_trace.getPrevious()
attrs = {"style": "filled"}
if variable_trace.hasDefiniteUsages():
attrs["color"] = "blue"
elif variable_trace.hasPotentialUsages():
attrs["color"] = "yellow"
else:
attrs["color"] = "red"
graph.add_node(node_name, **attrs)
if type(previous) is tuple:
for prev_trace in previous:
graph.add_edge(node_names[prev_trace], node_name)
assert prev_trace is not variable_trace
elif previous is not None:
assert previous is not variable_trace
graph.add_edge(node_names[previous], node_name)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
getFunctions = ChildrenHavingMixin.childGetter("functions")
setFunctions = ChildrenHavingMixin.childSetter("functions")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
# We have never have a parent
return None
def getParentVariableProvider(self):
# We have never have a provider
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(variable_name=variable_name)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(module=self, variable_name=variable_name)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
@staticmethod
def isEarlyClosure():
# Modules should immediately closure variables on use.
return True
def getEntryPoint(self):
return self
def getCodeName(self):
# For code name of modules, we need to translate to C identifiers,
# removing characters illegal for that.
return encodePythonIdentifierToC(self.getFullName())
def addFunction(self, function_body):
functions = self.getFunctions()
assert function_body not in functions
functions += (function_body,)
self.setFunctions(functions)
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.getFunctions()
assert (
function_body.isExpressionFunctionBody()
or function_body.isExpressionClassBody()
or function_body.isExpressionGeneratorObjectBody()
or function_body.isExpressionCoroutineObjectBody()
or function_body.isExpressionAsyncgenObjectBody()
)
result = function_body not in self.active_functions
if result:
self.active_functions.add(function_body)
return result
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.getFunctions():
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getFunctionFromCodeName(self, code_name):
for function in self.getFunctions():
if function.getCodeName() == code_name:
return function
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
elif main_filename.endswith(".pyw"):
result = main_filename[:-4]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(")", "").replace("(", "")
def computeModule(self):
old_collection = self.trace_collection
self.trace_collection = TraceCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
trace_collection=self.trace_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.trace_collection.signalChange(
source_ref=new_module.getSourceReference(),
tags="new_code",
message="Recursed to module package.",
)
self.trace_collection.updateVariablesFromCollection(old_collection)
def getTraceCollections(self):
yield self.trace_collection
for function in self.getUsedFunctions():
yield function.trace_collection
def isUnoptimized(self):
# Modules don't do this, pylint: disable=no-self-use
return False
def getLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getUserLocalVariables(self):
# Modules don't do this, pylint: disable=no-self-use
return ()
def getOutlineLocalVariables(self):
outlines = self.getTraceCollection().getOutlineFunctions()
if outlines is None:
return ()
result = []
for outline in outlines:
result.extend(outline.getUserLocalVariables())
return tuple(result)
def hasClosureVariable(self, variable):
# Modules don't do this, pylint: disable=no-self-use,unused-argument
return False
def removeUserVariable(self, variable):
outlines = self.getTraceCollection().getOutlineFunctions()
for outline in outlines:
user_locals = outline.getUserLocalVariables()
if variable in user_locals:
outline.removeUserVariable(variable)
break
@staticmethod
def getFunctionLocalsScope():
""" Modules have no locals scope. """
return None
def getModuleDictScope(self):
return getLocalsDictHandle(self.module_dict_name)
def startTraversal(self):
self.active_functions = OrderedSet()
class CompiledPythonModule(PythonModuleMixin, ChildrenHavingMixin,
ClosureGiverNodeBase):
""" Compiled Python Module
"""
kind = "COMPILED_PYTHON_MODULE"
named_children = (
"body",
)
checkers = {
"body": checkStatementsSequenceOrNone
}
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(
self,
name = name,
code_prefix = "module",
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
ChildrenHavingMixin.__init__(
self,
values = {
"body" : None # delayed
},
)
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name = "<module>" if self.isMainModule() else self.getName(),
code_kind = "Module",
arg_names = (),
kw_only_count = 0,
has_stardict = False,
has_starlist = False,
)
def getCodeObject(self):
return self.code_object
def getDetails(self):
return {
"filename" : self.source_ref.getFilename(),
"package" : self.package_name,
"name" : self.name
}
def asXml(self):
result = super(CompiledPythonModule, self).asXml()
for function_body in self.active_functions:
result.append(function_body.asXml())
return result
def asGraph(self, computation_counter):
from graphviz import Digraph # @UnresolvedImport pylint: disable=F0401,I0021
graph = Digraph("cluster_%d" % computation_counter, comment = "Graph for %s" % self.getName())
graph.body.append("style=filled")
graph.body.append("color=lightgrey")
graph.body.append("label=Iteration_%d" % computation_counter)
def makeTraceNodeName(variable_trace):
return "%d/ %s %s %s" % (
computation_counter,
variable_trace.getVariable(),
variable_trace.getVersion(),
variable_trace.__class__.__name__
)
for function_body in self.active_functions:
constraint_collection = function_body.constraint_collection
for (_variable, _version), variable_trace in constraint_collection.getVariableTracesAll().items():
node = makeTraceNodeName(variable_trace)
previous = variable_trace.getPrevious()
if variable_trace.hasDefiniteUsages():
graph.attr("node", style = "filled", color = "blue")
elif variable_trace.hasPotentialUsages():
graph.attr("node", style = "filled", color = "yellow")
else:
graph.attr("node", style = "filled", color = "red")
graph.node(node)
if type(previous) is tuple:
for previous in previous:
graph.edge(makeTraceNodeName(previous), node)
elif previous is not None:
graph.edge(makeTraceNodeName(previous), node)
return graph
getBody = ChildrenHavingMixin.childGetter("body")
setBody = ChildrenHavingMixin.childSetter("body")
@staticmethod
def isCompiledPythonModule():
return True
def getParent(self):
assert False
def getParentVariableProvider(self):
return None
def hasVariableName(self, variable_name):
return variable_name in self.variables or variable_name in self.temp_variables
def getVariables(self):
return self.variables.values()
def getFilename(self):
return self.source_ref.getFilename()
def getVariableForAssignment(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForReference(self, variable_name):
return self.getProvidedVariable(variable_name)
def getVariableForClosure(self, variable_name):
return self.getProvidedVariable(
variable_name = variable_name
)
def createProvidedVariable(self, variable_name):
assert variable_name not in self.variables
result = Variables.ModuleVariable(
module = self,
variable_name = variable_name
)
self.variables[variable_name] = result
return result
@staticmethod
def getContainingClassDictCreation():
return None
def isEarlyClosure(self):
# Modules should immediately closure variables on use.
# pylint: disable=R0201
return True
def getCodeName(self):
def r(match):
c = match.group()
if c == '.':
return '$'
else:
return "$$%d$" % ord(c)
return "".join(
re.sub("[^a-zA-Z0-9_]", r ,c)
for c in
self.getFullName()
)
def addFunction(self, function_body):
assert function_body not in self.functions
self.functions.add(function_body)
def getFunctions(self):
return self.functions
def startTraversal(self):
self.active_functions = OrderedSet()
def addUsedFunction(self, function_body):
assert function_body in self.functions
assert function_body.isExpressionFunctionBody() or \
function_body.isExpressionClassBody() or \
function_body.isExpressionGeneratorObjectBody() or \
function_body.isExpressionCoroutineObjectBody()
if function_body not in self.active_functions:
self.active_functions.add(function_body)
def getUsedFunctions(self):
return self.active_functions
def getUnusedFunctions(self):
for function in self.functions:
if function not in self.active_functions:
yield function
def addCrossUsedFunction(self, function_body):
if function_body not in self.cross_used_functions:
self.cross_used_functions.add(function_body)
def getCrossUsedFunctions(self):
return self.cross_used_functions
def getOutputFilename(self):
main_filename = self.getFilename()
if main_filename.endswith(".py"):
result = main_filename[:-3]
else:
result = main_filename
# There are some characters that somehow are passed to shell, by
# Scons or unknown, so lets avoid them for now.
return result.replace(')',"").replace('(',"")
# TODO: Can't really use locals for modules, this should probably be made
# sure to not be used.
@staticmethod
def getLocalsMode():
return "copy"
def computeModule(self):
old_collection = self.constraint_collection
self.constraint_collection = ConstraintCollectionModule(self)
module_body = self.getBody()
if module_body is not None:
result = module_body.computeStatementsSequence(
constraint_collection = self.constraint_collection
)
if result is not module_body:
self.setBody(result)
new_modules = self.attemptRecursion()
for new_module in new_modules:
self.constraint_collection.signalChange(
source_ref = new_module.getSourceReference(),
tags = "new_code",
message = "Recursed to module package."
)
self.constraint_collection.updateFromCollection(old_collection)
def getTraceCollections(self):
yield self.constraint_collection
for function in self.getUsedFunctions():
yield function.constraint_collection