def __init__(self, name, package_name, mode, source_ref):
ClosureGiverNodeBase.__init__(self,
name=name,
code_prefix="module",
source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
ChildrenHavingMixin.__init__(
self,
values={
"body": None, # delayed
"functions": (),
},
)
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.trace_collection = None
def __init__(self, name, package_name, is_top, mode, future_spec,
source_ref):
PythonModuleBase.__init__(self,
name=name,
package_name=package_name,
source_ref=source_ref)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self,
values={
"body": None,
"functions": ()
} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
self.module_dict_name = "globals_%s" % (self.getCodeName(), )
setLocalsDictType(self.module_dict_name, "module_dict")
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(self,
name=name,
code_prefix="module",
source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
ChildrenHavingMixin.__init__(
self,
values={
"body": None # delayed
},
)
self.variables = set()
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
def __init__(self, name, package_name, mode, future_spec, source_ref):
PythonModuleBase.__init__(self,
name=name,
package_name=package_name,
source_ref=source_ref)
ClosureGiverNodeMixin.__init__(self, name=name, code_prefix="module")
ChildrenHavingMixin.__init__(
self,
values={
"body": None, # delayed
"functions": (),
},
)
MarkNeedsAnnotationsMixin.__init__(self)
self.mode = mode
self.variables = {}
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.trace_collection = None
# Often None until tree building finishes its part.
self.future_spec = future_spec
def __init__(self, name, package_name, source_ref):
ClosureGiverNodeBase.__init__(self,
name=name,
code_prefix="module",
source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
ChildrenHavingMixin.__init__(
self,
values={
"body": None # delayed
},
)
self.variables = {}
# The list functions contained in that module.
self.functions = OrderedSet()
self.active_functions = OrderedSet()
self.cross_used_functions = OrderedSet()
# SSA trace based information about the module.
self.constraint_collection = None
self.code_object = CodeObjectSpec(
code_name="<module>" if self.isMainModule() else self.getName(),
code_kind="Module",
arg_names=(),
kw_only_count=0,
has_stardict=False,
has_starlist=False,
)
def startTraversal():
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=W0603
global active_modules, done_modules
active_modules = OrderedSet(root_modules)
done_modules = OrderedSet()
for active_module in active_modules:
active_module.startTraversal()
def __init__(self, module_name, is_top, mode, future_spec, source_ref):
PythonModuleBase.__init__(self,
module_name=module_name,
source_ref=source_ref)
ClosureGiverNodeMixin.__init__(self,
name=module_name.getBasename(),
code_prefix="module")
ChildrenHavingMixin.__init__(
self,
values={
"body": None,
"functions": ()
} # delayed
)
MarkNeedsAnnotationsMixin.__init__(self)
EntryPointMixin.__init__(self)
self.is_top = is_top
self.mode = mode
self.variables = {}
# Functions that have been used.
self.active_functions = OrderedSet()
# Functions that should be visited again.
self.visited_functions = set()
self.cross_used_functions = OrderedSet()
self.used_modules = OrderedSet()
# Often "None" until tree building finishes its part.
self.future_spec = future_spec
# The source code of the module if changed or not from disk.
self.source_code = None
self.module_dict_name = "globals_%s" % (self.getCodeName(), )
self.locals_scope = getLocalsDictHandle(self.module_dict_name,
"module_dict", self)
self.used_modules = OrderedSet()
def _parseDependsExeOutput2(lines):
result = OrderedSet()
inside = False
first = False
for line in lines:
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
dll_filename = line[line.find("]") + 2 :].rstrip()
dll_filename = os.path.normcase(dll_filename)
# Skip DLLs that failed to load, apparently not needed anyway.
if "E" in line[: line.find("]")]:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[: line.find("]")]:
# One exception are PythonXY.DLL
if dll_filename.startswith("python") and dll_filename.endswith(".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64" if getArchitecture() == "x86_64" else "System32",
dll_filename,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
dll_filename = os.path.abspath(dll_filename)
dll_name = os.path.basename(dll_filename)
# Ignore this runtime DLL of Python2, will be coming via manifest.
if dll_name in ("msvcr90.dll",):
continue
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), (dll_filename, line)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
return result
def __init__(self, qt_plugins, no_qt_translations):
self.qt_plugins = OrderedSet(x.strip().lower()
for x in qt_plugins.split(","))
self.no_qt_translations = no_qt_translations
self.webengine_done_binaries = False
self.webengine_done_data = False
self.qt_plugins_dirs = None
self.binding_package_name = ModuleName(self.binding_name)
# Allow to specify none.
if self.qt_plugins == set(["none"]):
self.qt_plugins = set()
# Prevent the list of binding names from being incomplete, it's used for conflicts.
assert self.binding_name in _qt_binding_names, self.binding_name
# Also lets have consistency in naming.
assert self.plugin_name in getQtPluginNames()
active_qt_plugin_name = getActiveQtPlugin()
if active_qt_plugin_name is not None:
self.sysexit(
"Error, confliciting plugin '%s', you can only have one enabled."
% active_qt_plugin_name)
self.warned_about = set()
def onMissingTrust(operation, source_ref, *args):
key = (operation, ) + args
if key not in _missing_trust:
_missing_trust[key] = OrderedSet()
_missing_trust[key].add(source_ref)
def removeDllDependencies(dll_filename, dll_filenames):
"""Create list of removable shared libraries by scanning through the plugins.
Args:
dll_filename: shared library filename
dll_filenames: list of shared library filenames
Returns:
list of removable files
"""
dll_filenames = tuple(sorted(dll_filenames))
to_remove = OrderedSet()
for plugin in getActivePlugins():
removed_dlls = tuple(
plugin.removeDllDependencies(dll_filename, dll_filenames))
if removed_dlls and Options.isShowInclusion():
plugin.info("Removing DLLs %s of %s by plugin decision." %
(dll_filename, removed_dlls))
for removed_dll in removed_dlls:
to_remove.add(removed_dll)
for removed in to_remove:
dll_filenames.discard(removed)
def buildOrderedSet(*producers):
values = []
for producer in producers:
values.extend(producer)
return OrderedSet(values)
def getDataFileTags(included_datafile):
tags = OrderedSet([included_datafile.kind])
tags.update(Options.getDataFileTags(tags))
for plugin in getActivePlugins():
plugin.updateDataFileTags(included_datafile)
return tags
def __init__(self, module):
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(self,
owner=module,
name="module",
parent=None)
self.used_modules = OrderedSet()
def __init__(self, qt_plugins):
self.webengine_done = False
self.qt_plugins_dirs = None
self.qt_plugins = OrderedSet(x.strip().lower() for x in qt_plugins.split(","))
# Allow to specify none.
if self.qt_plugins == set(["none"]):
self.qt_plugins = set()
def getPackageFiles(module, packages, folders_only):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest), if folders_only is False,
else tuples (_createEmptyDirNone, dest).
"""
# TODO: Maybe use isinstance(basestring) for this
if not hasattr(packages, "__getitem__"): # so should be a string type
packages = (packages, )
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(pkg_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", ))
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
if folders_only is False: # normal case: indeed copy the files
item_set.add((f, target))
else: # just create the empty folder structure
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def replaceRootModule(old, new):
# Using global here, as this is really a singleton, in the form of a module,
# pylint: disable=global-statement
global root_modules
new_root_modules = OrderedSet()
for module in root_modules:
new_root_modules.add(module if module is not old else new)
root_modules = new_root_modules
def getDataFileTags(dest_path):
result = OrderedSet()
for value in options.data_file_tags:
pattern, tags = value.rsplit(":", 1)
if fnmatch.fnmatch(dest_path, pattern):
result.update(tags.split(","))
return result
def getSubDirectoryFiles(module, subdirs, folders_only):
"""Yield filenames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else tuples (_createEmptyDirNone, dest) are yielded.
"""
module_folder = module.getCompileTimeDirectory()
elements = module.getFullName().split(".")
filename_start = module_folder.find(elements[0])
file_list = []
item_set = OrderedSet()
if subdirs is None:
data_dirs = [module_folder]
elif isinstance(subdirs, basestring):
data_dirs = [os.path.join(module_folder, subdirs)]
else:
data_dirs = [os.path.join(module_folder, subdir) for subdir in subdirs]
# Gather the full file list, probably makes no sense to include bytecode files
file_list = sum(
(getFileList(data_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", )) for data_dir in data_dirs),
[],
)
if not file_list:
msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % (
module.getFullName(),
subdirs,
data_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for f in file_list:
target = f[filename_start:]
if folders_only is False:
item_set.add((f, target))
else:
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def __init__(self, module):
assert module.isCompiledPythonModule(), module
CollectionStartpointMixin.__init__(self)
TraceCollectionBase.__init__(self,
owner=module,
name="module:" + module.getFullName(),
parent=None)
self.used_modules = OrderedSet()
def _get_subdir_files(module, subdirs, folders_only):
"""Yield filenames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else tuples (_createEmptyDirNone, dest) are yielded.
"""
module_folder = module.getCompileTimeDirectory()
elements = module.getFullName().split(".")
filename_start = module_folder.find(elements[0])
file_list = []
item_set = OrderedSet()
if subdirs is None:
file_list = getFileList(module_folder)
elif type(subdirs) is str:
data_dir = os.path.join(module_folder, subdirs)
file_list = getFileList(data_dir)
else:
for subdir in subdirs:
data_dir = os.path.join(module_folder, subdir)
file_list.extend(getFileList(data_dir))
if file_list == []:
msg = "No files or folders found for '%s' in subfolder(s) '%s'." % (
module.getFullName(),
str(subdirs),
)
warning(msg)
yield ()
for f in file_list:
if "__pycache__" in f or f.endswith(".pyc"): # probably makes no sense
continue
target = f[filename_start:]
if folders_only is False:
item_set.add((f, target))
else:
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in open(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith('('):
# No multiline imports please
assert imported.endswith(')')
imported = imported[1:-1]
assert imported
if imported == '*':
continue
for name in imported.split(','):
name = name.strip()
pyi_deps.add(parts[1] + '.' + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple(
(pyi_dep, None)
for pyi_dep in
pyi_deps
)
else:
self.used_modules = ()
def getModuleSpecificDllPaths(module_name):
"""Provide a list of directories, where DLLs should be searched for this package (or module).
Args:
module_name: name of a package or module, for which the DLL path addition applies.
"""
result = OrderedSet()
for plugin in getActivePlugins():
for dll_path in plugin.getModuleSpecificDllPaths(module_name):
result.add(dll_path)
return result
def getExtraLinkLibraries():
result = OrderedSet()
for plugin in getActivePlugins():
value = plugin.getExtraLinkLibraries()
if value is not None:
if isinstance(value, basestring):
result.add(value)
else:
for library_name in value:
result.add(library_name)
return result
def getPluginsEnabled():
"""*tuple*, user enabled (standard) plugins (not including user plugins)
Note:
Do not use this outside of main binary, as other plugins, e.g.
hinted compilation will activate plugins themselves and this
will not be visible here.
"""
result = OrderedSet()
if options:
for plugin_enabled in options.plugins_enabled:
result.update(plugin_enabled.split(","))
return tuple(result)
def __init__(self, name, code_prefix, source_ref):
CodeNodeBase.__init__(
self,
name = name,
code_prefix = code_prefix,
source_ref = source_ref
)
self.providing = OrderedDict()
self.keeper_variables = OrderedSet()
self.temp_variables = OrderedDict()
self.temp_scopes = OrderedDict()
def getExtraLinkLibraries(cls):
if cls.extra_link_libraries is None:
cls.extra_link_libraries = OrderedSet()
for plugin in getActivePlugins():
value = plugin.getExtraLinkLibraries()
if value is not None:
if isinstance(value, basestring):
cls.extra_link_libraries.add(value)
else:
for library_name in value:
cls.extra_link_libraries.add(library_name)
return cls.extra_link_libraries
def _getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
package_dir = locateModule(module_name=package_name,
parent_package=None,
level=0)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def getPluginsEnabled():
"""*tuple*, user enabled (standard) plugins (not including user plugins)
Note:
Do not use this outside of main binary, as plugins are allowed
to activate plugins themselves and that will not be visible here.
"""
result = OrderedSet()
if options:
for plugin_enabled in options.plugins_enabled:
result.update(
getPluginNameConsideringRenames(plugin_name)
for plugin_name in plugin_enabled.split(","))
return tuple(result)
