def getModuleNameAndKindFromFilename(module_filename):
"""Given a filename, decide the module name and kind.
Args:
module_name - file path of the module
Returns:
Tuple with the name of the module basename, and the kind of the
module derived from the file suffix. Can be None, None if is is not a
known file suffix.
Notes:
This doesn't concern itself with packages, that needs to be tracked
by the using code. It cannot be decided from the filename at all.
"""
# TODO: This does not handle ".pyw" files it seems.
if os.path.isdir(module_filename):
module_name = ModuleName(os.path.basename(module_filename))
module_kind = "py"
elif module_filename.endswith(".py"):
module_name = ModuleName(os.path.basename(module_filename)[:-3])
module_kind = "py"
else:
for suffix in getSharedLibrarySuffixes():
if module_filename.endswith(suffix):
module_name = ModuleName(
os.path.basename(module_filename)[: -len(suffix)]
)
module_kind = "extension"
break
else:
module_kind = None
module_name = None
return module_name, module_kind
def buildMainModuleTree(filename, is_main):
# Detect to be frozen modules if any, so we can consider to not follow
# to them.
if is_main:
# TODO: Doesn't work for deeply nested packages at all.
if Options.hasPythonFlagPackageMode():
module_name = ModuleName(os.path.basename(filename) + ".__main__")
else:
module_name = ModuleName("__main__")
else:
module_name = Importing.getModuleNameAndKindFromFilename(filename)[0]
module, _added = buildModule(
module_name=module_name,
module_filename=filename,
source_code=None,
is_top=True,
is_main=is_main,
is_extension=False,
is_fake=False,
hide_syntax_error=False,
)
if is_main and Options.isStandaloneMode():
module.setEarlyModules(detectEarlyImports())
# Main modules do not get added to the import cache, but plugins get to see it.
if module.isMainModule():
Plugins.onModuleDiscovered(module)
else:
addImportedModule(imported_module=module)
return module
def _find_to_build(self):
"""
Helper for _build
Returns list containing bool (is_package) and module_names
Algorithm for finding distinct packages:
1) Take minimum package
2) Find related packages that start with this name
3) Add this to the list to return, then repeat steps 1 & 2
until no more packages exist
"""
builds = []
# Namespace packages can use / rather than dots.
py_packages = [
ModuleName(m.replace("/", "."))
for m in sorted(set(self.compile_packages))
]
py_modules = [ModuleName(m) for m in sorted(set(self.py_modules))]
for script_module_name in self.script_module_names:
script_module_filename = locateModule(
module_name=script_module_name, parent_package=None,
level=0)[1]
# Decide package or module.
(
_main_added,
is_package,
_is_namespace,
_source_ref,
_source_filename,
) = decideModuleSourceRef(
filename=script_module_filename,
module_name=script_module_name,
is_main=False,
is_fake=False,
logger=wheel_logger,
)
if is_package:
py_packages.append(script_module_name)
else:
py_modules.append(script_module_name)
# Plain modules if they are not in packages to build.
builds.extend((False, current_module) for current_module in py_modules
if not current_module.hasOneOfNamespaces(py_packages))
while py_packages:
current_package = min(py_packages)
py_packages = [
p for p in py_packages if not p.hasNamespace(current_package)
]
builds.append((True, current_package))
return builds
def considerImplicitImports(self, module, signal_change):
"""Provide additional modules to import implicitly when encountering the module.
Notes:
Better do not overload this method.
The standard plugin 'ImplicitImports.py' already contains MANY of these.
If you do have a new candidate, consider a PR to get it included there.
Args:
module: the module object
signal_change: bool
Returns:
None
"""
from nuitka.importing.Importing import getModuleNameAndKindFromFilename
for full_name in self.getImplicitImports(module):
if type(full_name) in (tuple, list):
raise NuitkaPluginError(
"Plugin %s needs to be change to only return modules names, not %r"
% (self, full_name))
full_name = ModuleName(full_name)
try:
module_filename = self.locateModule(importing=module,
module_name=full_name)
except Exception:
self.warning("Problem locating '%s' implicit imports '%s'." %
(module.getFullName(), full_name))
raise
if module_filename is None:
if Options.isShowInclusion():
self.info(
"Implicit module '%s' suggested by '%s' not found." %
(full_name, module.getFullName()))
continue
_module_name2, module_kind = getModuleNameAndKindFromFilename(
module_filename)
# TODO: This should get back to plug-ins, they should be allowed to
# preempt or override the decision.
decision, reason = self.decideRecursion(
module_filename=module_filename,
module_name=full_name,
module_kind=module_kind,
)
if decision:
self.recurseTo(
module_package=full_name.getPackageName(),
module_filename=module_filename,
module_kind=module_kind,
reason=reason,
signal_change=signal_change,
)
def considerImplicitImports(self, module, signal_change):
""" Provide additional modules to import implicitly when encountering the module.
Notes:
Better do not overload this method.
The standard plugin 'ImplicitImports.py' already contains MANY of these.
If you do have a new candidate, consider a PR to get it included there.
Args:
module: the module object
signal_change: bool
Returns:
None
"""
from nuitka.importing.Importing import getModuleNameAndKindFromFilename
for item in self.getImplicitImports(module):
# TODO: Temporary, until all plugins are caught up, turn into an error later.
if type(item) in (tuple, list):
full_name, _required = item
else:
full_name = item
full_name = ModuleName(full_name)
module_filename = self.locateModule(importing=module,
module_name=full_name)
if module_filename is None:
if Options.isShowInclusion():
self.info(
"Implicit module '%s' suggested by '%s' not found." %
(full_name, module.getFullName()))
continue
_module_name2, module_kind = getModuleNameAndKindFromFilename(
module_filename)
# TODO: This should get back to plug-ins, they should be allowed to
# preempt or override the decision.
decision, reason = self.decideRecursion(
module_filename=module_filename,
module_name=full_name,
module_kind=module_kind,
)
if decision:
self.recurseTo(
module_package=full_name.getPackageName(),
module_filename=module_filename,
module_kind=module_kind,
reason=reason,
signal_change=signal_change,
)
def considerImplicitImports(self, module, signal_change):
""" Provide additional modules to import implicitly when encountering the module.
Notes:
Better do not overload this method.
The standard plugin 'ImplicitImports.py' already contains MANY of these.
If you do have a new candidate, consider a PR to get it included there.
Args:
module: the module object
signal_change: bool
Returns:
None
"""
from nuitka.importing.Importing import getModuleNameAndKindFromFilename
for full_name, required in self.getImplicitImports(module):
full_name = ModuleName(full_name)
module_filename = self.locateModule(
importing=module, module_name=full_name, warn=required
)
if module_filename is None:
if required:
sys.exit(
"Error, implicit module '%s' expected by '%s' not found."
% (full_name, module.getFullName())
)
else:
continue
_module_name2, module_kind = getModuleNameAndKindFromFilename(
module_filename
)
# TODO: This should get back to plug-ins, they should be allowed to
# preempt or override the decision.
decision, reason = self.decideRecursion(
module_filename=module_filename,
module_name=full_name,
module_kind=module_kind,
)
if decision:
self.recurseTo(
module_package=full_name.getPackageName(),
module_filename=module_filename,
module_kind=module_kind,
reason=reason,
signal_change=signal_change,
)
def locateModules(package_name):
"""Determine child module names.
Return:
generator of ModuleName objects
"""
package_name = ModuleName(package_name)
module_filename = locateModule(
module_name=ModuleName(package_name), parent_package=None, level=0
)[1]
if module_filename is not None:
for sub_module in iter_modules([module_filename]):
yield package_name.getChildNamed(sub_module.name)
def getCachedImportedModulesNames(module_name, source_code):
cache_name = makeCacheName(module_name, source_code)
return [
ModuleName(line) for line in getFileContents(
_getCacheFilename(cache_name, "txt")).strip().split("\n")
]
def __init__(self, qt_plugins, no_qt_translations):
self.qt_plugins = OrderedSet(x.strip().lower()
for x in qt_plugins.split(","))
self.no_qt_translations = no_qt_translations
self.webengine_done_binaries = False
self.webengine_done_data = False
self.qt_plugins_dirs = None
self.binding_package_name = ModuleName(self.binding_name)
# Allow to specify none.
if self.qt_plugins == set(["none"]):
self.qt_plugins = set()
# Prevent the list of binding names from being incomplete, it's used for conflicts.
assert self.binding_name in _qt_binding_names, self.binding_name
# Also lets have consistency in naming.
assert self.plugin_name in getQtPluginNames()
active_qt_plugin_name = getActiveQtPlugin()
if active_qt_plugin_name is not None:
self.sysexit(
"Error, confliciting plugin '%s', you can only have one enabled."
% active_qt_plugin_name)
self.warned_about = set()
def _scanModules(path, prefix):
for module_info in pkgutil.walk_packages((path, ),
prefix=prefix + "."):
result.append(ModuleName(module_info[1]))
if module_info[2]:
_scanModules(module_info[1], module_name + module_info[1])
def locateModule(module_name, parent_package, level):
"""Locate a module with given package name as parent.
The package name can be None of course. Level is the same
as with "__import__" built-in.
Returns:
Returns a triple of module name the module has considering
package containing it, and filename of it which can be a
directory for packages, and the location method used.
"""
module_package, module_filename, finding = findModule(
module_name=module_name,
parent_package=parent_package,
level=level,
)
assert module_package is None or (
type(module_package) is ModuleName and module_package != ""
), repr(module_package)
if module_filename is not None:
module_filename = os.path.normpath(module_filename)
module_name, module_kind = getModuleNameAndKindFromFilename(module_filename)
assert module_kind is not None, module_filename
module_name = ModuleName.makeModuleNameInPackage(module_name, module_package)
return module_name, module_filename, finding
def _consider(self, trace_collection, module_filename, module_package):
assert module_package is None or (
type(module_package) is ModuleName
and module_package != ""), repr(module_package)
module_filename = os.path.normpath(module_filename)
module_name, module_kind = getModuleNameAndKindFromFilename(
module_filename)
if module_kind is not None:
module_fullpath = ModuleName.makeModuleNameInPackage(
module_name, module_package)
decision, reason = decideRecursion(
module_filename=module_filename,
module_name=module_fullpath,
module_kind=module_kind,
)
if decision:
module_relpath = relpath(module_filename)
imported_module, added_flag = recurseTo(
module_package=module_package,
module_filename=module_filename,
module_relpath=module_relpath,
module_kind=module_kind,
reason=reason,
)
if added_flag:
trace_collection.signalChange(
"new_code",
imported_module.getSourceReference(),
"Recursed to module.",
)
return imported_module
elif decision is False and module_kind == "py":
uncompiled_module = getUncompiledModule(
module_fullpath, module_filename)
if uncompiled_module is not None:
return uncompiled_module
elif decision is None and module_kind == "py":
if (module_filename not in self._warned_about
and module_fullpath not in getModuleIgnoreList()):
self._warned_about.add(module_filename)
inclusion_logger.warning("""\
Not recursing to '%(full_path)s' (%(filename)s), please specify \
--nofollow-imports (do not warn), \
--follow-imports (recurse to all), \
--nofollow-import-to=%(full_path)s (ignore it), \
--follow-import-to=%(full_path)s (recurse to it) to change.""" % {
"full_path": module_fullpath,
"filename": module_filename
})
def considerFailedImportReferrals(module_name):
for plugin in active_plugin_list:
new_module_name = plugin.considerFailedImportReferrals(module_name)
if new_module_name is not None:
return ModuleName(new_module_name)
return None
def considerFailedImportReferrals(module_name):
for plugin in getActivePlugins():
new_module_name = plugin.considerFailedImportReferrals(module_name)
if new_module_name is not None:
return ModuleName(new_module_name)
return None
def normalizePackageName(module_name):
# The "os.path" is strangely hacked into the "os" module, dispatching per
# platform, we either cannot look into it, or we require that we resolve it
# here correctly.
if module_name == "os.path":
module_name = ModuleName(os.path.basename(os.path.__name__))
return module_name
def makeIncludedEntryPoint(kind, source_path, dest_path, package_name,
executable):
if package_name is not None:
package_name = ModuleName(package_name)
assert type(executable) is bool
return IncludedEntryPoint(kind, source_path, dest_path, package_name,
executable)
def __init__(self):
CompiledPythonModule.__init__(
self,
module_name=ModuleName("__internal__"),
is_top=False,
mode="compiled",
source_ref=SourceCodeReference.fromFilenameAndLine(
filename="internal", line=0),
future_spec=FutureSpec(),
)
def makeIncludedEntryPoint(
kind,
source_path,
dest_path,
package_name,
):
if package_name is not None:
package_name = ModuleName(package_name)
return IncludedEntryPoint(kind, source_path, dest_path, package_name)
def getImplicitImports(self, module):
if module.getFullName() == "OpenGL":
opengl_infos = self.queryRuntimeInformationSingle(
setup_codes="import OpenGL.plugins",
value=
"[(f.name, f.import_path) for f in OpenGL.plugins.FormatHandler.all()]",
)
# TODO: Filter by name.
for _name, import_path in opengl_infos:
yield ModuleName(import_path).getPackageName()
def __init__(
self,
noinclude_setuptools_mode,
noinclude_pytest_mode,
noinclude_ipython_mode,
noinclude_default_mode,
custom_choices,
):
# Default manually to default argument value:
if noinclude_setuptools_mode is None:
noinclude_setuptools_mode = noinclude_default_mode
if noinclude_pytest_mode is None:
noinclude_pytest_mode = noinclude_default_mode
if noinclude_ipython_mode is None:
noinclude_ipython_mode = noinclude_default_mode
self.config = parsePackageYaml(__package__, "anti-bloat.yml")
self.handled_modules = OrderedDict()
# These should be checked, to allow disabling anti-bloat contents.
self.control_tags = set()
if noinclude_setuptools_mode != "allow":
self.handled_modules["setuptools"] = noinclude_setuptools_mode
else:
self.control_tags.add("allow_setuptools")
if noinclude_pytest_mode != "allow":
self.handled_modules["pytest"] = noinclude_pytest_mode
else:
self.control_tags.add("allow_pytest")
if noinclude_ipython_mode != "allow":
self.handled_modules["IPython"] = noinclude_ipython_mode
else:
self.control_tags.add("allow_ipython")
for custom_choice in custom_choices:
if ":" not in custom_choice:
self.sysexit(
"Error, malformed value '%s' for '--noinclude-custom-mode' used."
% custom_choice
)
module_name, mode = custom_choice.rsplit(":", 1)
if mode not in ("error", "warning", "nofollow", "allow", "bytecode"):
self.sysexit(
"Error, illegal mode given '%s' in '--noinclude-custom-mode=%s'"
% (mode, custom_choice)
)
self.handled_modules[ModuleName(module_name)] = mode
def fromXML(cls, provider, source_ref, **args):
if "code_flags" in args:
future_spec = fromFlags(args["code_flags"])
result = cls(
main_added=args["main_added"] == "True",
mode=args["mode"],
module_name=ModuleName(args["module_name"]),
future_spec=future_spec,
source_ref=source_ref,
)
from nuitka.ModuleRegistry import addRootModule
addRootModule(result)
function_work = []
for xml in args["functions"]:
_kind, node_class, func_args, source_ref = extractKindAndArgsFromXML(
xml, source_ref)
if "provider" in func_args:
func_args["provider"] = getOwnerFromCodeName(
func_args["provider"])
else:
func_args["provider"] = result
if "flags" in args:
func_args["flags"] = set(func_args["flags"].split(","))
if "doc" not in args:
func_args["doc"] = None
function = node_class.fromXML(source_ref=source_ref, **func_args)
# Could do more checks for look up of body here, but so what...
function_work.append((function, iter(iter(xml).next()).next()))
for function, xml in function_work:
function.setChild(
"body",
fromXML(provider=function,
xml=xml,
source_ref=function.getSourceReference()),
)
result.setChild(
"body",
fromXML(provider=result,
xml=args["body"][0],
source_ref=source_ref))
return result
def __init__(self, main_added, mode, future_spec, source_ref):
self.main_added = main_added
CompiledPythonModule.__init__(
self,
module_name=ModuleName("__main__"),
is_top=True,
mode=mode,
future_spec=future_spec,
source_ref=source_ref,
)
def _createTriggerLoadedModule(module, trigger_name, code):
"""Create a "trigger" for a module to be imported.
Notes:
The trigger will incorpaorate the code to be prepended / appended.
Called by @onModuleDiscovered.
Args:
module: the module object (serves as dict key)
trigger_name: string ("-preload"/"-postload")
code: the code string
Returns
trigger_module
"""
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.tree.Building import createModuleTree
from .Plugins import Plugins
module_name = ModuleName(module.getFullName() + trigger_name)
source_ref = fromFilename(module.getCompileTimeFilename() +
trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode=mode,
future_spec=None,
source_ref=source_ref,
)
createModuleTree(
module=trigger_module,
source_ref=module.getSourceReference(),
source_code=code,
is_main=False,
)
if mode == "bytecode":
trigger_module.setSourceCode(code)
if Options.isDebug():
source_path = os.path.join(
OutputDirectories.getSourceDirectoryPath(),
module_name + ".py")
with open(source_path, "w") as output:
output.write(code)
return trigger_module
def onModuleInitialSet(self):
from nuitka.importing.ImportCache import addImportedModule
from nuitka.ModuleRegistry import getRootTopModule
from nuitka.plugins.Plugins import Plugins
from nuitka.tree.Building import (
CompiledPythonModule,
createModuleTree,
readSourceCodeFromFilename,
)
# First, build the module node and then read again from the
# source code.
root_module = getRootTopModule()
module_name = ModuleName("__parents_main__")
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename(module_name,
root_module.getFilename())
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.
if python_version >= 0x340:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
addImportedModule(imported_module=slave_main_module)
yield slave_main_module
def _createTriggerLoadedModule(module, trigger_name, code, flags):
"""Create a "trigger" for a module to be imported.
Notes:
The trigger will incorpaorate the code to be prepended / appended.
Called by @onModuleDiscovered.
Args:
module: the module object (serves as dict key)
trigger_name: string ("-preload"/"-postload")
code: the code string
Returns
trigger_module
"""
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.tree.Building import createModuleTree
from .Plugins import Plugins
module_name = ModuleName(module.getFullName() + trigger_name)
source_ref = fromFilename(module.getCompileTimeFilename() +
trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode=mode,
future_spec=None,
source_ref=source_ref,
)
createModuleTree(
module=trigger_module,
source_ref=module.getSourceReference(),
source_code=code,
is_main=False,
)
if mode == "bytecode":
trigger_module.setSourceCode(code)
# In debug mode, put the files in the build folder, so they can be looked up easily.
if Options.is_debug and "HIDE_SOURCE" not in flags:
source_path = os.path.join(
OutputDirectories.getSourceDirectoryPath(),
module_name + ".py")
putTextFileContents(filename=source_path, contents=code)
return trigger_module
def _considerImplicitImports(plugin, module):
from nuitka.importing import Importing
result = []
for full_name in plugin.getImplicitImports(module):
if type(full_name) in (tuple, list):
raise NuitkaPluginError(
"Plugin %r needs to be change to only return modules names, not %r"
% (plugin, full_name)
)
full_name = ModuleName(full_name)
try:
_module_package, module_filename, _finding = Importing.findModule(
importing=module,
module_name=full_name,
parent_package=None,
level=-1,
warn=False,
)
module_filename = plugin.locateModule(
importing=module, module_name=full_name
)
except Exception:
plugin.warning(
"Problem locating '%s' for implicit imports of '%s'."
% (module.getFullName(), full_name)
)
raise
if module_filename is None:
if Options.isShowInclusion():
plugin.info(
"Implicit module '%s' suggested for '%s' not found."
% (full_name, module.getFullName())
)
continue
result.append((full_name, module_filename))
if result:
plugin.info(
"Implicit dependencies of module '%s' added '%s'."
% (module.getFullName(), ",".join(r[0] for r in result))
)
return result
def __init__(self, main_added, mode, future_spec, source_ref):
# Is this one from a "__main__.py" file.
self.main_added = main_added
CompiledPythonModule.__init__(
self,
module_name=ModuleName("__main__"),
is_top=True,
mode=mode,
future_spec=future_spec,
source_ref=source_ref,
)
self.early_modules = ()
def locateModule(module_name):
"""Provide a filename / -path for a to-be-imported module.
Args:
importing: module object that asked for it (tracing only)
module_name: (str or ModuleName) full name of module
Returns:
filename for module
"""
from nuitka.importing.Importing import locateModule
_module_name, module_filename, _finding = locateModule(
module_name=ModuleName(module_name), parent_package=None, level=0)
return module_filename
def __init__(self, module_name, source_ref):
ExpressionBase.__init__(self, source_ref=source_ref)
self.module_name = ModuleName(module_name)
self.finding = None
self.module_filename = None
_module_name, self.module_filename, self.finding = locateModule(
module_name=self.module_name,
parent_package=None,
level=0,
)
# Expect to find them and to match the name of course.
assert self.finding != "not-found", self.module_name
assert _module_name == self.module_name
def getCachedImportedModulesNames(module_name, source_code):
cache_name = makeCacheName(module_name, source_code)
cache_filename = _getCacheFilename(cache_name, "json")
if not os.path.exists(cache_filename):
return None
data = loadJsonFromFilename(cache_filename)
if data is None:
return None
if data.get("file_format_version") != _cache_format_version:
return None
if data["module_name"] != module_name:
return None
return [(ModuleName(used_module_name), line_number)
for (used_module_name, line_number) in data["modules_used"]]
