def optimizeUncompiledPythonModule(module):
full_name = module.getFullName()
progress_logger.info(
"Doing module dependency considerations for '{module_name}':".format(
module_name=full_name
)
)
for used_module_name, used_module_path in module.getUsedModules():
used_module = ImportCache.getImportedModuleByNameAndPath(
used_module_name, used_module_path
)
ModuleRegistry.addUsedModule(used_module)
package_name = full_name.getPackageName()
if package_name is not None:
# TODO: It's unclear why, but some standard library modules on older Python3
# seem to not have parent packages after the scan.
try:
used_module = ImportCache.getImportedModuleByName(package_name)
except KeyError:
pass
else:
ModuleRegistry.addUsedModule(used_module)
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def _detectedPrecompiledFile(filename, module_name, result, user_provided,
technical):
if filename.endswith(".pyc"):
if os.path.isfile(filename[:-1]):
return _detectedSourceFile(
filename=filename[:-1],
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
if module_name in module_names:
return
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=loadCodeObjectData(precompiled_filename=filename),
is_package="__init__" in filename,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def _detectedShlibFile(filename, module_name):
# That is not a shared library, but looks like one.
if module_name == "__main__":
return
from nuitka import ModuleRegistry
if ModuleRegistry.hasRootModule(module_name):
return
parts = module_name.split('.')
if len(parts) == 1:
package_name = None
name = module_name
else:
package_name = '.'.join(parts[:-1])
name = parts[-1]
source_ref = SourceCodeReferences.fromFilename(
filename = filename
)
shlib_module = PythonShlibModule(
name = name,
package_name = package_name,
source_ref = source_ref
)
ModuleRegistry.addRootModule(shlib_module)
ImportCache.addImportedModule(shlib_module)
module_names.add(module_name)
def _detectedShlibFile(filename, module_name):
# That is not a shared library, but looks like one.
if module_name == "__main__":
return
from nuitka import ModuleRegistry
if ModuleRegistry.hasRootModule(module_name):
return
parts = module_name.split('.')
if len(parts) == 1:
package_name = None
name = module_name
else:
package_name = '.'.join(parts[:-1])
name = parts[-1]
source_ref = SourceCodeReferences.fromFilename(
filename = filename
)
shlib_module = PythonShlibModule(
name = name,
package_name = package_name,
source_ref = source_ref
)
ModuleRegistry.addRootModule(shlib_module)
ImportCache.addImportedModule(shlib_module)
module_names.add(module_name)
def _detectedSourceFile(filename, module_name, result, user_provided,
technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(filename=filename,
module_name="_collections_abc",
result=result,
user_provided=user_provided,
technical=technical)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep, os.path.basename(filename), source_code)
# Debian stretch site.py
source_code = source_code.replace(
"PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")
# Anaconda3 4.1.2 site.py
source_code = source_code.replace(
"def main():",
"def main():return\n\nif 0:\n def _unused():",
)
debug("Freezing module '%s' (from '%s').", module_name, filename)
is_package = os.path.basename(filename) == "__init__.py"
source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
is_package=is_package,
source_code=source_code)
bytecode = compile(source_code, filename, "exec", dont_inherit=True)
bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
is_package=is_package,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(bytecode),
is_package=is_package,
filename=filename,
user_provided=user_provided,
technical=technical)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def optimizeUncompiledPythonModule(module):
if _progress:
info("Doing module dependency considerations for '{module_name}':".
format(module_name=module.getFullName()))
for used_module_name in module.getUsedModules():
used_module = ImportCache.getImportedModuleByName(used_module_name)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
def recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
if not ImportCache.isImportedModuleByPath(module_relpath):
module, source_ref, source_filename = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib")
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
debug("Recurse to import '%s' from %s. (%s)", module.getFullName(),
module_relpath, reason)
if module_kind == "py" and source_filename is not None:
try:
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=readSourceCodeFromFilename(
source_filename),
is_main=False)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath,
module_filename, e.__class__.__name__)
return None, False
ImportCache.addImportedModule(module_relpath, module)
is_added = True
else:
ImportCache.addImportedModule(
module_relpath,
ImportCache.getImportedModuleByName(module.getFullName()))
module = ImportCache.getImportedModuleByName(module.getFullName())
is_added = False
assert not module_relpath.endswith("/__init__.py"), module
return module, is_added
else:
return ImportCache.getImportedModuleByPath(module_relpath), False
def optimizeUncompiledPythonModule(module):
if _progress:
info("Doing module dependency considerations for '{module_name}':".
format(module_name=module.getFullName()))
for used_module_name, used_module_path in module.getUsedModules():
used_module = ImportCache.getImportedModuleByNameAndPath(
used_module_name, used_module_path)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def _recurseTo(module_name, module_filename, module_kind):
from nuitka.tree import Building
module, is_added = Building.buildModule(
module_filename=module_filename,
module_name=module_name,
source_code=None,
is_top=False,
is_main=False,
is_extension=module_kind == "extension",
is_fake=False,
hide_syntax_error=True,
)
ImportCache.addImportedModule(module)
return module, is_added
def optimizeUncompiledPythonModule(module):
if _progress:
printLine(
"Doing module dependency considerations for '{module_name}':".format(
module_name = module.getFullName()
)
)
for used_module_name in module.getUsedModules():
used_module = ImportCache.getImportedModuleByName(used_module_name)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
def recurseTo(module_package, module_filename, module_relpath, module_kind, reason):
if ImportCache.isImportedModuleByPath(module_relpath):
try:
module = ImportCache.getImportedModuleByPath(module_relpath, module_package)
except KeyError:
module = None
else:
module = None
if module is None:
return _recurseTo(
module_package=module_package,
module_filename=module_filename,
module_relpath=module_relpath,
module_kind=module_kind,
reason=reason,
)
else:
return module, False
def recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
if ImportCache.isImportedModuleByPath(module_relpath):
module = ImportCache.getImportedModuleByPath(module_relpath)
if module.getCompileTimeFilename().endswith("__init__.py") and \
module.getPackage() != module_package:
module = None
else:
module = None
if module is None:
return _recurseTo(module_package=module_package,
module_filename=module_filename,
module_relpath=module_relpath,
module_kind=module_kind,
reason=reason)
else:
return module, False
def recurseTo(module_package, module_filename, module_relpath, module_kind, reason):
if ImportCache.isImportedModuleByPath(module_relpath):
try:
module = ImportCache.getImportedModuleByPath(module_relpath, module_package)
except KeyError:
module = None
else:
module = None
if module is None:
return _recurseTo(
module_package=module_package,
module_filename=module_filename,
module_relpath=module_relpath,
module_kind=module_kind,
reason=reason,
)
else:
return module, False
def _detectedShlibFile(filename, module_name):
# That is not a shared library, but looks like one.
if module_name == "__main__":
return
# Cyclic dependency
from nuitka import ModuleRegistry
if ModuleRegistry.hasRootModule(module_name):
return
source_ref = SourceCodeReferences.fromFilename(filename=filename)
shlib_module = PythonShlibModule(module_name=module_name, source_ref=source_ref)
ModuleRegistry.addRootModule(shlib_module)
ImportCache.addImportedModule(shlib_module)
module_names.add(module_name)
def optimizeUncompiledPythonModule(module):
if _progress:
info(
"Doing module dependency considerations for '{module_name}':".format(
module_name=module.getFullName()
)
)
for used_module_name, used_module_path in module.getUsedModules():
used_module = ImportCache.getImportedModuleByNameAndPath(
used_module_name, used_module_path
)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def _detectedExtensionModule(filename, module_name, result, technical):
# That is not a shared library, but looks like one.
if module_name == "__main__":
return
# Extension modules are not tracked outside of standalone
# mode.
if not Options.isStandaloneMode():
return
# Avoid duplicates
if module_name in module_names:
return
source_ref = SourceCodeReferences.fromFilename(filename=filename)
extension_module = PythonExtensionModule(module_name=module_name,
technical=technical,
source_ref=source_ref)
ImportCache.addImportedModule(extension_module)
module_names.add(module_name)
result.append(extension_module)
def recurseTo(signal_change, module_name, module_filename, module_kind,
reason):
try:
module = ImportCache.getImportedModuleByNameAndPath(
module_name, module_filename)
except KeyError:
module = None
if module is None:
module, added_flag = _recurseTo(
module_name=module_name,
module_filename=module_filename,
module_kind=module_kind,
)
if added_flag and signal_change is not None:
signal_change("new_code", module.getSourceReference(), reason)
return module
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
debug("Checking detail plug-in path '%s' '%s':", plugin_filename, module_package)
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename
)
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_package=module_package,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(
module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind=module_kind,
reason=reason,
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_filename,
)
if not isSameModulePath(module.getFilename(), plugin_filename):
warning(
"Duplicate '%s' of '%s' ignored .",
plugin_filename,
module.getFilename(),
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module,
)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug("Package directory %s", package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or sub_path.endswith(".py"):
checkPluginSinglePath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_filename)
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
debug("Checking detail plug-in path '%s' '%s':", plugin_filename,
module_package)
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
if module_kind is not None:
decision, reason = decideRecursion(module_filename=plugin_filename,
module_name=module_name,
module_package=module_package,
module_kind=module_kind,
extra_recursion=True)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind="py",
reason=reason)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.", "package"
if module.isCompiledPythonPackage() else "module",
module.getName(), plugin_filename)
if not isSameModulePath(module.getFilename(),
plugin_filename):
warning("Duplicate '%s' of '%s' ignored .",
plugin_filename, module.getFilename())
return
debug("Recursed to %s %s %s", module.getName(),
module.getPackage(), module)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug("Package directory %s", package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_filename)
def recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
if not ImportCache.isImportedModuleByPath(module_relpath):
module, source_ref, source_filename = Building.decideModuleTree(
filename = module_filename,
package = module_package,
is_top = False,
is_main = False,
is_shlib = module_kind == "shlib"
)
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
debug(
"Recurse to import '%s' from %s. (%s)",
module.getFullName(),
module_relpath,
reason
)
if module_kind == "py" and source_filename is not None:
try:
Building.createModuleTree(
module = module,
source_ref = source_ref,
source_code = readSourceCodeFromFilename(
module_name = module.getFullName(),
source_filename = source_filename
),
is_main = False
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__
)
return None, False
ImportCache.addImportedModule(
module_relpath,
module
)
is_added = True
else:
ImportCache.addImportedModule(
module_relpath,
ImportCache.getImportedModuleByName(module.getFullName())
)
module = ImportCache.getImportedModuleByName(
module.getFullName()
)
is_added = False
assert not module_relpath.endswith("/__init__.py"), module
return module, is_added
else:
return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
module, source_ref, source_filename = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
logRecursion(
"Recurse to import '%s' from '%s'. (%s)",
module.getFullName(),
module_relpath,
reason,
)
if module_kind == "py" and source_filename is not None:
try:
source_code = readSourceCodeFromFilename(
module_name=module.getFullName(), source_filename=source_filename
)
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__,
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(
source_code, module_filename, "exec", dont_inherit=True
)
),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
if Options.isShowInclusion():
recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
(plugin_filename, module_package))
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
module_name = ModuleName.makeModuleNameInPackage(module_name,
module_package)
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(
module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind=module_kind,
reason=reason,
)
if module:
if not is_added:
recursion_logger.warning(
"Recursed to %s '%s' at '%s' twice." % (
"package"
if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_filename,
))
if not isSameModulePath(module.getFilename(),
plugin_filename):
recursion_logger.warning(
"Duplicate '%s' of '%s' ignored ." % (
plugin_filename,
module.getFilename(),
))
return
if Options.isShowInclusion():
recursion_logger.info("Recursed to '%s' %s" % (
module.getFullName(),
module,
))
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 0x300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
if Options.isShowInclusion():
recursion_logger.info("Package directory '%s'." %
package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(
sub_path) and not os.path.exists(sub_path +
".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif sub_path.endswith(".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
elif module.isPythonShlibModule():
if Options.isStandaloneMode():
ModuleRegistry.addRootModule(module)
else:
recursion_logger.warning(
"Failed to include module from '%s'." % plugin_filename)
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
# The importing wants these to be unique.
plugin_filename = os.path.abspath(plugin_filename)
if Options.isShowInclusion():
recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
(plugin_filename, module_package))
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
module_name = ModuleName.makeModuleNameInPackage(module_name,
module_package)
if module_kind == "extension" and not Options.isStandaloneMode():
recursion_logger.warning(
"Cannot include '%s' unless using at least standalone mode." %
module_name.asString())
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module = recurseTo(
signal_change=None,
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
reason=reason,
)
if module:
if Options.isShowInclusion():
recursion_logger.info("Included '%s' as '%s'." % (
module.getFullName(),
module,
))
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 0x300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
if Options.isShowInclusion():
recursion_logger.info("Package directory '%s'." %
package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(
sub_path) and not os.path.exists(sub_path +
".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif sub_path.endswith(".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
elif module.isPythonExtensionModule():
if Options.isStandaloneMode():
ModuleRegistry.addRootModule(module)
else:
recursion_logger.warning(
"Failed to include module from '%s'." % plugin_filename)
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
from nuitka.tree import Building
module, source_ref, source_code = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
if Options.isShowInclusion():
recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" %
(module.getFullName(), module_relpath, reason))
if source_code is not None:
try:
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning(
"""\
Cannot follow import to module %r (%r) because of %r""" %
(module_relpath, module_filename, e.__class__.__name__))
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot recurse to import module %r (%r) because code is too complex.""" % (
module_relpath,
module_filename,
))
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code,
module_filename,
"exec",
dont_inherit=True)),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
if not ImportCache.isImportedModuleByPath(module_relpath):
module, source_ref, source_filename = Building.decideModuleTree(
filename = module_filename,
package = module_package,
is_top = False,
is_main = False,
is_shlib = module_kind == "shlib"
)
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
debug(
"Recurse to import '%s' from %s. (%s)",
module.getFullName(),
module_relpath,
reason
)
if module_kind == "py" and source_filename is not None:
try:
Building.createModuleTree(
module = module,
source_ref = source_ref,
source_code = readSourceCodeFromFilename(
module_name = module.getFullName(),
source_filename = source_filename
),
is_main = False
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name = module.getFullName(),
filename = module_filename,
bytecode = marshal.dumps(
compile(
readSourceCodeFromFilename(module.getFullName(), module_filename),
module_filename,
"exec"
)
),
is_package = module.isCompiledPythonPackage(),
user_provided = True,
technical = False
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
is_added = True
else:
module = ImportCache.getImportedModuleByName(
module.getFullName()
)
is_added = False
assert not module_relpath.endswith("/__init__.py"), module
return module, is_added
else:
return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
module, source_ref, source_filename = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib")
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
logRecursion("Recurse to import '%s' from '%s'. (%s)",
module.getFullName(), module_relpath, reason)
if module_kind == "py" and source_filename is not None:
try:
source_code = readSourceCodeFromFilename(
module_name=module.getFullName(),
source_filename=source_filename)
Building.createModuleTree(module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath,
module_filename, e.__class__.__name__)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code,
module_filename,
"exec",
dont_inherit=True)),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
is_added = True
else:
module = ImportCache.getImportedModuleByName(module.getFullName())
is_added = False
return module, is_added
def _detectedSourceFile(filename, module_name, result, user_provided,
technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename=filename,
module_name=ModuleName("_collections_abc"),
result=result,
user_provided=user_provided,
technical=technical,
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = "\n" + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
os.path.basename(filename),
source_code,
)
# Debian stretch site.py
source_code = source_code.replace(
"PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
is_package = os.path.basename(filename) == "__init__.py"
# Plugins can modify source code:
source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
is_package=is_package,
source_code=source_code)
bytecode = compileSourceToBytecode(
source_code=source_code,
filename=module_name.replace(".", os.path.sep) + ".py",
)
# Plugins can modify bytecode code:
bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
is_package=is_package,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(bytecode),
is_package=is_package,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex
debug(
"Checking detail plug-in path '%s' '%s':",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_filename = plugin_filename
)
if plugin_info is not None:
module, is_added = recurseTo(
module_filename = plugin_info[0],
module_relpath = plugin_info[1],
module_package = module_package,
module_kind = "py",
reason = "Lives in plug-in directory."
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_info[0]
)
if not isSameModulePath(module.getFilename(), plugin_info[0]):
warning(
"Duplicate ignored '%s'.",
plugin_info[1]
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module
)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if Utils.isDir(package_filename):
# Must be a namespace package.
assert python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = Utils.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug(
"Package directory %s",
package_dir
)
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
