def optimizeShlibModule(module):
# Pick up parent package if any.
_attemptRecursion(module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
Plugins.considerImplicitImports(module, signal_change = signalChange)
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
if not tag_set:
break
if graph is not None:
computation_counters[module] = computation_counters.get(module, 0) + 1
module_graph = module.asGraph(computation_counters[module])
graph.subgraph(module_graph)
touched = True
if _progress:
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def optimizeCompiledPythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
touched = False
if _progress and Options.isShowMemory():
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
Graphs.onModuleOptimizationStep(module)
# Search for local change tags.
for tag in tag_set:
if tag == "new_code":
continue
break
else:
break
# Otherwise we did stuff, so note that for return value.
touched = True
if _progress and Options.isShowMemory():
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def _createTriggerLoadedModule(module, trigger_name, code):
from nuitka.tree.Building import createModuleTree
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.plugins.Plugins import Plugins
module_name = module.getName() + trigger_name
source_ref = fromFilename(module.getCompileTimeFilename() + trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(
name = module_name,
package_name = module.getPackage(),
mode = mode,
source_ref = source_ref
)
createModuleTree(
module = trigger_module,
source_ref = module.getSourceReference(),
source_code = code,
is_main = False
)
return trigger_module
def addImportedModule(imported_module):
key = (
Utils.relpath(imported_module.getFilename()),
imported_module.getFullName()
)
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def _findModule(module_name):
if _debug_module_finding:
print(
"_findModule: Enter to search '%s'." % (
module_name,
)
)
assert not module_name.endswith('.'), module_name
key = module_name
if key in module_search_cache:
result = module_search_cache[key]
if _debug_module_finding:
print("_findModule: Cached result (see previous call).")
if result is ImportError:
raise ImportError
else:
return result
try:
module_search_cache[key] = _findModule2(module_name)
except ImportError:
new_module_name = Plugins.considerFailedImportReferrals(module_name)
if new_module_name is None:
module_search_cache[key] = ImportError
raise
else:
module_search_cache[key] = _findModule(new_module_name)
return module_search_cache[key]
def addImportedModule(imported_module):
module_filename = relpath(imported_module.getFilename())
if os.path.basename(module_filename) == "__init__.py":
module_filename = os.path.dirname(module_filename)
key = (module_filename, imported_module.getFullName())
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def demoteCompiledModuleToBytecode(module):
""" Demote a compiled module to uncompiled (bytecode).
"""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
source_code = readSourceCodeFromFilename(full_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(
module_name = full_name,
is_package = False,
source_code = source_code
)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = full_name,
is_package = False,
bytecode = bytecode
)
uncompiled_module = makeUncompiledPythonModule(
module_name = module.getFullName(),
filename = filename,
bytecode = marshal.dumps(bytecode),
is_package = module.isCompiledPythonPackage(),
user_provided = True,
technical = False
)
replaceImportedModule(
old = module,
new = uncompiled_module
)
replaceRootModule(
old = module,
new = uncompiled_module
)
assert module.constraint_collection is not None
uncompiled_module.setUsedModules(module.constraint_collection.getUsedModules())
def optimizeCompiledPythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()))
touched = False
if _progress and Options.isShowMemory():
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
Graphs.onModuleOptimizationStep(module)
# Search for local change tags.
for tag in tag_set:
if tag == "new_code":
continue
break
else:
break
# Otherwise we did stuff, so note that for return value.
touched = True
if _progress and Options.isShowMemory():
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
Plugins.considerImplicitImports(module, signal_change=signalChange)
return touched
def readSourceCodeFromFilename(module_name, source_filename):
if python_version < 300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plug-ins to mess with source code.
source_code = Plugins.onModuleSourceCode(module_name, source_code)
return source_code
def buildModuleTree(filename, package, is_top, is_main):
module, source_ref, source_filename = decideModuleTree(
filename=filename,
package=package,
is_top=is_top,
is_main=is_main,
is_shlib=False,
)
if is_top:
ModuleRegistry.addRootModule(module)
OutputDirectories.setMainModule(module)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
detectEarlyImports()
# If there is source code associated (not the case for namespace packages of
# Python3.3 or higher, then read it.
if source_filename is not None:
source_code = readSourceCodeFromFilename(
module_name=module.getFullName(), source_filename=source_filename)
if is_main:
checkPythonVersionFromCode(source_code)
# Read source code.
createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=is_main,
)
# Main modules do not get added to the import cache, but plugins get to see it.
if module.isMainModule():
Plugins.onModuleDiscovered(module)
else:
addImportedModule(imported_module=module)
return module
def readSourceCodeFromFilename(module_name, source_filename):
if python_version < 300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plug-ins to mess with source code.
source_code = Plugins.onModuleSourceCode(module_name, source_code)
return source_code
def decideCompilationMode(is_top, module_name, source_ref):
result = Plugins.decideCompilation(module_name, source_ref)
if result == "bytecode" and is_top:
plugins_logger.warning("""\
Ignoring plugin decision to compile top level package '%s'
as bytecode, the extension module entry point is technically
required to compiled.""" % module_name)
result = "compiled"
return result
def getModuleMetapathLoaderEntryCode(module, bytecode_accessor):
module_c_name = encodePythonStringToC(
Plugins.encodeDataComposerName(module.getFullName().asString()))
flags = ["NUITKA_TRANSLATED_FLAG"]
if not Options.isStandaloneMode() and python_version >= 0x370:
if Options.isWin32Windows():
file_path = encodePythonUnicodeToC(module.getCompileTimeFilename())
else:
file_path = encodePythonStringToC(
module.getCompileTimeFilename().encode(
sys.getfilesystemencoding()))
else:
file_path = "NULL"
if module.isUncompiledPythonModule():
code_data = module.getByteCode()
is_package = module.isUncompiledPythonPackage()
flags.append("NUITKA_BYTECODE_FLAG")
if is_package:
flags.append("NUITKA_PACKAGE_FLAG")
accessor_code = bytecode_accessor.getBlobDataCode(
data=code_data,
name="bytecode of module '%s'" % module.getFullName().asString(),
)
return template_metapath_loader_bytecode_module_entry % {
"module_name": module_c_name,
"bytecode": accessor_code[accessor_code.find("[") + 1:-1],
"size": len(code_data),
"flags": " | ".join(flags),
"file_path": file_path,
}
elif module.isPythonExtensionModule():
flags.append("NUITKA_EXTENSION_MODULE_FLAG")
return template_metapath_loader_extension_module_entry % {
"module_name": module_c_name,
"flags": " | ".join(flags),
"file_path": file_path,
}
else:
if module.isCompiledPythonPackage():
flags.append("NUITKA_PACKAGE_FLAG")
return template_metapath_loader_compiled_module_entry % {
"module_name": module_c_name,
"module_identifier": module.getCodeName(),
"flags": " | ".join(flags),
"file_path": file_path,
}
def decideRecursion(module_filename, module_name, module_kind, extra_recursion=False):
# Many branches, which make decisions immediately, by returning
# pylint: disable=too-many-return-statements
if module_name == "__main__":
return False, "Main program is not recursed to again."
plugin_decision = Plugins.onModuleEncounter(
module_filename, module_name, module_kind
)
if plugin_decision:
return plugin_decision
if module_kind == "shlib":
if Options.isStandaloneMode():
return True, "Shared library for inclusion."
else:
return False, "Shared library cannot be inspected."
no_case, reason = matchesModuleNameToPatterns(
module_name=module_name, patterns=Options.getShallFollowInNoCase()
)
if no_case:
return (False, "Module %s instructed by user to not recurse to." % reason)
any_case, reason = matchesModuleNameToPatterns(
module_name=module_name, patterns=Options.getShallFollowModules()
)
if any_case:
return (True, "Module %s instructed by user to recurse to." % reason)
if Options.shallFollowNoImports():
return (False, "Requested to not recurse at all.")
if StandardLibrary.isStandardLibraryPath(module_filename):
return (
Options.shallFollowStandardLibrary(),
"Requested to %srecurse to standard library."
% ("" if Options.shallFollowStandardLibrary() else "not "),
)
if Options.shallFollowAllImports():
return (True, "Requested to recurse to all non-standard library modules.")
# Means, we were not given instructions how to handle things.
if extra_recursion:
return (True, "Lives in plug-in directory.")
if Options.shallMakeModule():
return (False, "Making a module, not following any imports by default.")
return (None, "Default behavior, not recursing without request.")
def onModuleInitialSet(self):
from nuitka.importing.ImportCache import addImportedModule
from nuitka.ModuleRegistry import getRootTopModule
from nuitka.plugins.Plugins import Plugins
from nuitka.tree.Building import (
CompiledPythonModule,
createModuleTree,
readSourceCodeFromFilename,
)
# First, build the module node and then read again from the
# source code.
root_module = getRootTopModule()
module_name = ModuleName("__parents_main__")
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename(module_name,
root_module.getFilename())
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.
if python_version >= 0x340:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
addImportedModule(imported_module=slave_main_module)
yield slave_main_module
def readSourceCodeFromFilename(module_name, source_filename):
if python_version < 300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plug-ins to mess with source code, test framework usages
# will pass None for module name.
if module_name is not None:
source_code = Plugins.onModuleSourceCode(module_name, source_code)
return source_code
def optimizeUncompiledPythonModule(module):
if _progress:
info(
"Doing module dependency considerations for '{module_name}':".format(
module_name=module.getFullName()
)
)
for used_module_name, used_module_path in module.getUsedModules():
used_module = ImportCache.getImportedModuleByNameAndPath(
used_module_name, used_module_path
)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def optimizeUncompiledPythonModule(module):
full_name = module.getFullName()
progress_logger.info(
"Doing module dependency considerations for '{module_name}':".format(
module_name=full_name
)
)
for used_module_name, used_module_path in module.getUsedModules():
used_module = ImportCache.getImportedModuleByNameAndPath(
used_module_name, used_module_path
)
ModuleRegistry.addUsedModule(used_module)
package_name = full_name.getPackageName()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def addImportedModule(imported_module):
module_filename = relpath(imported_module.getFilename())
if os.path.basename(module_filename) == "__init__.py":
module_filename = os.path.dirname(module_filename)
key = (
module_filename,
imported_module.getFullName()
)
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import (
CompiledPythonModule,
readSourceCodeFromFilename,
createModuleTree,
)
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
from sys import hexversion
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name=module_name,
package_name=None,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename(
"__parents_main__", root_module.getFilename()
)
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.0.a2
if hexversion >= 0x030400A2:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import (
CompiledPythonModule,
readSourceCodeFromFilename,
createModuleTree,
)
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
from sys import hexversion
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name=module_name,
package_name=None,
is_top=False,
mode=mode,
future_spec=None,
source_ref=root_module.getSourceReference(),
)
source_code = readSourceCodeFromFilename("__parents_main__",
root_module.getFilename())
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace. The forking module was split up
# into multiple modules in Python 3.4.0.a2
if hexversion >= 0x030400A2:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.spawn").spawn.freeze_support()"""
else:
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""
createModuleTree(
module=slave_main_module,
source_ref=root_module.getSourceReference(),
source_code=source_code,
is_main=False,
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def warnAbout(importing, module_name, parent_package, level, tried_names):
# This probably should not be dealt with here, pylint: disable=too-many-branches
if module_name == "":
return
if not isIgnoreListedImportMaker(importing) and not isIgnoreListedNotExistingModule(
module_name
):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if _debug_module_finding:
if parent_package is not None:
recursion_logger.warning(
"%s: Cannot find '%s' in package '%s' %s (tried %s)."
% (
importing.getSourceReference().getAsString(),
module_name,
parent_package,
level_desc,
",".join(tried_names),
)
)
else:
recursion_logger.warning(
"%s: Cannot find '%s' %s."
% (
importing.getSourceReference().getAsString(),
module_name,
level_desc,
)
)
def demoteCompiledModuleToBytecode(module):
""" Demote a compiled module to uncompiled (bytecode).
"""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
debug("Demoting module '%s' to bytecode from '%s'.", full_name, filename)
source_code = readSourceCodeFromFilename(full_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name,
is_package=False,
source_code=source_code)
bytecode = compile(source_code, filename, "exec", dont_inherit=True)
bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name,
is_package=False,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=full_name,
filename=filename,
bytecode=marshal.dumps(bytecode),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
replaceImportedModule(old=module, new=uncompiled_module)
replaceRootModule(old=module, new=uncompiled_module)
assert module.trace_collection is not None
uncompiled_module.setUsedModules(module.trace_collection.getUsedModules())
def buildMainModuleTree(filename, is_main):
# Detect to be frozen modules if any, so we can consider to not follow
# to them.
Plugins.onBeforeCodeParsing()
if is_main:
# TODO: Doesn't work for deeply nested packages at all.
if Options.hasPythonFlagPackageMode():
module_name = ModuleName(os.path.basename(filename) + ".__main__")
else:
module_name = ModuleName("__main__")
else:
module_name = Importing.getModuleNameAndKindFromFilename(filename)[0]
module, _added = buildModule(
module_name=module_name,
module_filename=filename,
source_code=None,
is_top=True,
is_main=is_main,
is_extension=False,
is_fake=False,
hide_syntax_error=False,
)
if is_main and Options.isStandaloneMode():
module.setEarlyModules(detectEarlyImports())
# Main modules do not get added to the import cache, but plugins get to see it.
if module.isMainModule():
Plugins.onModuleDiscovered(module)
else:
addImportedModule(imported_module=module)
return module
def getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def _getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
package_dir = locateModule(module_name=package_name,
parent_package=None,
level=0)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def warnAbout(importing, module_name, parent_package, level, tried_names):
# This probably should not be dealt with here, pylint: disable=too-many-branches
if module_name == "":
return
if not isWhiteListedImport(importing) and not isWhiteListedNotExistingModule(
module_name
):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if _debug_module_finding:
if parent_package is not None:
warning(
"%s: Cannot find '%s' in package '%s' %s (tried %s).",
importing.getSourceReference().getAsString(),
module_name,
parent_package,
level_desc,
",".join(tried_names),
)
else:
warning(
"%s: Cannot find '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
level_desc,
)
def warnAbout(importing, module_name, level, source_ref):
# This probably should not be dealt with here
if module_name == "":
return
if not isIgnoreListedNotExistingModule(
module_name
) and not isIgnoreListedImportMaker(source_ref):
key = module_name, level
if key not in warned_about:
warned_about.add(key)
if Plugins.suppressUnknownImportWarning(
importing=importing, source_ref=source_ref, module_name=module_name
):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if _debug_module_finding:
if importing.getPackageName() is not None:
recursion_logger.warning(
"%s: Cannot find '%s' in package '%s' %s."
% (
importing.getSourceReference().getAsString(),
module_name,
importing.getPackageName().asString(),
level_desc,
)
)
else:
recursion_logger.warning(
"%s: Cannot find '%s' %s."
% (
importing.getSourceReference().getAsString(),
module_name,
level_desc,
)
)
def warnAbout(importing, module_name, parent_package, level):
# This probably should not be dealt with here.
if module_name == "":
return
if not isWhiteListedImport(importing) and \
not isWhiteListedNotExistingModule(module_name):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if parent_package is not None:
warning(
"%s: Cannot find '%s' in package '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
parent_package,
level_desc
)
else:
warning(
"%s: Cannot find '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
level_desc
)
def warnAbout(importing, module_name, parent_package, level):
# This probably should not be dealt with here.
if module_name == "":
return
if not isWhiteListedNotExistingModule(module_name):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if parent_package is not None:
warning(
"%s: Cannot find '%s' in package '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
parent_package,
level_desc
)
else:
warning(
"%s: Cannot find '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
level_desc
)
def _parsePEFileOutput(binary_filename, scan_dirs, result):
pe = _getPEFile(binary_filename)
# Some DLLs (eg numpy) don't have imports
if not hasattr(pe, "DIRECTORY_ENTRY_IMPORT"):
info(
"Warning: no DIRECTORY_ENTRY_IMPORT PE section for library '%s'!" %
binary_filename)
return
# Get DLL imports from PE file
for imported_module in pe.DIRECTORY_ENTRY_IMPORT:
dll_filename = imported_module.dll.decode()
# Try to guess DLL path from scan dirs
for scan_dir in scan_dirs:
try:
guessed_path = os.path.join(scan_dir, dll_filename)
if os.path.isfile(guessed_path):
dll_filename = guessed_path
break
except TypeError:
pass
dll_name = os.path.basename(dll_filename).upper()
# Win API can be assumed.
if dll_name.startswith("API-MS-WIN-") or dll_name.startswith(
"EXT-MS-WIN-"):
continue
if dll_name in _win_dll_whitelist:
continue
# Allow plugins to prevent inclusion.
blocked = Plugins.removeDllDependencies(dll_filename=dll_filename,
dll_filenames=result)
for to_remove in blocked:
result.discard(to_remove)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
def getModuleMetapathLoaderEntryCode(module, bytecode_accessor):
module_c_name = encodePythonStringToC(
Plugins.encodeDataComposerName(module.getFullName().asString()))
flags = ["NUITKA_TRANSLATED_FLAG"]
if module.isUncompiledPythonModule():
code_data = module.getByteCode()
is_package = module.isUncompiledPythonPackage()
flags.append("NUITKA_BYTECODE_FLAG")
if is_package:
flags.append("NUITKA_PACKAGE_FLAG")
accessor_code = bytecode_accessor.getBlobDataCode(
data=code_data,
name="bytecode of module '%s'" % module.getFullName().asString(),
)
return template_metapath_loader_bytecode_module_entry % {
"module_name": module_c_name,
"bytecode": accessor_code[accessor_code.find("[") + 1:-1],
"size": len(code_data),
"flags": " | ".join(flags) or "0",
}
elif module.isPythonExtensionModule():
flags.append("NUITKA_EXTENSION_MODULE_FLAG")
return template_metapath_loader_extension_module_entry % {
"module_name": module_c_name,
"flags": " | ".join(flags) or "0",
}
else:
if module.isCompiledPythonPackage():
flags.append("NUITKA_PACKAGE_FLAG")
return template_metapath_loader_compiled_module_entry % {
"module_name": module_c_name,
"module_identifier": module.getCodeName(),
"flags": " | ".join(flags),
}
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name = module_name,
package_name = None,
mode = mode,
source_ref = root_module.getSourceReference()
)
source_code = readSourceCodeFromFilename(
"__parents_main__",
root_module.getFilename()
)
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace.
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.main()"""
createModuleTree(
module = slave_main_module,
source_ref = root_module.getSourceReference(),
source_code = source_code,
is_main = False
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def readSourceCodeFromFilename(module_name, source_filename):
if python_version < 0x300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plugins to mess with source code. Test code calls this
# without a module and doesn't want changes from plugins.
if module_name is not None:
source_code_modified = Plugins.onModuleSourceCode(module_name, source_code)
else:
source_code_modified = source_code
if Options.shallPersistModifications() and source_code_modified != source_code:
orig_source_filename = source_filename + ".orig"
if not os.path.exists(orig_source_filename):
putTextFileContents(filename=orig_source_filename, contents=source_code)
putTextFileContents(filename=source_filename, contents=source_code_modified)
return source_code_modified
def _addSlaveMainModule(root_module):
from nuitka.tree.Building import CompiledPythonModule, readSourceCodeFromFilename, createModuleTree
from nuitka.ModuleRegistry import addRootModule
from nuitka.plugins.Plugins import Plugins
# First, build the module node and then read again from the
# source code.
module_name = "__parents_main__"
source_ref = root_module.getSourceReference()
mode = Plugins.decideCompilation(module_name, source_ref)
slave_main_module = CompiledPythonModule(
name = module_name,
package_name = None,
mode = mode,
source_ref = root_module.getSourceReference()
)
source_code = readSourceCodeFromFilename(
"__parents_main__",
root_module.getFilename()
)
# For the call stack, this may look bad or different to what
# CPython does. Using the "__import__" built-in to not spoil
# or use the module namespace.
source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.main()"""
createModuleTree(
module = slave_main_module,
source_ref = root_module.getSourceReference(),
source_code = source_code,
is_main = False
)
# This is an alternative entry point of course.
addRootModule(slave_main_module)
def _createTriggerLoadedModule(module, trigger_name, code):
from nuitka.tree.Building import createModuleTree
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.plugins.Plugins import Plugins
module_name = module.getName() + trigger_name
source_ref = fromFilename(module.getCompileTimeFilename() +
trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(name=module_name,
package_name=module.getPackage(),
mode=mode,
source_ref=source_ref)
createModuleTree(module=trigger_module,
source_ref=module.getSourceReference(),
source_code=code,
is_main=False)
return trigger_module
def decideCompilationMode(is_top, module_name, for_pgo):
"""Decide the compilation mode for a module.
module_name - The module to decide compilation mode for.
for_pgo - consider PGO information or not
"""
result = Plugins.decideCompilation(module_name)
# Cannot change mode of __main__ to bytecode, that is not going
# to work currently.
if result == "bytecode" and is_top:
plugins_logger.warning("""\
Ignoring plugin decision to compile top level package '%s'
as bytecode, the extension module entry point is technically
required to compiled.""" % module_name)
result = "compiled"
# Include all of standard library as bytecode, for now. We need to identify
# which ones really need that.
if not is_top:
module_filename = Importing.locateModule(module_name=module_name,
parent_package=None,
level=0)[1]
if module_filename is not None and isStandardLibraryPath(
module_filename):
result = "bytecode"
# Plugins need to win over PGO, as they might know it better
if result is None and not for_pgo:
result = decideCompilationFromPGO(module_name=module_name)
# Default if neither plugins nor PGO have expressed an opinion
if result is None:
result = "compiled"
return result
def _onEnterNode(self, node):
# This has many different things it deals with, so there need to be a
# lot of branches and statements, pylint: disable=too-many-branches,too-many-statements
# Also all self specific things have been done on the outside,
# pylint: disable=no-self-use
if node.isStatementReturn() or node.isStatementGeneratorReturn():
search = node
in_tried_block = False
# Search up to the containing function, and check for a try/finally
# containing the "return" statement.
search = search.getParentReturnConsumer()
if (
search.isExpressionGeneratorObjectBody()
or search.isExpressionCoroutineObjectBody()
or search.isExpressionAsyncgenObjectBody()
):
if in_tried_block:
search.markAsNeedsGeneratorReturnHandling(2)
else:
search.markAsNeedsGeneratorReturnHandling(1)
if (
node.isExpressionBuiltinImport()
and not Options.getShallFollowExtra()
and not Options.getShallFollowExtraFilePatterns()
and not Options.shallFollowNoImports()
and not isWhiteListedImport(node)
and not node.recurse_attempted
and not Plugins.suppressBuiltinImportWarning(
node.getParentModule(), node.getSourceReference()
)
):
warning(
"""Unresolved '__import__' call at '%s' may require use \
of '--include-plugin-directory' or '--include-plugin-files'."""
% (node.getSourceReference().getAsString())
)
if node.isExpressionBuiltinImport() and node.recurse_attempted:
module_name = node.getImportName()
if module_name.isCompileTimeConstant():
imported_module_name = module_name.getCompileTimeConstant()
if type(imported_module_name) in (str, unicode):
if imported_module_name:
imported_names.add(imported_module_name)
if node.isExpressionFunctionCreation():
if (
not node.getParent().isExpressionFunctionCall()
or node.getParent().getFunction() is not node
):
node.getFunctionRef().getFunctionBody().markAsNeedsCreation()
if node.isExpressionFunctionCall():
node.getFunction().getFunctionRef().getFunctionBody().markAsDirectlyCalled()
if node.isExpressionFunctionRef():
function_body = node.getFunctionBody()
parent_module = function_body.getParentModule()
node_module = node.getParentModule()
if node_module is not parent_module:
function_body.markAsCrossModuleUsed()
node_module.addCrossUsedFunction(function_body)
if node.isStatementAssignmentVariable():
target_var = node.getVariable()
assign_source = node.getAssignSource()
if assign_source.isExpressionOperationBinary():
left_arg = assign_source.getLeft()
if left_arg.isExpressionVariableRef():
if assign_source.getLeft().getVariable() is target_var:
if assign_source.isInplaceSuspect():
node.markAsInplaceSuspect()
elif left_arg.isExpressionLocalsVariableRefORFallback():
assign_source.unmarkAsInplaceSuspect()
if node.isStatementLocalsDictOperationSet():
assign_source = node.getAssignSource()
if assign_source.isExpressionOperationBinary():
assign_source.unmarkAsInplaceSuspect()
if python_version < 300 and node.isStatementPublishException():
node.getParentStatementsFrame().markAsFrameExceptionPreserving()
if python_version >= 300:
if (
node.isExpressionYield()
or node.isExpressionYieldFrom()
or node.isExpressionYieldFromWaitable()
):
search = node.getParent()
while (
not search.isExpressionGeneratorObjectBody()
and not search.isExpressionCoroutineObjectBody()
and not search.isExpressionAsyncgenObjectBody()
):
last_search = search
search = search.getParent()
if (
search.isStatementTry()
and last_search == search.getBlockExceptHandler()
):
node.markAsExceptionPreserving()
break
def _detectedSourceFile(filename, module_name, result, user_provided,
technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename=filename,
module_name=ModuleName("_collections_abc"),
result=result,
user_provided=user_provided,
technical=technical,
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = "\n" + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
os.path.basename(filename),
source_code,
)
# Debian stretch site.py
source_code = source_code.replace(
"PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")
# Anaconda3 4.1.2 site.py
source_code = source_code.replace(
"def main():", "def main():return\n\nif 0:\n def _unused():")
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
is_package = os.path.basename(filename) == "__init__.py"
# Plugins can modify source code:
source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
is_package=is_package,
source_code=source_code)
bytecode = compileSourceToBytecode(
source_code=source_code,
filename=module_name.replace(".", os.path.sep) + ".py",
)
# Plugins can modify bytecode code:
bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
is_package=is_package,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(bytecode),
is_package=is_package,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def copyDataFiles(dist_dir):
"""Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Many details to deal with, pylint: disable=too-many-branches,too-many-locals
for pattern, dest, arg in Options.getShallIncludeDataFiles():
filenames = resolveShellPatternToFilenames(pattern)
if not filenames:
inclusion_logger.warning("No match data file to be included: %r" %
pattern)
for filename in filenames:
file_reason = "specified data file %r on command line" % arg
rel_path = dest
if rel_path.endswith(("/", os.path.sep)):
rel_path = os.path.join(rel_path, os.path.basename(filename))
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
for src, dest in Options.getShallIncludeDataDirs():
filenames = getFileList(src)
if not filenames:
inclusion_logger.warning("No files in directory" % src)
for filename in filenames:
relative_filename = relpath(filename, src)
file_reason = "specified data dir %r on command line" % src
rel_path = os.path.join(dest, relative_filename)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for plugin, included_datafile in Plugins.considerDataFiles(module):
_handleDataFile(dist_dir=dist_dir,
tracer=plugin,
included_datafile=included_datafile)
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=Options.getShallIncludePackageData())
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(pkg_filename, rel_path,
file_reason),
)
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements
assert package is None or type(package) is str
assert filename is not None
if is_main and os.path.isdir(filename):
source_filename = os.path.join(filename, "__main__.py")
if not os.path.isfile(source_filename):
sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
(os.path.basename(sys.argv[0]), filename))
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if os.path.isfile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(filename=filename, )
if is_main:
module_name = "__main__"
else:
module_name = os.path.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
if is_shlib:
module_name = module_name.split('.')[0]
if '.' in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" %
(module_name))
sys.exit(2)
if is_shlib:
result = PythonShlibModule(name=module_name,
package_name=package,
source_ref=source_ref)
elif is_main:
result = PythonMainModule(main_added=main_added,
mode=Plugins.decideCompilation(
"__main__", source_ref),
future_spec=None,
source_ref=source_ref)
else:
if package is not None:
full_name = package + '.' + module_name
else:
full_name = module_name
result = CompiledPythonModule(name=module_name,
package_name=package,
mode=Plugins.decideCompilation(
full_name, source_ref),
future_spec=None,
source_ref=source_ref)
elif Importing.isPackageDir(filename):
if is_top:
module_name = splitPath(filename)[-1]
else:
module_name = os.path.basename(filename)
source_filename = os.path.join(filename, "__init__.py")
if not os.path.isfile(source_filename):
source_ref, result = createNamespacePackage(
module_name=module_name,
package_name=package,
module_relpath=filename)
source_filename = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename=os.path.abspath(source_filename), )
if package is not None:
full_name = package + '.' + module_name
else:
full_name = module_name
result = CompiledPythonPackage(name=module_name,
package_name=package,
mode=Plugins.decideCompilation(
full_name, source_ref),
future_spec=None,
source_ref=source_ref)
assert result.getFullName() == full_name, result
else:
sys.stderr.write("%s: can't open file '%s'.\n" %
(os.path.basename(sys.argv[0]), filename))
sys.exit(2)
if not Options.shallHaveStatementLines():
source_ref = source_ref.atInternal()
return result, source_ref, source_filename
def setCommonOptions(options):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches
if Options.shallRunInDebugger():
options["full_names"] = "true"
if Options.assumeYesForDownloads():
options["assume_yes_for_downloads"] = asBoolStr(True)
if not Options.shallUseProgressBar():
options["progress_bar"] = "false"
if Options.isClang():
options["clang_mode"] = "true"
if Options.isShowScons():
options["show_scons"] = "true"
if Options.isMingw64():
options["mingw_mode"] = "true"
if Options.getMsvcVersion():
options["msvc_version"] = Options.getMsvcVersion()
if Options.shallDisableCCacheUsage():
options["disable_ccache"] = asBoolStr(True)
if Options.shallDisableConsoleWindow():
options["disable_console"] = asBoolStr(True)
if Options.getLtoMode() != "auto":
options["lto_mode"] = Options.getLtoMode()
cpp_defines = Plugins.getPreprocessorSymbols()
if cpp_defines:
options["cpp_defines"] = ",".join(
"%s%s%s" % (key, "=" if value else "", value or "")
for key, value in cpp_defines.items())
cpp_include_dirs = Plugins.getExtraIncludeDirectories()
if cpp_include_dirs:
options["cpp_include_dirs"] = ",".join(cpp_include_dirs)
link_dirs = Plugins.getExtraLinkDirectories()
if link_dirs:
options["link_dirs"] = ",".join(link_dirs)
link_libraries = Plugins.getExtraLinkLibraries()
if link_libraries:
options["link_libraries"] = ",".join(link_libraries)
if Utils.isMacOS():
macos_min_version = detectBinaryMinMacOS(sys.executable)
if macos_min_version is None:
Tracing.general.sysexit(
"Could not detect minimum macOS version for %r." %
sys.executable)
options["macos_min_version"] = macos_min_version
macos_target_arch = Options.getMacOSTargetArch()
if macos_target_arch == "universal":
Tracing.general.sysexit(
"Cannot create universal macOS binaries (yet), please pick an arch and create two binaries."
)
options["macos_target_arch"] = macos_target_arch
def decideRecursion(
module_filename, module_name, module_package, module_kind, extra_recursion=False
):
# Many branches, which make decisions immediately, by returning
# pylint: disable=too-many-branches,too-many-return-statements
if module_name == "__main__":
return False, "Main program is not recursed to again."
plugin_decision = Plugins.onModuleEncounter(
module_filename, module_name, module_package, module_kind
)
if plugin_decision:
return plugin_decision
if module_kind == "shlib":
if Options.isStandaloneMode():
return True, "Shared library for inclusion."
else:
return False, "Shared library cannot be inspected."
if module_package is None:
full_name = module_name
else:
full_name = module_package + "." + module_name
no_case, reason = matchesModuleNameToPatterns(
module_name=full_name, patterns=Options.getShallFollowInNoCase()
)
if no_case:
return (False, "Module %s instructed by user to not recurse to." % reason)
any_case, reason = matchesModuleNameToPatterns(
module_name=full_name, patterns=Options.getShallFollowModules()
)
if any_case:
return (True, "Module %s instructed by user to recurse to." % reason)
if Options.shallFollowNoImports():
return (False, "Requested to not recurse at all.")
if StandardLibrary.isStandardLibraryPath(module_filename):
return (
Options.shallFollowStandardLibrary(),
"Requested to %srecurse to standard library."
% ("" if Options.shallFollowStandardLibrary() else "not "),
)
if Options.shallFollowAllImports():
return (True, "Requested to recurse to all non-standard library modules.")
# Means, we were not given instructions how to handle things.
if extra_recursion:
return (True, "Lives in plug-in directory.")
if Options.shallMakeModule():
return (False, "Making a module, not following any imports by default.")
return (None, "Default behavior, not recursing without request.")
def _onEnterNode(self, node):
# This has many different things it deals with, so there need to be a
# lot of branches and statements, pylint: disable=R0912,R0915
# Also all self specific things have been done on the outside,
# pylint: disable=R0201
# Find nodes with only compile time constant children, these are
# missing some obvious optimization potentially.
if False: # For searching only, pylint: disable=W0125
if not node.isStatementReturn() and \
not node.isExpressionYield() and \
not node.isStatementRaiseException() and \
not node.isExpressionCall() and \
not node.isExpressionBuiltinIter1():
children = node.getVisitableNodes()
if children:
for child in children:
if child.isStatement() or child.isStatementsSequence():
break
if not child.isCompileTimeConstant():
break
else:
assert False, (node, node.parent, children)
if node.isExpressionFunctionBody():
if node.isUnoptimized():
node.markAsLocalsDict()
if node.needsLocalsDict():
provider = node.getParentVariableProvider()
if not provider.isCompiledPythonModule():
provider.markAsLocalsDict()
if node.isStatementReturn() or node.isStatementGeneratorReturn():
search = node
in_tried_block = False
# Search up to the containing function, and check for a try/finally
# containing the "return" statement.
search = search.getParentReturnConsumer()
if search.isExpressionGeneratorObjectBody() or \
search.isExpressionCoroutineObjectBody():
if in_tried_block:
search.markAsNeedsGeneratorReturnHandling(2)
else:
search.markAsNeedsGeneratorReturnHandling(1)
if node.isExpressionBuiltinImport() and \
not Options.getShallFollowExtra() and \
not Options.getShallFollowExtraFilePatterns() and \
not Options.shallFollowNoImports() and \
not isWhiteListedImport(node) and \
not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()):
warning("""Unresolved '__import__' call at '%s' may require use \
of '--recurse-directory'.""" % (
node.getSourceReference().getAsString()
)
)
if node.isExpressionFunctionCreation():
if not node.getParent().isExpressionFunctionCall() or \
node.getParent().getFunction() is not node:
node.getFunctionRef().getFunctionBody().markAsNeedsCreation()
if node.isExpressionFunctionCall():
node.getFunction().getFunctionRef().getFunctionBody().\
markAsDirectlyCalled()
if node.isExpressionFunctionRef():
function_body = node.getFunctionBody()
parent_module = function_body.getParentModule()
node_module = node.getParentModule()
if node_module is not parent_module:
function_body.markAsCrossModuleUsed()
node_module.addCrossUsedFunction(function_body)
if node.isStatementAssignmentVariable():
target_var = node.getTargetVariableRef().getVariable()
assign_source = node.getAssignSource()
if assign_source.isExpressionOperationBinary():
left_arg = assign_source.getLeft()
if left_arg.isExpressionVariableRef():
if assign_source.getLeft().getVariable().isModuleVariable():
assign_source.unmarkAsInplaceSuspect()
elif assign_source.getLeft().getVariable() is target_var:
if assign_source.isInplaceSuspect():
node.markAsInplaceSuspect()
if node.isStatementPublishException():
node.getParentStatementsFrame().markAsFrameExceptionPreserving()
if python_version >= 300:
if node.isExpressionYield() or node.isExpressionYieldFrom():
search = node.getParent()
while not search.isExpressionGeneratorObjectBody():
last_search = search
search = search.getParent()
if search.isStatementTry() and \
last_search == search.getBlockExceptHandler():
node.markAsExceptionPreserving()
break
def decideRecursion(module_filename, module_name, module_package, module_kind):
# Many branches, which make decisions immediately, by returning
# pylint: disable=R0911,R0912
Plugins.onModuleEncounter(
module_filename,
module_name,
module_package,
module_kind
)
if module_kind == "shlib":
if Options.isStandaloneMode():
return True, "Shared library for inclusion."
else:
return False, "Shared library cannot be inspected."
if module_package is None:
full_name = module_name
else:
full_name = module_package + '.' + module_name
no_case_modules = Options.getShallFollowInNoCase()
for no_case_module in no_case_modules:
if full_name == no_case_module:
return (
False,
"Module listed explicitly to not recurse to."
)
if full_name.startswith(no_case_module + '.'):
return (
False,
"Module in package listed explicitly to not recurse to."
)
any_case_modules = Options.getShallFollowModules()
for any_case_module in any_case_modules:
if full_name == any_case_module:
return (
True,
"Module listed explicitly to recurse to."
)
if full_name.startswith(any_case_module + '.'):
return (
True,
"Module in package listed explicitly to recurse to."
)
if Options.shallFollowNoImports():
return (
False,
"Requested to not recurse at all."
)
if StandardLibrary.isStandardLibraryPath(module_filename):
return (
Options.shallFollowStandardLibrary(),
"Requested to %srecurse to standard library." % (
"" if Options.shallFollowStandardLibrary() else "not "
)
)
if Options.shallFollowAllImports():
return (
True,
"Requested to recurse to all non-standard library modules."
)
# Means, we were not given instructions how to handle things.
return (
None,
"Default behavior, not recursing without request."
)
def _detectedSourceFile(filename, module_name, result, user_provided, technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename = filename,
module_name = "_collections_abc",
result = result,
user_provided = user_provided,
technical = technical
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
Utils.basename(filename),
source_code
)
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
is_package = Utils.basename(filename) == "__init__.py"
source_code = Plugins.onFrozenModuleSourceCode(
module_name = module_name,
is_package = is_package,
source_code = source_code
)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = module_name,
is_package = is_package,
bytecode = bytecode
)
result.append(
makeUncompiledPythonModule(
module_name = module_name,
bytecode = marshal.dumps(
bytecode
),
is_package = is_package,
filename = filename,
user_provided = user_provided,
technical = technical
)
)
module_names.add(module_name)
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases, pylint: disable=R0912,R0915
assert package is None or type(package) is str
assert filename is not None
if is_main and Utils.isDir(filename):
source_filename = Utils.joinpath(filename, "__main__.py")
if not Utils.isFile(source_filename):
sys.stderr.write(
"%s: can't find '__main__' module in '%s'\n" % (
Utils.basename(sys.argv[0]),
filename
)
)
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if Utils.isFile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(
filename = filename,
)
if is_main:
module_name = "__main__"
else:
module_name = Utils.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
if is_shlib:
module_name = module_name.split('.')[0]
if '.' in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" % (
module_name
)
)
sys.exit(2)
if is_shlib:
result = PythonShlibModule(
name = module_name,
package_name = package,
source_ref = source_ref
)
elif is_main:
result = PythonMainModule(
main_added = main_added,
source_ref = source_ref
)
else:
if package is not None:
full_name = package + "." + module_name
else:
full_name = module_name
decision = Plugins.decideCompilation(full_name, source_ref)
if decision == "compiled":
result = CompiledPythonModule(
name = module_name,
package_name = package,
source_ref = source_ref
)
else:
source_code = readSourceCodeFromFilename(module_name, filename)
source_code = Plugins.onFrozenModuleSourceCode(
module_name = full_name,
is_package = False,
source_code = source_code
)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = module_name,
is_package = False,
bytecode = bytecode
)
result = UncompiledPythonModule(
name = module_name,
package_name = package,
bytecode = bytecode,
filename = filename,
user_provided = True,
technical = False,
source_ref = source_ref
)
# Don't read it anymore.
source_filename = None
elif Importing.isPackageDir(filename):
if is_top:
package_name = Utils.splitpath(filename)[-1]
else:
package_name = Utils.basename(filename)
source_filename = Utils.joinpath(filename, "__init__.py")
if not Utils.isFile(source_filename):
source_ref, result = createNamespacePackage(
package_name = package_name,
module_relpath = filename
)
source_filename = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename = Utils.abspath(source_filename),
)
if package is not None:
full_name = package + "." + package_name
else:
full_name = package_name
decision = Plugins.decideCompilation(full_name, source_ref)
if decision == "compiled":
result = CompiledPythonPackage(
name = package_name,
package_name = package,
source_ref = source_ref
)
else:
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = module_name,
is_package = False,
bytecode = bytecode
)
result = UncompiledPythonPackage(
name = module_name,
package_name = package,
bytecode = bytecode,
filename = filename,
user_provided = True,
technical = False,
source_ref = source_ref
)
# Don't read it anymore.
source_filename = None
else:
sys.stderr.write(
"%s: can't open file '%s'.\n" % (
Utils.basename(sys.argv[0]),
filename
)
)
sys.exit(2)
if not Options.shallHaveStatementLines():
source_ref = source_ref.atInternal()
return result, source_ref, source_filename
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing(getResultFullpath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(0, (binary_filename, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
source_dir=getSourceDirectoryPath(main_module),
dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points,
)
data_files = []
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
copyDataFiles(dist_dir=dist_dir, data_files=data_files)
Plugins.onStandaloneDistributionFinished(dist_dir)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(
path=getSourceDirectoryPath(main_module), ignore_errors=False
)
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, "w") as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
"""
% {
"imports": "\n".join(
"import %s" % module_name
for module_name in getImportedNames()
)
}
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment(),
)
def _onEnterNode(self, node):
# This has many different things it deals with, so there need to be a
# lot of branches and statements, pylint: disable=R0912,R0915
# Also all self specific things have been done on the outside,
# pylint: disable=R0201
# Find nodes with only compile time constant children, these are
# missing some obvious optimization potentially.
if False: # For searching only, pylint: disable=W0125
if not node.isStatementReturn() and \
not node.isExpressionYield() and \
not node.isStatementRaiseException() and \
not node.isExpressionCall() and \
not node.isExpressionBuiltinIter1():
children = node.getVisitableNodes()
if children:
for child in children:
if child.isStatement() or child.isStatementsSequence():
break
if not child.isCompileTimeConstant():
break
else:
assert False, (node, node.parent, children)
if node.isExpressionFunctionBody():
if node.isUnoptimized():
node.markAsLocalsDict()
if node.needsLocalsDict():
provider = node.getParentVariableProvider()
if not provider.isCompiledPythonModule():
provider.markAsLocalsDict()
if node.isStatementReturn() or node.isStatementGeneratorReturn():
search = node
in_tried_block = False
# Search up to the containing function, and check for a try/finally
# containing the "return" statement.
search = search.getParentReturnConsumer()
if search.isExpressionGeneratorObjectBody() or \
search.isExpressionCoroutineObjectBody():
if in_tried_block:
search.markAsNeedsGeneratorReturnHandling(2)
else:
search.markAsNeedsGeneratorReturnHandling(1)
if node.isExpressionBuiltinImport() and \
not Options.getShallFollowExtra() and \
not Options.getShallFollowExtraFilePatterns() and \
not Options.shallFollowNoImports() and \
not isWhiteListedImport(node) and \
not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()):
warning("""Unresolved '__import__' call at '%s' may require use \
of '--recurse-directory'.""" % (node.getSourceReference().getAsString()))
if node.isExpressionFunctionCreation():
if not node.getParent().isExpressionFunctionCall() or \
node.getParent().getFunction() is not node:
node.getFunctionRef().getFunctionBody().markAsNeedsCreation()
if node.isExpressionFunctionCall():
node.getFunction().getFunctionRef().getFunctionBody().\
markAsDirectlyCalled()
if node.isExpressionFunctionRef():
function_body = node.getFunctionBody()
parent_module = function_body.getParentModule()
node_module = node.getParentModule()
if node_module is not parent_module:
function_body.markAsCrossModuleUsed()
node_module.addCrossUsedFunction(function_body)
if node.isStatementAssignmentVariable():
target_var = node.getTargetVariableRef().getVariable()
assign_source = node.getAssignSource()
if assign_source.isExpressionOperationBinary():
left_arg = assign_source.getLeft()
if left_arg.isExpressionVariableRef():
if assign_source.getLeft().getVariable().isModuleVariable(
):
assign_source.unmarkAsInplaceSuspect()
elif assign_source.getLeft().getVariable() is target_var:
if assign_source.isInplaceSuspect():
node.markAsInplaceSuspect()
if node.isStatementPublishException():
node.getParentStatementsFrame().markAsFrameExceptionPreserving()
if python_version >= 300:
if node.isExpressionYield() or node.isExpressionYieldFrom():
search = node.getParent()
while not search.isExpressionGeneratorObjectBody():
last_search = search
search = search.getParent()
if search.isStatementTry() and \
last_search == search.getBlockExceptHandler():
node.markAsExceptionPreserving()
break
def optimizeShlibModule(module):
# Pick up parent package if any.
_attemptRecursion(module)
Plugins.considerImplicitImports(module, signal_change = signalChange)
def _detectedSourceFile(filename, module_name, result, user_provided, technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename = filename,
module_name = "_collections_abc",
result = result,
user_provided = user_provided,
technical = technical
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
Utils.basename(filename),
source_code
)
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
is_package = Utils.basename(filename) == "__init__.py"
source_code = Plugins.onFrozenModuleSourceCode(
module_name = module_name,
is_package = is_package,
source_code = source_code
)
bytecode = compile(source_code, filename, "exec")
bytecode = Plugins.onFrozenModuleBytecode(
module_name = module_name,
is_package = is_package,
bytecode = bytecode
)
result.append(
makeUncompiledPythonModule(
module_name = module_name,
bytecode = marshal.dumps(
bytecode
),
is_package = is_package,
filename = filename,
user_provided = user_provided,
technical = technical
)
)
module_names.add(module_name)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(main_dir=Utils.dirname(Utils.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()), "orig-prefix.txt"
)
if Utils.isFile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(SyntaxErrors.formatOutput(e))
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(getSourceDirectoryPath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir, standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = Utils.joinpath(getStandaloneDirectoryPath(main_module), target_filename)
Utils.makePath(Utils.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(tree=main_module, clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module), clean_path=Options.shallClearPythonPathEnvironment()
)
