def optimizeShlibModule(module):
# Pick up parent package if any.
_attemptRecursion(module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
Plugins.considerImplicitImports(module, signal_change = signalChange)
def addImportedModule(imported_module):
key = (Utils.relpath(imported_module.getFilename()), imported_module.getFullName())
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
if not tag_set:
break
if graph is not None:
computation_counters[module] = computation_counters.get(module, 0) + 1
module_graph = module.asGraph(computation_counters[module])
graph.subgraph(module_graph)
touched = True
if _progress:
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def decideRecursion(module_filename, module_name, module_package, module_kind):
# Many branches, which make decisions immediately, by returning
# pylint: disable=R0911,R0912
Plugins.onModuleEncounter(module_filename, module_name, module_package,
module_kind)
if module_kind == "shlib":
if Options.isStandaloneMode():
return True, "Shared library for inclusion."
else:
return False, "Shared library cannot be inspected."
if module_package is None:
full_name = module_name
else:
full_name = module_package + '.' + module_name
no_case_modules = Options.getShallFollowInNoCase()
for no_case_module in no_case_modules:
if full_name == no_case_module:
return (False, "Module listed explicitly to not recurse to.")
if full_name.startswith(no_case_module + '.'):
return (False,
"Module in package listed explicitly to not recurse to.")
any_case_modules = Options.getShallFollowModules()
for any_case_module in any_case_modules:
if full_name == any_case_module:
return (True, "Module listed explicitly to recurse to.")
if full_name.startswith(any_case_module + '.'):
return (True, "Module in package listed explicitly to recurse to.")
if Options.shallFollowNoImports():
return (False, "Requested to not recurse at all.")
if StandardLibrary.isStandardLibraryPath(module_filename):
return (Options.shallFollowStandardLibrary(),
"Requested to %srecurse to standard library." %
("" if Options.shallFollowStandardLibrary() else "not "))
if Options.shallFollowAllImports():
return (True,
"Requested to recurse to all non-standard library modules.")
# Means, we were not given instructions how to handle things.
return (None, "Default behavior, not recursing without request.")
def addImportedModule(module_relpath, imported_module):
if (module_relpath, "__main__") in imported_modules:
warning("""\
Re-importing __main__ module via its filename duplicates the module.""")
key = module_relpath, imported_module.getFullName()
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
def addImportedModule(module_relpath, imported_module):
if (module_relpath, "__main__") in imported_modules:
warning("""\
Re-importing __main__ module via its filename duplicates the module.""")
key = module_relpath, imported_module.getFullName()
if key in imported_modules:
assert imported_module is imported_modules[ key ], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[ key ] = imported_module
imported_by_name[ imported_module.getFullName() ] = imported_module
def addImportedModule(imported_module):
key = (Utils.relpath(imported_module.getFilename()),
imported_module.getFullName())
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def _findModule(module_name):
if _debug_module_finding:
print(
"_findModule: Enter to search '%s'." % (
module_name,
)
)
assert not module_name.endswith('.'), module_name
key = module_name
if key in module_search_cache:
result = module_search_cache[key]
if _debug_module_finding:
print("_findModule: Cached result (see previous call).")
if result is ImportError:
raise ImportError
else:
return result
try:
module_search_cache[key] = _findModule2(module_name)
except ImportError:
new_module_name = Plugins.considerFailedImportReferrals(module_name)
if new_module_name is None:
module_search_cache[key] = ImportError
raise
else:
module_search_cache[key] = _findModule(new_module_name)
return module_search_cache[key]
def _findModule(module_name):
if _debug_module_finding:
print("_findModule: Enter to search '%s'." % (module_name, ))
assert not module_name.endswith('.'), module_name
key = module_name
if key in module_search_cache:
result = module_search_cache[key]
if _debug_module_finding:
print("_findModule: Cached result (see previous call).")
if result is ImportError:
raise ImportError
else:
return result
try:
module_search_cache[key] = _findModule2(module_name)
except ImportError:
new_module_name = Plugins.considerFailedImportReferrals(module_name)
if new_module_name is None:
module_search_cache[key] = ImportError
raise
else:
module_search_cache[key] = _findModule(new_module_name)
return module_search_cache[key]
def readSourceCodeFromFilename(module_name, source_filename):
if Utils.python_version < 300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plug-ins to mess with source code.
source_code = Plugins.onModuleSourceCode(module_name, source_code)
return source_code
def readSourceCodeFromFilename(module_name, source_filename):
if Utils.python_version < 300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plug-ins to mess with source code.
source_code = Plugins.onModuleSourceCode(module_name, source_code)
return source_code
def warnAbout(importing, module_name, parent_package, level):
# This probably should not be dealt with here.
if module_name == "":
return
if not isWhiteListedNotExistingModule(module_name):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if parent_package is not None:
warning(
"%s: Cannot find '%s' in package '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
parent_package,
level_desc
)
else:
warning(
"%s: Cannot find '%s' %s.",
importing.getSourceReference().getAsString(),
module_name,
level_desc
)
def warnAbout(importing, module_name, parent_package, level):
# This probably should not be dealt with here.
if module_name == "":
return
if not isWhiteListedNotExistingModule(module_name):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if parent_package is not None:
warning("%s: Cannot find '%s' in package '%s' %s.",
importing.getSourceReference().getAsString(),
module_name, parent_package, level_desc)
else:
warning("%s: Cannot find '%s' %s.",
importing.getSourceReference().getAsString(),
module_name, level_desc)
def decideRecursion(module_filename, module_name, module_package, module_kind):
# Many branches, which make decisions immediately, by returning
# pylint: disable=R0911,R0912
Plugins.onModuleEncounter(
module_filename,
module_name,
module_package,
module_kind
)
if module_kind == "shlib":
if Options.isStandaloneMode():
return True, "Shared library for inclusion."
else:
return False, "Shared library cannot be inspected."
if module_package is None:
full_name = module_name
else:
full_name = module_package + '.' + module_name
no_case_modules = Options.getShallFollowInNoCase()
for no_case_module in no_case_modules:
if full_name == no_case_module:
return (
False,
"Module listed explicitly to not recurse to."
)
if full_name.startswith(no_case_module + '.'):
return (
False,
"Module in package listed explicitly to not recurse to."
)
any_case_modules = Options.getShallFollowModules()
for any_case_module in any_case_modules:
if full_name == any_case_module:
return (
True,
"Module listed explicitly to recurse to."
)
if full_name.startswith(any_case_module + '.'):
return (
True,
"Module in package listed explicitly to recurse to."
)
if Options.shallFollowNoImports():
return (
False,
"Requested to not recurse at all."
)
if StandardLibrary.isStandardLibraryPath(module_filename):
return (
Options.shallFollowStandardLibrary(),
"Requested to %srecurse to standard library." % (
"" if Options.shallFollowStandardLibrary() else "not "
)
)
if Options.shallFollowAllImports():
return (
True,
"Requested to recurse to all non-standard library modules."
)
# Means, we were not given instructions how to handle things.
return (
None,
"Default behavior, not recursing without request."
)
def _onEnterNode(self, node):
# This has many different things it deals with, so there need to be a
# lot of branches and statements, pylint: disable=R0912,R0915
# Also all self specific things have been done on the outside,
# pylint: disable=R0201
# Find nodes with only compile time constant children, these are
# missing some obvious optimization potentially.
if False: # For searching only, pylint: disable=W0125
if not node.isStatementReturn() and \
not node.isExpressionYield() and \
not node.isStatementRaiseException() and \
not node.isExpressionCall() and \
not node.isExpressionBuiltinIter1():
children = node.getVisitableNodes()
if children:
for child in children:
if child.isStatement() or child.isStatementsSequence():
break
if not child.isCompileTimeConstant():
break
else:
assert False, (node, node.parent, children)
if node.isExpressionFunctionBody():
if node.isUnoptimized():
node.markAsLocalsDict()
if node.needsLocalsDict():
provider = node.getParentVariableProvider()
if not provider.isCompiledPythonModule():
provider.markAsLocalsDict()
if node.isStatementReturn() or node.isStatementGeneratorReturn():
search = node
in_tried_block = False
# Search up to the containing function, and check for a try/finally
# containing the "return" statement.
search = search.getParentReturnConsumer()
if search.isExpressionGeneratorObjectBody() or \
search.isExpressionCoroutineObjectBody():
if in_tried_block:
search.markAsNeedsGeneratorReturnHandling(2)
else:
search.markAsNeedsGeneratorReturnHandling(1)
if node.isExpressionBuiltinImport() and \
not Options.getShallFollowExtra() and \
not Options.getShallFollowExtraFilePatterns() and \
not Options.shallFollowNoImports() and \
not isWhiteListedImport(node) and \
not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()):
warning("""Unresolved '__import__' call at '%s' may require use \
of '--recurse-directory'.""" % (node.getSourceReference().getAsString()))
if node.isExpressionFunctionCreation():
if not node.getParent().isExpressionFunctionCall() or \
node.getParent().getFunction() is not node:
node.getFunctionRef().getFunctionBody().markAsNeedsCreation()
if node.isExpressionFunctionCall():
node.getFunction().getFunctionRef().getFunctionBody().\
markAsDirectlyCalled()
if node.isExpressionFunctionRef():
function_body = node.getFunctionBody()
parent_module = function_body.getParentModule()
node_module = node.getParentModule()
if node_module is not parent_module:
function_body.markAsCrossModuleUsed()
node_module.addCrossUsedFunction(function_body)
if node.isStatementAssignmentVariable():
target_var = node.getTargetVariableRef().getVariable()
assign_source = node.getAssignSource()
if assign_source.isExpressionOperationBinary():
left_arg = assign_source.getLeft()
if left_arg.isExpressionVariableRef():
if assign_source.getLeft().getVariable().isModuleVariable(
):
assign_source.unmarkAsInplaceSuspect()
elif assign_source.getLeft().getVariable() is target_var:
if assign_source.isInplaceSuspect():
node.markAsInplaceSuspect()
if node.isStatementPublishException():
node.getParentStatementsFrame().markAsFrameExceptionPreserving()
if python_version >= 300:
if node.isExpressionYield() or node.isExpressionYieldFrom():
search = node.getParent()
while not search.isExpressionGeneratorObjectBody():
last_search = search
search = search.getParent()
if search.isStatementTry() and \
last_search == search.getBlockExceptHandler():
node.markAsExceptionPreserving()
break
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = Utils.dirname(Utils.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if Utils.isFile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(
SyntaxErrors.formatOutput(e)
)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(
getSourceDirectoryPath(main_module)
)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(
0,
(binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
if Utils.getOS() == "NetBSD":
warning("Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported.")
copyUsedDLLs(
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for source_filename, target_filename in data_files:
shutil.copy2(
source_filename,
Utils.joinpath(
getStandaloneDirectoryPath(main_module),
target_filename
)
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
def _onEnterNode(self, node):
# This has many different things it deals with, so there need to be a
# lot of branches and statements, pylint: disable=R0912,R0915
# Also all self specific things have been done on the outside,
# pylint: disable=R0201
# Find nodes with only compile time constant children, these are
# missing some obvious optimization potentially.
if False:
if not node.isStatementReturn() and \
not node.isExpressionYield() and \
not node.isStatementRaiseException() and \
not node.isExpressionCall() and \
not node.isExpressionBuiltinIter1():
children = node.getVisitableNodes()
if children:
for child in children:
if child.isStatement() or child.isStatementsSequence():
break
if not child.isCompileTimeConstant():
break
else:
assert False, (node, node.parent, children)
if node.isExpressionFunctionBody():
if node.isUnoptimized():
node.markAsLocalsDict()
if node.needsLocalsDict():
provider = node.getParentVariableProvider()
if provider.isExpressionFunctionBody():
provider.markAsLocalsDict()
if node.isStatementReturn() or node.isStatementGeneratorReturn():
search = node
in_tried_block = False
# Search up to the containing function, and check for a try/finally
# containing the "return" statement.
search = search.getParentReturnConsumer()
if search.isExpressionFunctionBody() and search.isGenerator():
if in_tried_block:
search.markAsNeedsGeneratorReturnHandling(2)
else:
search.markAsNeedsGeneratorReturnHandling(1)
if node.isExpressionBuiltinImport() and \
not Options.getShallFollowExtra() and \
not Options.getShallFollowExtraFilePatterns() and \
not Options.shallFollowNoImports() and \
not isWhiteListedImport(node) and \
not Plugins.suppressBuiltinImportWarning(node.getParentModule(), node.getSourceReference()):
warning("""Unresolved '__import__' call at '%s' may require use \
of '--recurse-directory'.""" % (
node.getSourceReference().getAsString()
)
)
if node.isExpressionFunctionCreation():
if not node.getParent().isExpressionFunctionCall() or \
node.getParent().getFunction() is not node:
node.getFunctionRef().getFunctionBody().markAsNeedsCreation()
if node.isExpressionFunctionCall():
node.getFunction().getFunctionRef().getFunctionBody().\
markAsDirectlyCalled()
if node.isExpressionFunctionRef():
function_body = node.getFunctionBody()
parent_module = function_body.getParentModule()
node_module = node.getParentModule()
if node_module is not parent_module:
function_body.markAsCrossModuleUsed()
node_module.addCrossUsedFunction(function_body)
if node.isStatementAssignmentVariable():
target_var = node.getTargetVariableRef().getVariable()
assign_source = node.getAssignSource()
if assign_source.isExpressionOperationBinary():
left_arg = assign_source.getLeft()
if left_arg.isExpressionVariableRef():
if assign_source.getLeft().getVariable().isModuleVariable():
assign_source.unmarkAsInplaceSuspect()
elif assign_source.getLeft().getVariable() is target_var:
if assign_source.isInplaceSuspect():
node.markAsInplaceSuspect()
if node.isStatementPublishException():
node.getParentStatementsFrame().markAsFrameExceptionPreserving()
if python_version >= 300:
if node.isExpressionYield() or node.isExpressionYieldFrom():
search = node.getParent()
while not search.isExpressionFunctionBody():
last_search = search
search = search.getParent()
if search.isStatementTry() and \
last_search == search.getBlockExceptHandler():
node.markAsExceptionPreserving()
break
