def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
if finished:
break
def _detectedShlibFile(filename, module_name):
# That is not a shared library, but looks like one.
if module_name == "__main__":
return
from nuitka import ModuleRegistry
if ModuleRegistry.hasRootModule(module_name):
return
parts = module_name.split('.')
if len(parts) == 1:
package_name = None
name = module_name
else:
package_name = '.'.join(parts[:-1])
name = parts[-1]
source_ref = SourceCodeReferences.fromFilename(
filename = filename
)
shlib_module = PythonShlibModule(
name = name,
package_name = package_name,
source_ref = source_ref
)
ModuleRegistry.addRootModule(shlib_module)
ImportCache.addImportedModule(shlib_module)
module_names.add(module_name)
def _detectedShlibFile(filename, module_name):
if Utils.python_version >= 300:
filename = filename.decode("utf-8")
# That is not a shared library, but looks like one.
if module_name == "__main__":
return
parts = module_name.split('.')
if len(parts) == 1:
package_name = None
name = module_name
else:
package_name = '.'.join(parts[:-1])
name = parts[-1]
source_ref = SourceCodeReferences.fromFilename(
filename = filename
)
shlib_module = PythonShlibModule(
name = name,
package_name = package_name,
source_ref = source_ref
)
from nuitka import ModuleRegistry
ModuleRegistry.addRootModule(shlib_module)
module_names.add(module_name)
def _detectedShlibFile(filename, module_name, result):
if Utils.python_version >= 300:
filename = filename.decode("utf-8")
parts = module_name.split(".")
if len(parts) == 1:
package_name = None
name = module_name
else:
package_name = ".".join(parts[:-1])
name = parts[-1]
from nuitka.nodes.FutureSpecs import FutureSpec
from nuitka.nodes.ModuleNodes import PythonShlibModule
from nuitka import SourceCodeReferences
source_ref = SourceCodeReferences.fromFilename(
filename = filename,
future_spec = FutureSpec()
)
shlib_module = PythonShlibModule(
package_name = package_name,
name = name,
source_ref = source_ref
)
from nuitka import ModuleRegistry
ModuleRegistry.addRootModule(shlib_module)
module_names.add(module_name)
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
printLine("Initial optimization pass.")
else:
printLine("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateFromCollection(
old_collection = function.trace_collection,
new_collection = None
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
optimizeVariables(current_module)
return finished
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
VariableRegistry.considerCompletion()
finished = makeOptimizationPass(False)
# Demote to bytecode if now.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
# Second, endless pass.
if _progress:
info("PASS 2..:")
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def optimize(output_filename):
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(initial_pass=True)
Variables.complete = True
finished = makeOptimizationPass(initial_pass=False)
if Options.isExperimental("check_xml_persistence"):
_checkXMLPersistence()
# Demote compiled modules to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule() and module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(initial_pass=False)
Graphs.endGraph(output_filename)
def optimizeUncompiledPythonModule(module):
if _progress:
printLine(
"Doing module dependency considerations for '{module_name}':".format(
module_name = module.getFullName()
)
)
for used_module_name in module.getUsedModules():
used_module = ImportCache.getImportedModuleByName(used_module_name)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
def optimizeUncompiledPythonModule(module):
if _progress:
info(
"Doing module dependency considerations for '{module_name}':".format(
module_name=module.getFullName()
)
)
for used_module_name, used_module_path in module.getUsedModules():
used_module = ImportCache.getImportedModuleByNameAndPath(
used_module_name, used_module_path
)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
def _traceProgress(current_module):
output = """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that.""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
)
if Options.isShowMemory():
output += "Memory usage {memory}:".format(
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
printLine(output)
def _checkXMLPersistence():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
with open("out.xml", "w") as f:
f.write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
with open("out2.xml", "w") as f:
f.write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), (
module.getOutputFilename(),
restored.getOutputFilename(),
)
# The variable versions give diffs.
if True: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(
text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded"
)
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
# Controls complex optimization, pylint: disable=too-many-branches
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
info("Initial optimization pass.")
else:
info("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection=function.trace_collection, new_collection=None
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
if optimizeVariables(current_module):
finished = False
used_functions = current_module.getUsedFunctions()
for unused_function in current_module.getUnusedFunctions():
unused_function.trace_collection = None
used_functions = tuple(
function
for function in current_module.getFunctions()
if function in used_functions
)
current_module.setFunctions(used_functions)
if Variables.complete:
if optimizeLocalsDictsHandles():
finished = False
return finished
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection=function.constraint_collection,
new_collection=None)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if VariableRegistry.considerCompletion():
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
# Controls complex optimization, pylint: disable=too-many-branches
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
info("Initial optimization pass.")
else:
info("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection = function.trace_collection,
new_collection = None
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
if optimizeVariables(current_module):
finished = False
used_functions = current_module.getUsedFunctions()
for unused_function in current_module.getUnusedFunctions():
unused_function.trace_collection = None
used_functions = tuple(
function
for function in
current_module.getFunctions()
if function in used_functions
)
current_module.setFunctions(used_functions)
return finished
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
debug("Checking detail plug-in path '%s' '%s':", plugin_filename, module_package)
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename
)
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_package=module_package,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(
module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind=module_kind,
reason=reason,
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_filename,
)
if not isSameModulePath(module.getFilename(), plugin_filename):
warning(
"Duplicate '%s' of '%s' ignored .",
plugin_filename,
module.getFilename(),
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module,
)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug("Package directory %s", package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or sub_path.endswith(".py"):
checkPluginSinglePath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_filename)
def recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
if not ImportCache.isImportedModuleByPath(module_relpath):
module, source_ref, source_filename = Building.decideModuleTree(
filename = module_filename,
package = module_package,
is_top = False,
is_main = False,
is_shlib = module_kind == "shlib"
)
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
debug(
"Recurse to import '%s' from %s. (%s)",
module.getFullName(),
module_relpath,
reason
)
if module_kind == "py" and source_filename is not None:
try:
Building.createModuleTree(
module = module,
source_ref = source_ref,
source_code = readSourceCodeFromFilename(
module_name = module.getFullName(),
source_filename = source_filename
),
is_main = False
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name = module.getFullName(),
filename = module_filename,
bytecode = marshal.dumps(
compile(
readSourceCodeFromFilename(module.getFullName(), module_filename),
module_filename,
"exec"
)
),
is_package = module.isCompiledPythonPackage(),
user_provided = True,
technical = False
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
is_added = True
else:
module = ImportCache.getImportedModuleByName(
module.getFullName()
)
is_added = False
assert not module_relpath.endswith("/__init__.py"), module
return module, is_added
else:
return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
module, source_ref, source_filename = Building.decideModuleTree(
filename = module_filename,
package = module_package,
is_top = False,
is_main = False,
is_shlib = module_kind == "shlib"
)
# Check if the module name is known. In order to avoid duplicates,
# learn the new filename, and continue build if its not.
if not ImportCache.isImportedModuleByName(module.getFullName()):
logRecursion(
"Recurse to import '%s' from '%s'. (%s)",
module.getFullName(),
module_relpath,
reason
)
if module_kind == "py" and source_filename is not None:
try:
source_code = readSourceCodeFromFilename(
module_name = module.getFullName(),
source_filename = source_filename
)
Building.createModuleTree(
module = module,
source_ref = source_ref,
source_code = source_code,
is_main = False
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name = module.getFullName(),
filename = module_filename,
bytecode = marshal.dumps(
compile(
source_code,
module_filename,
"exec"
)
),
is_package = module.isCompiledPythonPackage(),
user_provided = True,
technical = False
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
is_added = True
else:
module = ImportCache.getImportedModuleByName(
module.getFullName()
)
is_added = False
return module, is_added
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
debug(
"Checking detail plug-in path '%s' '%s':",
plugin_filename,
module_package
)
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(plugin_filename)
if module_kind is not None:
decision, _reason = decideRecursion(
module_filename = plugin_filename,
module_name = module_name,
module_package = module_package,
module_kind = module_kind
)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(
module_filename = plugin_filename,
module_relpath = module_relpath,
module_package = module_package,
module_kind = "py",
reason = "Lives in plug-in directory."
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_filename
)
if not isSameModulePath(module.getFilename(), plugin_filename):
warning(
"Duplicate ignored '%s'.",
plugin_filename
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module
)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug(
"Package directory %s",
package_dir
)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_filename)
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
from nuitka.tree import Building
module, source_ref, source_code = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
if Options.isShowInclusion():
recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" %
(module.getFullName(), module_relpath, reason))
if source_code is not None:
try:
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning(
"""\
Cannot follow import to module %r (%r) because of %r""" %
(module_relpath, module_filename, e.__class__.__name__))
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot recurse to import module %r (%r) because code is too complex.""" % (
module_relpath,
module_filename,
))
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code,
module_filename,
"exec",
dont_inherit=True)),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
if Options.isShowInclusion():
recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
(plugin_filename, module_package))
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
module_name = ModuleName.makeModuleNameInPackage(module_name,
module_package)
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(
module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind=module_kind,
reason=reason,
)
if module:
if not is_added:
recursion_logger.warning(
"Recursed to %s '%s' at '%s' twice." % (
"package"
if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_filename,
))
if not isSameModulePath(module.getFilename(),
plugin_filename):
recursion_logger.warning(
"Duplicate '%s' of '%s' ignored ." % (
plugin_filename,
module.getFilename(),
))
return
if Options.isShowInclusion():
recursion_logger.info("Recursed to '%s' %s" % (
module.getFullName(),
module,
))
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 0x300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
if Options.isShowInclusion():
recursion_logger.info("Package directory '%s'." %
package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(
sub_path) and not os.path.exists(sub_path +
".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif sub_path.endswith(".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
elif module.isPythonShlibModule():
if Options.isStandaloneMode():
ModuleRegistry.addRootModule(module)
else:
recursion_logger.warning(
"Failed to include module from '%s'." % plugin_filename)
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason):
from nuitka.tree import Building
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
module, source_ref, source_filename = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
logRecursion(
"Recurse to import '%s' from '%s'. (%s)",
module.getFullName(),
module_relpath,
reason,
)
if module_kind == "py" and source_filename is not None:
try:
source_code = readSourceCodeFromFilename(
module_name=module.getFullName(), source_filename=source_filename
)
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because of '%s'""",
module_relpath,
module_filename,
e.__class__.__name__,
)
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
warning(
"""\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
module_relpath,
module_filename,
)
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(
source_code, module_filename, "exec", dont_inherit=True
)
),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def copyDataFiles(dist_dir):
"""Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Many details to deal with, pylint: disable=too-many-locals
for pattern, dest, arg in Options.getShallIncludeDataFiles():
filenames = resolveShellPatternToFilenames(pattern)
if not filenames:
inclusion_logger.warning("No match data file to be included: %r" % pattern)
for filename in filenames:
file_reason = "specified data file %r on command line" % arg
rel_path = dest
if rel_path.endswith(("/", os.path.sep)):
rel_path = os.path.join(rel_path, os.path.basename(filename))
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for plugin, included_datafile in Plugins.considerDataFiles(module):
_handleDataFile(
dist_dir=dist_dir, tracer=plugin, included_datafile=included_datafile
)
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage() or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=Options.getShallIncludePackageData()
)
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__",),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll")
+ getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(pkg_filename, rel_path, file_reason),
)
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
Variables.complete = True
finished = makeOptimizationPass(False)
if Options.isExperimental():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", 'w').write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", 'w').write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), \
(module.getOutputFilename(),restored.getOutputFilename())
# The variable versions give diffs.
if False: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(
text.splitlines(),
retext.splitlines(),
"xml orig",
"xml reloaded"
)
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
# Demote to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
Variables.complete = True
finished = makeOptimizationPass(False)
if Options.isExperimental():
new_roots = ModuleRegistry.root_modules.__class__(
) # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", 'w').write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", 'w').write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), \
(module.getOutputFilename(),restored.getOutputFilename())
# The variable versions give diffs.
if False: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(text.splitlines(),
retext.splitlines(), "xml orig",
"xml reloaded")
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
# Demote to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def buildModule(
module_name,
module_filename,
source_code,
is_top,
is_main,
is_extension,
is_fake,
hide_syntax_error,
):
# Many details to deal with, pylint: disable=too-many-branches,too-many-locals
(
main_added,
is_package,
is_namespace,
source_ref,
source_filename,
) = Importing.decideModuleSourceRef(
filename=module_filename,
module_name=module_name,
is_main=is_main,
is_fake=is_fake,
logger=general,
)
if Options.hasPythonFlagPackageMode():
if is_top and Options.shallMakeModule():
optimization_logger.warning(
"Python flag -m (package_mode) has no effect in module mode, it's only for executables."
)
elif is_main and not main_added:
optimization_logger.warning(
"Python flag -m (package_mode) only works on packages with '__main__.py'."
)
# Read source code if necessary. Might give a SyntaxError due to not being proper
# encoded source.
if source_filename is not None and not is_namespace and not is_extension:
try:
# For fake modules, source is provided directly.
if source_code is None:
source_code = readSourceCodeFromFilename(
module_name=module_name, source_filename=source_filename)
except SyntaxError as e:
# Avoid hiding our own syntax errors.
if not hasattr(e, "generated_by_nuitka"):
raise
# Do not hide SyntaxError in main module.
if not hide_syntax_error:
raise
module = _makeModuleBodyFromSyntaxError(
exc=e,
module_name=module_name,
module_filename=module_filename)
return module, True
try:
ast_tree = parseSourceCodeToAst(
source_code=source_code,
module_name=module_name,
filename=source_filename,
line_offset=0,
)
except (SyntaxError, IndentationError) as e:
# Do not hide SyntaxError if asked not to.
if not hide_syntax_error:
raise
module = _makeModuleBodyFromSyntaxError(
exc=e,
module_name=module_name,
module_filename=module_filename)
return module, True
except CodeTooComplexCode:
# Do not hide CodeTooComplexCode in main module.
if is_main:
raise
module = _makeModuleBodyTooComplex(
module_name=module_name,
module_filename=module_filename,
source_code=source_code,
is_package=is_package,
)
return module, False
else:
ast_tree = None
source_code = None
module = _createModule(
module_name=module_name,
source_code=source_code,
source_ref=source_ref,
is_top=is_top,
is_main=is_main,
is_extension=is_extension,
is_namespace=is_namespace,
is_package=is_package,
main_added=main_added,
)
if is_top:
ModuleRegistry.addRootModule(module)
OutputDirectories.setMainModule(module)
if module.isCompiledPythonModule() and source_code is not None:
createModuleTree(
module=module,
source_ref=source_ref,
ast_tree=ast_tree,
is_main=is_main,
)
return module, True
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
# The importing wants these to be unique.
plugin_filename = os.path.abspath(plugin_filename)
if Options.isShowInclusion():
recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
(plugin_filename, module_package))
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
module_name = ModuleName.makeModuleNameInPackage(module_name,
module_package)
if module_kind == "extension" and not Options.isStandaloneMode():
recursion_logger.warning(
"Cannot include '%s' unless using at least standalone mode." %
module_name.asString())
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module = recurseTo(
signal_change=None,
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
reason=reason,
)
if module:
if Options.isShowInclusion():
recursion_logger.info("Included '%s' as '%s'." % (
module.getFullName(),
module,
))
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 0x300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
if Options.isShowInclusion():
recursion_logger.info("Package directory '%s'." %
package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(
sub_path) and not os.path.exists(sub_path +
".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif sub_path.endswith(".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
elif module.isPythonExtensionModule():
if Options.isStandaloneMode():
ModuleRegistry.addRootModule(module)
else:
recursion_logger.warning(
"Failed to include module from '%s'." % plugin_filename)
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=R0912
debug("Checking detail plug-in path '%s' '%s':", plugin_filename,
module_package)
plugin_info = considerFilename(module_package=module_package,
module_filename=plugin_filename)
if plugin_info is not None:
module, is_added = recurseTo(module_filename=plugin_info[0],
module_relpath=plugin_info[1],
module_package=module_package,
module_kind="py",
reason="Lives in plug-in directory.")
if module:
if not is_added:
warning("Recursed to %s '%s' at '%s' twice.",
"package" if module.isPythonPackage() else "module",
module.getName(), plugin_info[0])
if not isSameModulePath(module.getFilename(), plugin_info[0]):
warning("Duplicate ignored '%s'.", plugin_info[1])
return
debug("Recursed to %s %s %s", module.getName(),
module.getPackage(), module)
if module.isPythonPackage():
package_filename = module.getFilename()
if Utils.isDir(package_filename):
# Must be a namespace package.
assert Utils.python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
useful = False
else:
package_dir = Utils.dirname(package_filename)
# Real packages will always be included.
useful = True
debug("Package directory %s", package_dir)
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
else:
# Modules should always be included.
useful = True
if useful:
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
def detectEarlyImports():
# Cyclic dependency
from nuitka import ModuleRegistry
for module in _detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=R0912
debug(
"Checking detail plug-in path '%s' '%s':",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
module, is_added = recurseTo(
module_filename = plugin_info[0],
module_relpath = plugin_info[1],
module_package = module_package,
module_kind = "py",
reason = "Lives in plug-in directory."
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isPythonPackage() else "module",
module.getName(),
plugin_info[0]
)
if not isSameModulePath(module.getFilename(), plugin_info[0]):
warning(
"Duplicate ignored '%s'.",
plugin_info[1]
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module
)
if module.isPythonPackage():
package_filename = module.getFilename()
if Utils.isDir(package_filename):
# Must be a namespace package.
assert Utils.python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
useful = False
else:
package_dir = Utils.dirname(package_filename)
# Real packages will always be included.
useful = True
debug(
"Package directory %s",
package_dir
)
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
else:
# Modules should always be included.
useful = True
if useful:
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
def makeOptimizationPass():
"""Make a single pass for optimization, indication potential completion."""
# Controls complex optimization
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for unused_function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection=unused_function.trace_collection,
new_collection=None,
source_ref=unused_function.getSourceReference(),
)
unused_function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
if optimizeVariables(current_module):
finished = False
used_functions = current_module.getUsedFunctions()
for unused_function in current_module.getUnusedFunctions():
unused_function.trace_collection = None
used_functions = tuple(
function for function in current_module.subnode_functions
if function in used_functions)
current_module.setChild("functions", used_functions)
if Variables.complete:
if optimizeLocalsDictsHandles():
finished = False
return finished
