def optimizeWhole(main_module):
done_modules = set()
optimizeModule(main_module)
done_modules.add(main_module)
if _progress:
printLine("Finished. %d more modules to go." %
len(getImportedModules()))
finished = False
while not finished:
finished = True
for module in list(getImportedModules()):
if module not in done_modules:
optimizeModule(module=module)
done_modules.add(module)
if _progress:
printLine("Finished. %d more modules to go." %
(len(getImportedModules()) - len(done_modules)))
finished = False
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()))
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
_optimizeModulePass(module=module, tag_set=tag_set)
if not tag_set:
break
touched = True
if _progress:
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
return touched
def endGraph():
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()))
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
_optimizeModulePass(module=module)
if not tag_set:
break
touched = True
if _progress:
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
return touched
def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def _checkXMLPersistence():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", "w").write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", "w").write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), (
module.getOutputFilename(),
restored.getOutputFilename(),
)
# The variable versions give diffs.
if True: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(text.splitlines(), retext.splitlines(),
"xml orig", "xml reloaded")
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
def optimizePythonModule(module):
if _progress:
printLine("Doing module local optimizations for '{module_name}'.".format(module_name=module.getFullName()))
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
_optimizeModulePass(module=module, tag_set=tag_set)
if not tag_set:
break
touched = True
if _progress:
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr()))
return touched
def optimizePythonModule(module):
if _progress:
printLine("Doing module local optimizations for '{module_name}'.".format(module_name=module.getFullName()))
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
module.computeModule()
if not tag_set:
break
touched = True
if _progress:
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr()))
return touched or module.hasUnclearLocals()
def optimizeWhole( main_module ):
done_modules = set()
optimizeModule( main_module )
done_modules.add( main_module )
if _progress:
printLine( "Finished. %d more modules to go." % len( getImportedModules() ) )
finished = False
while not finished:
finished = True
for module in list( getImportedModules() ):
if module not in done_modules:
optimizeModule(
module = module
)
done_modules.add( module )
if _progress:
printLine(
"Finished. %d more modules to go." % (
len( getImportedModules() ) - len( done_modules )
)
)
finished = False
def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
if finished:
break
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def _traceProgress(current_module):
output = """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that.""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
)
if Options.isShowMemory():
output += "Memory usage {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage())
printLine(output)
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
if not tag_set:
break
if graph is not None:
computation_counters[module] = computation_counters.get(module, 0) + 1
module_graph = module.asGraph(computation_counters[module])
graph.subgraph(module_graph)
touched = True
if _progress:
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
if not tag_set:
break
if graph is not None:
computation_counters[module] = computation_counters.get(module, 0) + 1
module_graph = module.asGraph(computation_counters[module])
graph.subgraph(module_graph)
touched = True
if _progress:
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def _traceProgress(current_module):
output = """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that.""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
)
if Options.isShowMemory():
output += "Memory usage {memory}:".format(
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
printLine(output)
def showMemoryTrace():
try:
import tracemalloc # @UnresolvedImport
except ImportError:
pass
else:
snapshot = tracemalloc.take_snapshot()
stats = snapshot.statistics("lineno")
printLine("Top 50 memory allocations:")
for count, stat in enumerate(stats):
if count == 50:
break
printLine(stat)
def showMemoryTrace():
try:
import tracemalloc # @UnresolvedImport
except ImportError:
pass
else:
snapshot = tracemalloc.take_snapshot()
stats = snapshot.statistics("lineno")
printLine("Top 50 memory allocations:")
for count, stat in enumerate(stats):
if count == 50:
break
printLine(stat)
def optimizeCompiledPythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
touched = False
if _progress and Options.isShowMemory():
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
Graphs.onModuleOptimizationStep(module)
# Search for local change tags.
for tag in tag_set:
if tag == "new_code":
continue
break
else:
break
# Otherwise we did stuff, so note that for return value.
touched = True
if _progress and Options.isShowMemory():
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
printLine("Initial optimization pass.")
else:
printLine("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateFromCollection(
old_collection = function.trace_collection,
new_collection = None
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
optimizeVariables(current_module)
return finished
def optimizeCompiledPythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name = module.getFullName()
)
)
touched = False
if _progress and Options.isShowMemory():
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
module.computeModule()
except BaseException:
info("Interrupted while working on '%s'." % module)
raise
Graphs.onModuleOptimizationStep(module)
# Search for local change tags.
for tag in tag_set:
if tag == "new_code":
continue
break
else:
break
# Otherwise we did stuff, so note that for return value.
touched = True
if _progress and Options.isShowMemory():
memory_watch.finish()
printLine(
"Memory usage changed during optimization of '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
Plugins.considerImplicitImports(module, signal_change = signalChange)
return touched
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
printLine("Initial optimization pass.")
else:
printLine("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
for current_module in ModuleRegistry.getDoneModules():
optimizeVariables(current_module)
return finished
def optimizeUncompiledPythonModule(module):
if _progress:
printLine(
"Doing module dependency considerations for '{module_name}':".
format(module_name=module.getFullName()))
for used_module_name in module.getUsedModules():
used_module = ImportCache.getImportedModuleByName(used_module_name)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
def optimizeModule(module):
if _progress:
printLine("Doing module local optimizations for '%s'." %
module.getFullName())
tag_set = TagSet()
while True:
tag_set.clear()
_optimizeModulePass(module=module, tag_set=tag_set)
if not tag_set:
break
return module
def optimizeUncompiledPythonModule(module):
if _progress:
printLine(
"Doing module dependency considerations for '{module_name}':".format(
module_name = module.getFullName()
)
)
for used_module_name in module.getUsedModules():
used_module = ImportCache.getImportedModuleByName(used_module_name)
ModuleRegistry.addUsedModule(used_module)
package_name = module.getPackage()
if package_name is not None:
used_module = ImportCache.getImportedModuleByName(package_name)
ModuleRegistry.addUsedModule(used_module)
def optimizeModule( module ):
if _progress:
printLine( "Doing module local optimizations for '%s'." % module.getFullName() )
tag_set = TagSet()
while True:
tag_set.clear()
_optimizeModulePass(
module = module,
tag_set = tag_set
)
if not tag_set:
break
return module
def _checkXMLPersistence():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
with open("out.xml", "w") as f:
f.write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
with open("out2.xml", "w") as f:
f.write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), (
module.getOutputFilename(),
restored.getOutputFilename(),
)
# The variable versions give diffs.
if True: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(
text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded"
)
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
def listPlugins():
"""Print available standard plugins."""
loadPlugins()
printLine("The following plugins are available in Nuitka".center(80))
printLine("-" * 80)
plist = []
name_len = 0
for plugin_name in sorted(plugin_name2plugin_classes):
plugin = plugin_name2plugin_classes[plugin_name][0]
if hasattr(plugin, "plugin_desc"):
plist.append((plugin_name, plugin.plugin_desc))
else:
plist.append((plugin_name, ""))
name_len = max(len(plugin_name) + 1, name_len)
for line in plist:
printLine(" " + line[0].ljust(name_len), line[1])
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if VariableRegistry.considerCompletion():
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def dump(node):
printLine(ast.dump(node))
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
Variables.complete = True
finished = makeOptimizationPass(False)
if Options.isExperimental():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", 'w').write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", 'w').write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), \
(module.getOutputFilename(),restored.getOutputFilename())
# The variable versions give diffs.
if False: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(
text.splitlines(),
retext.splitlines(),
"xml orig",
"xml reloaded"
)
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
# Demote to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection=function.constraint_collection,
new_collection=None)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def printStats():
printLine("Init/del calls:")
for name, count in sorted(counted_inits.items()):
dels = counted_dels.get(name, 0)
printIndented(1, name, count, dels, count - dels)
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
Variables.complete = True
finished = makeOptimizationPass(False)
if Options.isExperimental():
new_roots = ModuleRegistry.root_modules.__class__(
) # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", 'w').write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", 'w').write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), \
(module.getOutputFilename(),restored.getOutputFilename())
# The variable versions give diffs.
if False: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(text.splitlines(),
retext.splitlines(), "xml orig",
"xml reloaded")
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
# Demote to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def printStats():
printLine("Init/del calls:")
for name, count in sorted(counted_inits.items()):
dels = counted_dels.get(name, 0)
printIndented(1, name, count, dels, count - dels)
