def _checkXMLPersistence():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", "w").write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", "w").write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), (
module.getOutputFilename(),
restored.getOutputFilename(),
)
# The variable versions give diffs.
if True: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(text.splitlines(), retext.splitlines(),
"xml orig", "xml reloaded")
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
if finished:
break
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def makeOptimizationPass():
"""Make a single pass for optimization, indication potential completion."""
# Controls complex optimization
finished = True
ModuleRegistry.startTraversal()
_restartProgress()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used. This must be done after global
# optimization due to cross module usages.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for unused_function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection=unused_function.trace_collection,
new_collection=None,
source_ref=unused_function.getSourceReference(),
)
unused_function.trace_collection = None
used_functions = tuple(
function
for function in current_module.subnode_functions
if function in current_module.getUsedFunctions()
)
current_module.setChild("functions", used_functions)
_endProgress()
return finished
def makeOptimizationPass():
"""Make a single pass for optimization, indication potential completion."""
# Controls complex optimization
finished = True
ModuleRegistry.startTraversal()
_restartProgress()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
# TODO: Internal module seems to cause extra passes.
# optimizeModule(getInternalModule())
break
_traceProgressModuleStart(current_module)
changed = optimizeModule(current_module)
_traceProgressModuleEnd(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used. This must be done after global
# optimization due to cross module usages.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for unused_function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection=unused_function.trace_collection,
new_collection=None,
source_ref=unused_function.getSourceReference(),
)
unused_function.trace_collection = None
used_functions = tuple(
function
for function in current_module.subnode_functions
if function in current_module.getUsedFunctions()
)
current_module.setChild("functions", used_functions)
_endProgress()
return finished
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
printLine("Initial optimization pass.")
else:
printLine("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateFromCollection(
old_collection = function.trace_collection,
new_collection = None
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
optimizeVariables(current_module)
return finished
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
info("Initial optimization pass.")
else:
info("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateFromCollection(
old_collection=function.trace_collection,
new_collection=None)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
if optimizeVariables(current_module):
finished = False
return finished
def _checkXMLPersistence():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
with open("out.xml", "w") as f:
f.write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
with open("out2.xml", "w") as f:
f.write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), (
module.getOutputFilename(),
restored.getOutputFilename(),
)
# The variable versions give diffs.
if True: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(
text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded"
)
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
Variables.complete = True
finished = makeOptimizationPass(False)
if Options.isExperimental():
new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", 'w').write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", 'w').write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), \
(module.getOutputFilename(),restored.getOutputFilename())
# The variable versions give diffs.
if False: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(
text.splitlines(),
retext.splitlines(),
"xml orig",
"xml reloaded"
)
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
# Demote to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def makeOptimizationPass(initial_pass):
""" Make a single pass for optimization, indication potential completion.
"""
# Controls complex optimization, pylint: disable=too-many-branches
finished = True
ModuleRegistry.startTraversal()
if _progress:
if initial_pass:
info("Initial optimization pass.")
else:
info("Next global optimization pass.")
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection=function.trace_collection, new_collection=None
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
if optimizeVariables(current_module):
finished = False
used_functions = current_module.getUsedFunctions()
for unused_function in current_module.getUnusedFunctions():
unused_function.trace_collection = None
used_functions = tuple(
function
for function in current_module.getFunctions()
if function in used_functions
)
current_module.setFunctions(used_functions)
if Variables.complete:
if optimizeLocalsDictsHandles():
finished = False
return finished
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection = function.constraint_collection,
new_collection = None
)
function.constraint_collection = None
if VariableRegistry.considerCompletion():
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
def makeOptimizationPass():
"""Make a single pass for optimization, indication potential completion."""
# Controls complex optimization, pylint: disable=too-many-branches
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
_traceProgress(current_module)
# The tag set is global, so it can react to changes without context.
# pylint: disable=global-statement
global tag_set
tag_set = TagSet()
changed = optimizeModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code, dropping the trace
# collections of functions no longer used.
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
for function in current_module.getUnusedFunctions():
Variables.updateVariablesFromCollection(
old_collection=function.trace_collection,
new_collection=None,
source_ref=function.getSourceReference(),
)
function.trace_collection = None
for current_module in ModuleRegistry.getDoneModules():
if current_module.isCompiledPythonModule():
if optimizeVariables(current_module):
finished = False
used_functions = current_module.getUsedFunctions()
for unused_function in current_module.getUnusedFunctions():
unused_function.trace_collection = None
used_functions = tuple(
function for function in current_module.getFunctions()
if function in used_functions)
current_module.setFunctions(used_functions)
if Variables.complete:
if optimizeLocalsDictsHandles():
finished = False
return finished
def optimize():
Graphs.startGraph()
# First pass.
if _progress:
info("PASS 1:")
makeOptimizationPass(False)
Variables.complete = True
finished = makeOptimizationPass(False)
if Options.isExperimental():
new_roots = ModuleRegistry.root_modules.__class__(
) # @UndefinedVariable
for module in tuple(ModuleRegistry.getDoneModules()):
ModuleRegistry.root_modules.remove(module)
if module.isPythonShlibModule():
continue
text = module.asXmlText()
open("out.xml", 'w').write(text)
restored = restoreFromXML(text)
retext = restored.asXmlText()
open("out2.xml", 'w').write(retext)
assert module.getOutputFilename() == restored.getOutputFilename(), \
(module.getOutputFilename(),restored.getOutputFilename())
# The variable versions give diffs.
if False: # To manually enable, pylint: disable=W0125
import difflib
diff = difflib.unified_diff(text.splitlines(),
retext.splitlines(), "xml orig",
"xml reloaded")
for line in diff:
printLine(line)
new_roots.add(restored)
ModuleRegistry.root_modules = new_roots
ModuleRegistry.startTraversal()
# Demote to bytecode, now that imports had a chance to be resolved, and
# dependencies were handled.
for module in ModuleRegistry.getDoneUserModules():
if module.isPythonShlibModule():
continue
if module.mode == "bytecode":
demoteCompiledModuleToBytecode(module)
if _progress:
info("PASS 2 ... :")
# Second, "endless" pass.
while not finished:
finished = makeOptimizationPass(True)
Graphs.endGraph()
def optimize():
# This is somewhat complex with many cases, pylint: disable=R0912
# We maintain this globally to make it accessible, pylint: disable=W0603
global graph
if Options.shouldCreateGraph():
try:
from graphviz import Digraph # pylint: disable=F0401,I0021
graph = Digraph('G')
except ImportError:
warning("Cannot import graphviz module, no graphing capability.")
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
# Unregister collection traces from now unused code.
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
for function in current_module.getUnusedFunctions():
VariableRegistry.updateFromCollection(
old_collection=function.constraint_collection,
new_collection=None)
function.constraint_collection = None
if not VariableRegistry.complete:
VariableRegistry.complete = True
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
if graph is not None:
graph.engine = "dot"
graph.graph_attr["rankdir"] = "TB"
graph.render("something.dot")
printLine(graph.source)
