def _checkXMLPersistence(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", "w").write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", "w").write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), ( module.getOutputFilename(), restored.getOutputFilename(), ) # The variable versions give diffs. if True: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff(text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded") for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal()
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False if finished: break
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None if not VariableRegistry.complete: VariableRegistry.complete = True finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def makeOptimizationPass(): """Make a single pass for optimization, indication potential completion.""" # Controls complex optimization finished = True ModuleRegistry.startTraversal() _restartProgress() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. This must be done after global # optimization due to cross module usages. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for unused_function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=unused_function.trace_collection, new_collection=None, source_ref=unused_function.getSourceReference(), ) unused_function.trace_collection = None used_functions = tuple( function for function in current_module.subnode_functions if function in current_module.getUsedFunctions() ) current_module.setChild("functions", used_functions) _endProgress() return finished
def makeOptimizationPass(): """Make a single pass for optimization, indication potential completion.""" # Controls complex optimization finished = True ModuleRegistry.startTraversal() _restartProgress() while True: current_module = ModuleRegistry.nextModule() if current_module is None: # TODO: Internal module seems to cause extra passes. # optimizeModule(getInternalModule()) break _traceProgressModuleStart(current_module) changed = optimizeModule(current_module) _traceProgressModuleEnd(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. This must be done after global # optimization due to cross module usages. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for unused_function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=unused_function.trace_collection, new_collection=None, source_ref=unused_function.getSourceReference(), ) unused_function.trace_collection = None used_functions = tuple( function for function in current_module.subnode_functions if function in current_module.getUsedFunctions() ) current_module.setChild("functions", used_functions) _endProgress() return finished
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: printLine("Initial optimization pass.") else: printLine("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateFromCollection( old_collection = function.trace_collection, new_collection = None ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): optimizeVariables(current_module) return finished
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: info("Initial optimization pass.") else: info("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateFromCollection( old_collection=function.trace_collection, new_collection=None) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): if optimizeVariables(current_module): finished = False return finished
def _checkXMLPersistence(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() with open("out.xml", "w") as f: f.write(text) restored = restoreFromXML(text) retext = restored.asXmlText() with open("out2.xml", "w") as f: f.write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), ( module.getOutputFilename(), restored.getOutputFilename(), ) # The variable versions give diffs. if True: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff( text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded" ) for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal()
def optimize(): Graphs.startGraph() # First pass. if _progress: info("PASS 1:") makeOptimizationPass(False) Variables.complete = True finished = makeOptimizationPass(False) if Options.isExperimental(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", 'w').write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", 'w').write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), \ (module.getOutputFilename(),restored.getOutputFilename()) # The variable versions give diffs. if False: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff( text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded" ) for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal() # Demote to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneUserModules(): if module.isPythonShlibModule(): continue if module.mode == "bytecode": demoteCompiledModuleToBytecode(module) if _progress: info("PASS 2 ... :") # Second, "endless" pass. while not finished: finished = makeOptimizationPass(True) Graphs.endGraph()
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ # Controls complex optimization, pylint: disable=too-many-branches finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: info("Initial optimization pass.") else: info("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=function.trace_collection, new_collection=None ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): if optimizeVariables(current_module): finished = False used_functions = current_module.getUsedFunctions() for unused_function in current_module.getUnusedFunctions(): unused_function.trace_collection = None used_functions = tuple( function for function in current_module.getFunctions() if function in used_functions ) current_module.setFunctions(used_functions) if Variables.complete: if optimizeLocalsDictsHandles(): finished = False return finished
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 # We maintain this globally to make it accessible, pylint: disable=W0603 global graph if Options.shouldCreateGraph(): try: from graphviz import Digraph # pylint: disable=F0401,I0021 graph = Digraph('G') except ImportError: warning("Cannot import graphviz module, no graphing capability.") while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = MemoryUsage.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None if VariableRegistry.considerCompletion(): finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)
def makeOptimizationPass(): """Make a single pass for optimization, indication potential completion.""" # Controls complex optimization, pylint: disable=too-many-branches finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=function.trace_collection, new_collection=None, source_ref=function.getSourceReference(), ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): if optimizeVariables(current_module): finished = False used_functions = current_module.getUsedFunctions() for unused_function in current_module.getUnusedFunctions(): unused_function.trace_collection = None used_functions = tuple( function for function in current_module.getFunctions() if function in used_functions) current_module.setFunctions(used_functions) if Variables.complete: if optimizeLocalsDictsHandles(): finished = False return finished
def optimize(): Graphs.startGraph() # First pass. if _progress: info("PASS 1:") makeOptimizationPass(False) Variables.complete = True finished = makeOptimizationPass(False) if Options.isExperimental(): new_roots = ModuleRegistry.root_modules.__class__( ) # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", 'w').write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", 'w').write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), \ (module.getOutputFilename(),restored.getOutputFilename()) # The variable versions give diffs. if False: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff(text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded") for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal() # Demote to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneUserModules(): if module.isPythonShlibModule(): continue if module.mode == "bytecode": demoteCompiledModuleToBytecode(module) if _progress: info("PASS 2 ... :") # Second, "endless" pass. while not finished: finished = makeOptimizationPass(True) Graphs.endGraph()
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 # We maintain this globally to make it accessible, pylint: disable=W0603 global graph if Options.shouldCreateGraph(): try: from graphviz import Digraph # pylint: disable=F0401,I0021 graph = Digraph('G') except ImportError: warning("Cannot import graphviz module, no graphing capability.") while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection=function.constraint_collection, new_collection=None) function.constraint_collection = None if not VariableRegistry.complete: VariableRegistry.complete = True finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)