def optimizeWhole(main_module): done_modules = set() optimizeModule(main_module) done_modules.add(main_module) if _progress: printLine("Finished. %d more modules to go." % len(getImportedModules())) finished = False while not finished: finished = True for module in list(getImportedModules()): if module not in done_modules: optimizeModule(module=module) done_modules.add(module) if _progress: printLine("Finished. %d more modules to go." % (len(getImportedModules()) - len(done_modules))) finished = False
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name=module.getFullName())) global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() _optimizeModulePass(module=module, tag_set=tag_set) if not tag_set: break touched = True if _progress: memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) return touched
def endGraph(): if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name=module.getFullName())) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() _optimizeModulePass(module=module) if not tag_set: break touched = True if _progress: memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) return touched
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def _checkXMLPersistence(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", "w").write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", "w").write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), ( module.getOutputFilename(), restored.getOutputFilename(), ) # The variable versions give diffs. if True: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff(text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded") for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal()
def optimizePythonModule(module): if _progress: printLine("Doing module local optimizations for '{module_name}'.".format(module_name=module.getFullName())) global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() _optimizeModulePass(module=module, tag_set=tag_set) if not tag_set: break touched = True if _progress: memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) return touched
def optimizePythonModule(module): if _progress: printLine("Doing module local optimizations for '{module_name}'.".format(module_name=module.getFullName())) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() module.computeModule() if not tag_set: break touched = True if _progress: memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) return touched or module.hasUnclearLocals()
def optimizeWhole( main_module ): done_modules = set() optimizeModule( main_module ) done_modules.add( main_module ) if _progress: printLine( "Finished. %d more modules to go." % len( getImportedModules() ) ) finished = False while not finished: finished = True for module in list( getImportedModules() ): if module not in done_modules: optimizeModule( module = module ) done_modules.add( module ) if _progress: printLine( "Finished. %d more modules to go." % ( len( getImportedModules() ) - len( done_modules ) ) ) finished = False
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False if finished: break
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None if not VariableRegistry.complete: VariableRegistry.complete = True finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def _traceProgress(current_module): output = """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that.""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), ) if Options.isShowMemory(): output += "Memory usage {memory}:".format( memory=MemoryUsage.getHumanReadableProcessMemoryUsage()) printLine(output)
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name = module.getFullName() ) ) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise if not tag_set: break if graph is not None: computation_counters[module] = computation_counters.get(module, 0) + 1 module_graph = module.asGraph(computation_counters[module]) graph.subgraph(module_graph) touched = True if _progress: memory_watch.finish() printLine( "Memory usage changed during optimization of '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) ) Plugins.considerImplicitImports(module, signal_change = signalChange) return touched
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name = module.getFullName() ) ) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = MemoryUsage.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise if not tag_set: break if graph is not None: computation_counters[module] = computation_counters.get(module, 0) + 1 module_graph = module.asGraph(computation_counters[module]) graph.subgraph(module_graph) touched = True if _progress: memory_watch.finish() printLine( "Memory usage changed during optimization of '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) ) Plugins.considerImplicitImports(module, signal_change = signalChange) return touched
def _traceProgress(current_module): output = """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that.""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), ) if Options.isShowMemory(): output += "Memory usage {memory}:".format( memory = MemoryUsage.getHumanReadableProcessMemoryUsage() ) printLine(output)
def showMemoryTrace(): try: import tracemalloc # @UnresolvedImport except ImportError: pass else: snapshot = tracemalloc.take_snapshot() stats = snapshot.statistics("lineno") printLine("Top 50 memory allocations:") for count, stat in enumerate(stats): if count == 50: break printLine(stat)
def optimizeCompiledPythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name = module.getFullName() ) ) touched = False if _progress and Options.isShowMemory(): memory_watch = MemoryUsage.MemoryWatch() while True: tag_set.clear() try: module.computeModule() except BaseException: info("Interrupted while working on '%s'." % module) raise Graphs.onModuleOptimizationStep(module) # Search for local change tags. for tag in tag_set: if tag == "new_code": continue break else: break # Otherwise we did stuff, so note that for return value. touched = True if _progress and Options.isShowMemory(): memory_watch.finish() printLine( "Memory usage changed during optimization of '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) ) Plugins.considerImplicitImports(module, signal_change = signalChange) return touched
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: printLine("Initial optimization pass.") else: printLine("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateFromCollection( old_collection = function.trace_collection, new_collection = None ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): optimizeVariables(current_module) return finished
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: printLine("Initial optimization pass.") else: printLine("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None for current_module in ModuleRegistry.getDoneModules(): optimizeVariables(current_module) return finished
def optimizeUncompiledPythonModule(module): if _progress: printLine( "Doing module dependency considerations for '{module_name}':". format(module_name=module.getFullName())) for used_module_name in module.getUsedModules(): used_module = ImportCache.getImportedModuleByName(used_module_name) ModuleRegistry.addUsedModule(used_module) package_name = module.getPackage() if package_name is not None: used_module = ImportCache.getImportedModuleByName(package_name) ModuleRegistry.addUsedModule(used_module)
def optimizeModule(module): if _progress: printLine("Doing module local optimizations for '%s'." % module.getFullName()) tag_set = TagSet() while True: tag_set.clear() _optimizeModulePass(module=module, tag_set=tag_set) if not tag_set: break return module
def optimizeUncompiledPythonModule(module): if _progress: printLine( "Doing module dependency considerations for '{module_name}':".format( module_name = module.getFullName() ) ) for used_module_name in module.getUsedModules(): used_module = ImportCache.getImportedModuleByName(used_module_name) ModuleRegistry.addUsedModule(used_module) package_name = module.getPackage() if package_name is not None: used_module = ImportCache.getImportedModuleByName(package_name) ModuleRegistry.addUsedModule(used_module)
def optimizeModule( module ): if _progress: printLine( "Doing module local optimizations for '%s'." % module.getFullName() ) tag_set = TagSet() while True: tag_set.clear() _optimizeModulePass( module = module, tag_set = tag_set ) if not tag_set: break return module
def _checkXMLPersistence(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() with open("out.xml", "w") as f: f.write(text) restored = restoreFromXML(text) retext = restored.asXmlText() with open("out2.xml", "w") as f: f.write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), ( module.getOutputFilename(), restored.getOutputFilename(), ) # The variable versions give diffs. if True: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff( text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded" ) for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal()
def listPlugins(): """Print available standard plugins.""" loadPlugins() printLine("The following plugins are available in Nuitka".center(80)) printLine("-" * 80) plist = [] name_len = 0 for plugin_name in sorted(plugin_name2plugin_classes): plugin = plugin_name2plugin_classes[plugin_name][0] if hasattr(plugin, "plugin_desc"): plist.append((plugin_name, plugin.plugin_desc)) else: plist.append((plugin_name, "")) name_len = max(len(plugin_name) + 1, name_len) for line in plist: printLine(" " + line[0].ljust(name_len), line[1])
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 # We maintain this globally to make it accessible, pylint: disable=W0603 global graph if Options.shouldCreateGraph(): try: from graphviz import Digraph # pylint: disable=F0401,I0021 graph = Digraph('G') except ImportError: warning("Cannot import graphviz module, no graphing capability.") while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = MemoryUsage.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None if VariableRegistry.considerCompletion(): finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)
def dump(node): printLine(ast.dump(node))
def optimize(): Graphs.startGraph() # First pass. if _progress: info("PASS 1:") makeOptimizationPass(False) Variables.complete = True finished = makeOptimizationPass(False) if Options.isExperimental(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", 'w').write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", 'w').write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), \ (module.getOutputFilename(),restored.getOutputFilename()) # The variable versions give diffs. if False: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff( text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded" ) for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal() # Demote to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneUserModules(): if module.isPythonShlibModule(): continue if module.mode == "bytecode": demoteCompiledModuleToBytecode(module) if _progress: info("PASS 2 ... :") # Second, "endless" pass. while not finished: finished = makeOptimizationPass(True) Graphs.endGraph()
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 # We maintain this globally to make it accessible, pylint: disable=W0603 global graph if Options.shouldCreateGraph(): try: from graphviz import Digraph # pylint: disable=F0401,I0021 graph = Digraph('G') except ImportError: warning("Cannot import graphviz module, no graphing capability.") while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection=function.constraint_collection, new_collection=None) function.constraint_collection = None if not VariableRegistry.complete: VariableRegistry.complete = True finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)
def printStats(): printLine("Init/del calls:") for name, count in sorted(counted_inits.items()): dels = counted_dels.get(name, 0) printIndented(1, name, count, dels, count - dels)
def optimize(): Graphs.startGraph() # First pass. if _progress: info("PASS 1:") makeOptimizationPass(False) Variables.complete = True finished = makeOptimizationPass(False) if Options.isExperimental(): new_roots = ModuleRegistry.root_modules.__class__( ) # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", 'w').write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", 'w').write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), \ (module.getOutputFilename(),restored.getOutputFilename()) # The variable versions give diffs. if False: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff(text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded") for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal() # Demote to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneUserModules(): if module.isPythonShlibModule(): continue if module.mode == "bytecode": demoteCompiledModuleToBytecode(module) if _progress: info("PASS 2 ... :") # Second, "endless" pass. while not finished: finished = makeOptimizationPass(True) Graphs.endGraph()