def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None if not VariableRegistry.complete: VariableRegistry.complete = True finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def addIncludedDataFilesFromPackageOptions(): """Late data files, from plugins and user options that work with packages""" # Cyclic dependency from nuitka import ModuleRegistry for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonPackage( ) or module.isUncompiledPythonPackage(): package_name = module.getFullName() match, reason = package_name.matchesToShellPatterns( patterns=getShallIncludePackageData()) if match: package_directory = module.getCompileTimeDirectory() pkg_filenames = getFileList( package_directory, ignore_dirs=("__pycache__", ), ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") + getSharedLibrarySuffixes(), ) if pkg_filenames: file_reason = "package '%s' %s" % (package_name, reason) for pkg_filename in pkg_filenames: rel_path = os.path.join( package_name.asPath(), os.path.relpath(pkg_filename, package_directory), ) addIncludedDataFile( makeIncludedDataFile( source_path=pkg_filename, dest_path=rel_path, reason=file_reason, tracer=options_logger, tags="user,package_data", )) # Circular dependency from nuitka.plugins.Plugins import Plugins # Plugins provide per module through this. for module in ModuleRegistry.getDoneModules(): for included_datafile in Plugins.considerDataFiles(module=module): addIncludedDataFile(included_datafile) for included_datafile in getIncludedDataFiles(): if isinstance(included_datafile, (IncludedDataFile)): Plugins.onDataFileTags(included_datafile, )
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: info("Initial optimization pass.") else: info("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateFromCollection( old_collection=function.trace_collection, new_collection=None) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): if optimizeVariables(current_module): finished = False return finished
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: printLine("Initial optimization pass.") else: printLine("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateFromCollection( old_collection = function.trace_collection, new_collection = None ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): optimizeVariables(current_module) return finished
def copyDataFiles(dist_dir): """ Copy the data files needed for standalone distribution. Args: dist_dir: The distribution folder under creation Notes: This is for data files only, not DLLs or even extension modules, those must be registered as entry points, and would not go through necessary handling if provided like this. """ # Cyclic dependency from nuitka import ModuleRegistry for module in ModuleRegistry.getDoneModules(): for _plugin_name, (source_desc, target_filename) in Plugins.considerDataFiles( module ): target_filename = os.path.join(dist_dir, target_filename) assert isPathBelow(dist_dir, target_filename) makePath(os.path.dirname(target_filename)) if inspect.isfunction(source_desc): content = source_desc(target_filename) if content is not None: # support creation of empty directories with open( target_filename, "wb" if type(content) is bytes else "w" ) as output: output.write(content) else: shutil.copy2(source_desc, target_filename)
def _checkXMLPersistence(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", "w").write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", "w").write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), ( module.getOutputFilename(), restored.getOutputFilename(), ) # The variable versions give diffs. if True: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff(text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded") for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal()
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def _checkXMLPersistence(): for module in ModuleRegistry.getDoneModules(): if not module.isCompiledPythonModule(): continue text = module.asXmlText() with open(_getCacheFilename(module), "w") as f: f.write(text)
def makeOptimizationPass(): """Make a single pass for optimization, indication potential completion.""" # Controls complex optimization finished = True ModuleRegistry.startTraversal() _restartProgress() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. This must be done after global # optimization due to cross module usages. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for unused_function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=unused_function.trace_collection, new_collection=None, source_ref=unused_function.getSourceReference(), ) unused_function.trace_collection = None used_functions = tuple( function for function in current_module.subnode_functions if function in current_module.getUsedFunctions() ) current_module.setChild("functions", used_functions) _endProgress() return finished
def makeOptimizationPass(): """Make a single pass for optimization, indication potential completion.""" # Controls complex optimization finished = True ModuleRegistry.startTraversal() _restartProgress() while True: current_module = ModuleRegistry.nextModule() if current_module is None: # TODO: Internal module seems to cause extra passes. # optimizeModule(getInternalModule()) break _traceProgressModuleStart(current_module) changed = optimizeModule(current_module) _traceProgressModuleEnd(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. This must be done after global # optimization due to cross module usages. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for unused_function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=unused_function.trace_collection, new_collection=None, source_ref=unused_function.getSourceReference(), ) unused_function.trace_collection = None used_functions = tuple( function for function in current_module.subnode_functions if function in current_module.getUsedFunctions() ) current_module.setChild("functions", used_functions) _endProgress() return finished
def optimizeModules(output_filename): Graphs.startGraph() finished = makeOptimizationPass() # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneModules(): if (module.isCompiledPythonModule() and module.getCompilationMode() == "bytecode"): demoteCompiledModuleToBytecode(module) # Second, "endless" pass. while not finished: finished = makeOptimizationPass() Graphs.endGraph(output_filename)
def optimize(output_filename): Graphs.startGraph() finished = makeOptimizationPass() if Options.isExperimental("check_xml_persistence"): _checkXMLPersistence() # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneModules(): if (module.isCompiledPythonModule() and module.getCompilationMode() == "bytecode"): demoteCompiledModuleToBytecode(module) global pass_count # Singleton, pylint: disable=global-statement # Second, "endless" pass. while not finished: finished = makeOptimizationPass() Graphs.endGraph(output_filename)
def _checkXMLPersistence(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() with open("out.xml", "w") as f: f.write(text) restored = restoreFromXML(text) retext = restored.asXmlText() with open("out2.xml", "w") as f: f.write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), ( module.getOutputFilename(), restored.getOutputFilename(), ) # The variable versions give diffs. if True: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff( text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded" ) for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal()
def optimize(output_filename): Graphs.startGraph() # First pass. if _progress: progress_logger.info("PASS 1:") makeOptimizationPass() Variables.complete = True if _progress: progress_logger.info("PASS 2:") finished = makeOptimizationPass() if Options.isExperimental("check_xml_persistence"): _checkXMLPersistence() # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneModules(): if (module.isCompiledPythonModule() and module.getCompilationMode() == "bytecode"): demoteCompiledModuleToBytecode(module) pass_count = 2 # Second, "endless" pass. while not finished: pass_count += 1 if _progress: progress_logger.info("PASS %d:" % pass_count) finished = makeOptimizationPass() Graphs.endGraph(output_filename)
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 # We maintain this globally to make it accessible, pylint: disable=W0603 global graph if Options.shouldCreateGraph(): try: from graphviz import Digraph # pylint: disable=F0401,I0021 graph = Digraph('G') except ImportError: warning("Cannot import graphviz module, no graphing capability.") while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection=function.constraint_collection, new_collection=None) function.constraint_collection = None if not VariableRegistry.complete: VariableRegistry.complete = True finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)
def optimize(): Graphs.startGraph() # First pass. if _progress: info("PASS 1:") makeOptimizationPass(False) Variables.complete = True finished = makeOptimizationPass(False) if Options.isExperimental(): new_roots = ModuleRegistry.root_modules.__class__( ) # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", 'w').write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", 'w').write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), \ (module.getOutputFilename(),restored.getOutputFilename()) # The variable versions give diffs. if False: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff(text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded") for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal() # Demote to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneUserModules(): if module.isPythonShlibModule(): continue if module.mode == "bytecode": demoteCompiledModuleToBytecode(module) if _progress: info("PASS 2 ... :") # Second, "endless" pass. while not finished: finished = makeOptimizationPass(True) Graphs.endGraph()
def optimize(): # This is somewhat complex with many cases, pylint: disable=R0912 # We maintain this globally to make it accessible, pylint: disable=W0603 global graph if Options.shouldCreateGraph(): try: from graphviz import Digraph # pylint: disable=F0401,I0021 graph = Digraph('G') except ImportError: warning("Cannot import graphviz module, no graphing capability.") while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = MemoryUsage.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False # Unregister collection traces from now unused code. for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): for function in current_module.getUnusedFunctions(): VariableRegistry.updateFromCollection( old_collection = function.constraint_collection, new_collection = None ) function.constraint_collection = None if VariableRegistry.considerCompletion(): finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break if graph is not None: graph.engine = "dot" graph.graph_attr["rankdir"] = "TB" graph.render("something.dot") printLine(graph.source)
def makeOptimizationPass(): """Make a single pass for optimization, indication potential completion.""" # Controls complex optimization, pylint: disable=too-many-branches finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=function.trace_collection, new_collection=None, source_ref=function.getSourceReference(), ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): if optimizeVariables(current_module): finished = False used_functions = current_module.getUsedFunctions() for unused_function in current_module.getUnusedFunctions(): unused_function.trace_collection = None used_functions = tuple( function for function in current_module.getFunctions() if function in used_functions) current_module.setFunctions(used_functions) if Variables.complete: if optimizeLocalsDictsHandles(): finished = False return finished
def optimize(): Graphs.startGraph() # First pass. if _progress: info("PASS 1:") makeOptimizationPass(False) Variables.complete = True finished = makeOptimizationPass(False) if Options.isExperimental(): new_roots = ModuleRegistry.root_modules.__class__() # @UndefinedVariable for module in tuple(ModuleRegistry.getDoneModules()): ModuleRegistry.root_modules.remove(module) if module.isPythonShlibModule(): continue text = module.asXmlText() open("out.xml", 'w').write(text) restored = restoreFromXML(text) retext = restored.asXmlText() open("out2.xml", 'w').write(retext) assert module.getOutputFilename() == restored.getOutputFilename(), \ (module.getOutputFilename(),restored.getOutputFilename()) # The variable versions give diffs. if False: # To manually enable, pylint: disable=W0125 import difflib diff = difflib.unified_diff( text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded" ) for line in diff: printLine(line) new_roots.add(restored) ModuleRegistry.root_modules = new_roots ModuleRegistry.startTraversal() # Demote to bytecode, now that imports had a chance to be resolved, and # dependencies were handled. for module in ModuleRegistry.getDoneUserModules(): if module.isPythonShlibModule(): continue if module.mode == "bytecode": demoteCompiledModuleToBytecode(module) if _progress: info("PASS 2 ... :") # Second, "endless" pass. while not finished: finished = makeOptimizationPass(True) Graphs.endGraph()
def copyDataFiles(dist_dir): """Copy the data files needed for standalone distribution. Args: dist_dir: The distribution folder under creation Notes: This is for data files only, not DLLs or even extension modules, those must be registered as entry points, and would not go through necessary handling if provided like this. """ # Many details to deal with, pylint: disable=too-many-branches,too-many-locals for pattern, dest, arg in Options.getShallIncludeDataFiles(): filenames = resolveShellPatternToFilenames(pattern) if not filenames: inclusion_logger.warning("No match data file to be included: %r" % pattern) for filename in filenames: file_reason = "specified data file %r on command line" % arg rel_path = dest if rel_path.endswith(("/", os.path.sep)): rel_path = os.path.join(rel_path, os.path.basename(filename)) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(filename, rel_path, file_reason), ) for src, dest in Options.getShallIncludeDataDirs(): filenames = getFileList(src) if not filenames: inclusion_logger.warning("No files in directory" % src) for filename in filenames: relative_filename = relpath(filename, src) file_reason = "specified data dir %r on command line" % src rel_path = os.path.join(dest, relative_filename) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(filename, rel_path, file_reason), ) # Cyclic dependency from nuitka import ModuleRegistry for module in ModuleRegistry.getDoneModules(): for plugin, included_datafile in Plugins.considerDataFiles(module): _handleDataFile(dist_dir=dist_dir, tracer=plugin, included_datafile=included_datafile) for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonPackage( ) or module.isUncompiledPythonPackage(): package_name = module.getFullName() match, reason = package_name.matchesToShellPatterns( patterns=Options.getShallIncludePackageData()) if match: package_directory = module.getCompileTimeDirectory() pkg_filenames = getFileList( package_directory, ignore_dirs=("__pycache__", ), ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") + getSharedLibrarySuffixes(), ) if pkg_filenames: file_reason = "package '%s' %s" % (package_name, reason) for pkg_filename in pkg_filenames: rel_path = os.path.join( package_name.asPath(), os.path.relpath(pkg_filename, package_directory), ) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(pkg_filename, rel_path, file_reason), )
def makeOptimizationPass(initial_pass): """ Make a single pass for optimization, indication potential completion. """ # Controls complex optimization, pylint: disable=too-many-branches finished = True ModuleRegistry.startTraversal() if _progress: if initial_pass: info("Initial optimization pass.") else: info("Next global optimization pass.") while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: _traceProgress(current_module) # The tag set is global, so it can react to changes without context. # pylint: disable=global-statement global tag_set tag_set = TagSet() changed = optimizeModule(current_module) if changed: finished = False # Unregister collection traces from now unused code, dropping the trace # collections of functions no longer used. for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): for function in current_module.getUnusedFunctions(): Variables.updateVariablesFromCollection( old_collection=function.trace_collection, new_collection=None ) function.trace_collection = None for current_module in ModuleRegistry.getDoneModules(): if current_module.isCompiledPythonModule(): if optimizeVariables(current_module): finished = False used_functions = current_module.getUsedFunctions() for unused_function in current_module.getUnusedFunctions(): unused_function.trace_collection = None used_functions = tuple( function for function in current_module.getFunctions() if function in used_functions ) current_module.setFunctions(used_functions) if Variables.complete: if optimizeLocalsDictsHandles(): finished = False return finished