Пример #1
0
def optimize():
    # This is somewhat complex with many cases, pylint: disable=R0912

    while True:
        finished = True

        ModuleRegistry.startTraversal()

        while True:
            current_module = ModuleRegistry.nextModule()

            if current_module is None:
                break

            if _progress:
                printLine(
                    """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
                        module_name = current_module.getFullName(),
                        remaining   = ModuleRegistry.remainingCount(),
                        memory      = Utils.getHumanReadableProcessMemoryUsage()
                    )
                )

            if current_module.isPythonShlibModule():
                optimizeShlibModule(current_module)
            else:
                changed = optimizePythonModule(current_module)

                if changed:
                    finished = False

        # Unregister collection traces from now unused code.
        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                for function in current_module.getUnusedFunctions():
                    VariableRegistry.updateFromCollection(
                        old_collection = function.constraint_collection,
                        new_collection = None
                    )

                    function.constraint_collection = None

        if not VariableRegistry.complete:
            VariableRegistry.complete = True

            finished = False

        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                optimizeVariables(current_module)

        if finished:
            break
Пример #2
0
def addIncludedDataFilesFromPackageOptions():
    """Late data files, from plugins and user options that work with packages"""
    # Cyclic dependency
    from nuitka import ModuleRegistry

    for module in ModuleRegistry.getDoneModules():
        if module.isCompiledPythonPackage(
        ) or module.isUncompiledPythonPackage():
            package_name = module.getFullName()

            match, reason = package_name.matchesToShellPatterns(
                patterns=getShallIncludePackageData())

            if match:
                package_directory = module.getCompileTimeDirectory()

                pkg_filenames = getFileList(
                    package_directory,
                    ignore_dirs=("__pycache__", ),
                    ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
                    getSharedLibrarySuffixes(),
                )

                if pkg_filenames:
                    file_reason = "package '%s' %s" % (package_name, reason)

                    for pkg_filename in pkg_filenames:
                        rel_path = os.path.join(
                            package_name.asPath(),
                            os.path.relpath(pkg_filename, package_directory),
                        )

                        addIncludedDataFile(
                            makeIncludedDataFile(
                                source_path=pkg_filename,
                                dest_path=rel_path,
                                reason=file_reason,
                                tracer=options_logger,
                                tags="user,package_data",
                            ))

    # Circular dependency
    from nuitka.plugins.Plugins import Plugins

    # Plugins provide per module through this.
    for module in ModuleRegistry.getDoneModules():
        for included_datafile in Plugins.considerDataFiles(module=module):
            addIncludedDataFile(included_datafile)

    for included_datafile in getIncludedDataFiles():
        if isinstance(included_datafile, (IncludedDataFile)):
            Plugins.onDataFileTags(included_datafile, )
Пример #3
0
def makeOptimizationPass(initial_pass):
    """ Make a single pass for optimization, indication potential completion.

    """
    finished = True

    ModuleRegistry.startTraversal()

    if _progress:
        if initial_pass:
            info("Initial optimization pass.")
        else:
            info("Next global optimization pass.")

    while True:
        current_module = ModuleRegistry.nextModule()

        if current_module is None:
            break

        if _progress:
            _traceProgress(current_module)

        # The tag set is global, so it can react to changes without context.
        # pylint: disable=global-statement
        global tag_set
        tag_set = TagSet()

        changed = optimizeModule(current_module)

        if changed:
            finished = False

    # Unregister collection traces from now unused code, dropping the trace
    # collections of functions no longer used.
    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            for function in current_module.getUnusedFunctions():
                Variables.updateFromCollection(
                    old_collection=function.trace_collection,
                    new_collection=None)

                function.trace_collection = None

    for current_module in ModuleRegistry.getDoneModules():
        if optimizeVariables(current_module):
            finished = False

    return finished
Пример #4
0
def makeOptimizationPass(initial_pass):
    """ Make a single pass for optimization, indication potential completion.

    """
    finished = True

    ModuleRegistry.startTraversal()

    if _progress:
        if initial_pass:
            printLine("Initial optimization pass.")
        else:
            printLine("Next global optimization pass.")

    while True:
        current_module = ModuleRegistry.nextModule()

        if current_module is None:
            break

        if _progress:
            _traceProgress(current_module)

        # The tag set is global, so it can react to changes without context.
        # pylint: disable=W0603
        global tag_set
        tag_set = TagSet()

        changed = optimizeModule(current_module)

        if changed:
            finished = False

    # Unregister collection traces from now unused code, dropping the trace
    # collections of functions no longer used.
    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            for function in current_module.getUnusedFunctions():
                Variables.updateFromCollection(
                    old_collection = function.trace_collection,
                    new_collection = None
                )

                function.trace_collection = None

    for current_module in ModuleRegistry.getDoneModules():
        optimizeVariables(current_module)

    return finished
Пример #5
0
def copyDataFiles(dist_dir):
    """ Copy the data files needed for standalone distribution.

    Args:
        dist_dir: The distribution folder under creation
    Notes:
        This is for data files only, not DLLs or even extension modules,
        those must be registered as entry points, and would not go through
        necessary handling if provided like this.
    """

    # Cyclic dependency
    from nuitka import ModuleRegistry

    for module in ModuleRegistry.getDoneModules():
        for _plugin_name, (source_desc, target_filename) in Plugins.considerDataFiles(
            module
        ):
            target_filename = os.path.join(dist_dir, target_filename)
            assert isPathBelow(dist_dir, target_filename)

            makePath(os.path.dirname(target_filename))

            if inspect.isfunction(source_desc):
                content = source_desc(target_filename)

                if content is not None:  # support creation of empty directories
                    with open(
                        target_filename, "wb" if type(content) is bytes else "w"
                    ) as output:
                        output.write(content)
            else:
                shutil.copy2(source_desc, target_filename)
Пример #6
0
def _checkXMLPersistence():
    new_roots = ModuleRegistry.root_modules.__class__()  # @UndefinedVariable

    for module in tuple(ModuleRegistry.getDoneModules()):
        ModuleRegistry.root_modules.remove(module)

        if module.isPythonShlibModule():
            continue

        text = module.asXmlText()
        open("out.xml", "w").write(text)
        restored = restoreFromXML(text)
        retext = restored.asXmlText()
        open("out2.xml", "w").write(retext)

        assert module.getOutputFilename() == restored.getOutputFilename(), (
            module.getOutputFilename(),
            restored.getOutputFilename(),
        )

        # The variable versions give diffs.
        if True:  # To manually enable, pylint: disable=W0125
            import difflib

            diff = difflib.unified_diff(text.splitlines(), retext.splitlines(),
                                        "xml orig", "xml reloaded")
            for line in diff:
                printLine(line)

        new_roots.add(restored)

    ModuleRegistry.root_modules = new_roots
    ModuleRegistry.startTraversal()
Пример #7
0
def optimize():
    while True:
        finished = True
        ModuleRegistry.startTraversal()

        while True:
            current_module = ModuleRegistry.nextModule()

            if current_module is None:
                break

            if _progress:
                printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
                    module_name=current_module.getFullName(),
                    remaining=ModuleRegistry.remainingCount(),
                    memory=Utils.getHumanReadableProcessMemoryUsage()))

            if current_module.isPythonShlibModule():
                optimizeShlibModule(current_module)
            else:
                changed = optimizePythonModule(current_module)

                if changed:
                    finished = False

        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                optimizeVariables(current_module)

        if finished:
            break
Пример #8
0
def _checkXMLPersistence():
    for module in ModuleRegistry.getDoneModules():
        if not module.isCompiledPythonModule():
            continue

        text = module.asXmlText()
        with open(_getCacheFilename(module), "w") as f:
            f.write(text)
Пример #9
0
def makeOptimizationPass():
    """Make a single pass for optimization, indication potential completion."""

    # Controls complex optimization

    finished = True

    ModuleRegistry.startTraversal()

    _restartProgress()

    while True:
        current_module = ModuleRegistry.nextModule()

        if current_module is None:
            break

        _traceProgress(current_module)

        # The tag set is global, so it can react to changes without context.
        # pylint: disable=global-statement
        global tag_set
        tag_set = TagSet()

        changed = optimizeModule(current_module)

        if changed:
            finished = False

    # Unregister collection traces from now unused code, dropping the trace
    # collections of functions no longer used. This must be done after global
    # optimization due to cross module usages.
    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            for unused_function in current_module.getUnusedFunctions():
                Variables.updateVariablesFromCollection(
                    old_collection=unused_function.trace_collection,
                    new_collection=None,
                    source_ref=unused_function.getSourceReference(),
                )

                unused_function.trace_collection = None

            used_functions = tuple(
                function
                for function in current_module.subnode_functions
                if function in current_module.getUsedFunctions()
            )

            current_module.setChild("functions", used_functions)

    _endProgress()

    return finished
Пример #10
0
def makeOptimizationPass():
    """Make a single pass for optimization, indication potential completion."""

    # Controls complex optimization

    finished = True

    ModuleRegistry.startTraversal()

    _restartProgress()

    while True:
        current_module = ModuleRegistry.nextModule()

        if current_module is None:
            # TODO: Internal module seems to cause extra passes.
            # optimizeModule(getInternalModule())
            break

        _traceProgressModuleStart(current_module)

        changed = optimizeModule(current_module)

        _traceProgressModuleEnd(current_module)

        if changed:
            finished = False

    # Unregister collection traces from now unused code, dropping the trace
    # collections of functions no longer used. This must be done after global
    # optimization due to cross module usages.
    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            for unused_function in current_module.getUnusedFunctions():
                Variables.updateVariablesFromCollection(
                    old_collection=unused_function.trace_collection,
                    new_collection=None,
                    source_ref=unused_function.getSourceReference(),
                )

                unused_function.trace_collection = None

            used_functions = tuple(
                function
                for function in current_module.subnode_functions
                if function in current_module.getUsedFunctions()
            )

            current_module.setChild("functions", used_functions)

    _endProgress()

    return finished
Пример #11
0
def optimizeModules(output_filename):
    Graphs.startGraph()

    finished = makeOptimizationPass()

    # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and
    # dependencies were handled.
    for module in ModuleRegistry.getDoneModules():
        if (module.isCompiledPythonModule()
                and module.getCompilationMode() == "bytecode"):
            demoteCompiledModuleToBytecode(module)

    # Second, "endless" pass.
    while not finished:
        finished = makeOptimizationPass()

    Graphs.endGraph(output_filename)
Пример #12
0
def optimize(output_filename):
    Graphs.startGraph()

    finished = makeOptimizationPass()

    if Options.isExperimental("check_xml_persistence"):
        _checkXMLPersistence()

    # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and
    # dependencies were handled.
    for module in ModuleRegistry.getDoneModules():
        if (module.isCompiledPythonModule()
                and module.getCompilationMode() == "bytecode"):
            demoteCompiledModuleToBytecode(module)

    global pass_count  # Singleton, pylint: disable=global-statement

    # Second, "endless" pass.
    while not finished:
        finished = makeOptimizationPass()

    Graphs.endGraph(output_filename)
Пример #13
0
def _checkXMLPersistence():
    new_roots = ModuleRegistry.root_modules.__class__()  # @UndefinedVariable

    for module in tuple(ModuleRegistry.getDoneModules()):
        ModuleRegistry.root_modules.remove(module)

        if module.isPythonShlibModule():
            continue

        text = module.asXmlText()
        with open("out.xml", "w") as f:
            f.write(text)
        restored = restoreFromXML(text)
        retext = restored.asXmlText()
        with open("out2.xml", "w") as f:
            f.write(retext)

        assert module.getOutputFilename() == restored.getOutputFilename(), (
            module.getOutputFilename(),
            restored.getOutputFilename(),
        )

        # The variable versions give diffs.
        if True:  # To manually enable, pylint: disable=W0125
            import difflib

            diff = difflib.unified_diff(
                text.splitlines(), retext.splitlines(), "xml orig", "xml reloaded"
            )
            for line in diff:
                printLine(line)

        new_roots.add(restored)

    ModuleRegistry.root_modules = new_roots
    ModuleRegistry.startTraversal()
Пример #14
0
def optimize(output_filename):
    Graphs.startGraph()

    # First pass.
    if _progress:
        progress_logger.info("PASS 1:")

    makeOptimizationPass()
    Variables.complete = True

    if _progress:
        progress_logger.info("PASS 2:")

    finished = makeOptimizationPass()

    if Options.isExperimental("check_xml_persistence"):
        _checkXMLPersistence()

    # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and
    # dependencies were handled.
    for module in ModuleRegistry.getDoneModules():
        if (module.isCompiledPythonModule()
                and module.getCompilationMode() == "bytecode"):
            demoteCompiledModuleToBytecode(module)

    pass_count = 2
    # Second, "endless" pass.
    while not finished:
        pass_count += 1

        if _progress:
            progress_logger.info("PASS %d:" % pass_count)

        finished = makeOptimizationPass()

    Graphs.endGraph(output_filename)
Пример #15
0
def optimize():
    while True:
        finished = True
        ModuleRegistry.startTraversal()

        while True:
            current_module = ModuleRegistry.nextModule()

            if current_module is None:
                break

            if _progress:
                printLine(
                    """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
                        module_name = current_module.getFullName(),
                        remaining   = ModuleRegistry.remainingCount(),
                        memory      = Utils.getHumanReadableProcessMemoryUsage()
                    )
                )

            if current_module.isPythonShlibModule():
                optimizeShlibModule(current_module)
            else:
                changed = optimizePythonModule(current_module)

                if changed:
                    finished = False

        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                optimizeVariables(current_module)

        if finished:
            break
Пример #16
0
def optimize():
    # This is somewhat complex with many cases, pylint: disable=R0912

    # We maintain this globally to make it accessible, pylint: disable=W0603
    global graph

    if Options.shouldCreateGraph():

        try:
            from graphviz import Digraph  # pylint: disable=F0401,I0021
            graph = Digraph('G')
        except ImportError:
            warning("Cannot import graphviz module, no graphing capability.")

    while True:
        finished = True

        ModuleRegistry.startTraversal()

        while True:
            current_module = ModuleRegistry.nextModule()

            if current_module is None:
                break

            if _progress:
                printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
                    module_name=current_module.getFullName(),
                    remaining=ModuleRegistry.remainingCount(),
                    memory=Utils.getHumanReadableProcessMemoryUsage()))

            if current_module.isPythonShlibModule():
                optimizeShlibModule(current_module)
            else:
                changed = optimizePythonModule(current_module)

                if changed:
                    finished = False

        # Unregister collection traces from now unused code.
        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                for function in current_module.getUnusedFunctions():
                    VariableRegistry.updateFromCollection(
                        old_collection=function.constraint_collection,
                        new_collection=None)

                    function.constraint_collection = None

        if not VariableRegistry.complete:
            VariableRegistry.complete = True

            finished = False

        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                optimizeVariables(current_module)

        if finished:
            break

    if graph is not None:
        graph.engine = "dot"
        graph.graph_attr["rankdir"] = "TB"
        graph.render("something.dot")

        printLine(graph.source)
Пример #17
0
def optimize():
    Graphs.startGraph()

    # First pass.
    if _progress:
        info("PASS 1:")

    makeOptimizationPass(False)
    Variables.complete = True

    finished = makeOptimizationPass(False)

    if Options.isExperimental():
        new_roots = ModuleRegistry.root_modules.__class__(
        )  # @UndefinedVariable

        for module in tuple(ModuleRegistry.getDoneModules()):
            ModuleRegistry.root_modules.remove(module)

            if module.isPythonShlibModule():
                continue

            text = module.asXmlText()
            open("out.xml", 'w').write(text)
            restored = restoreFromXML(text)
            retext = restored.asXmlText()
            open("out2.xml", 'w').write(retext)

            assert module.getOutputFilename() == restored.getOutputFilename(), \
               (module.getOutputFilename(),restored.getOutputFilename())

            # The variable versions give diffs.
            if False:  # To manually enable, pylint: disable=W0125
                import difflib
                diff = difflib.unified_diff(text.splitlines(),
                                            retext.splitlines(), "xml orig",
                                            "xml reloaded")
                for line in diff:
                    printLine(line)

            new_roots.add(restored)

        ModuleRegistry.root_modules = new_roots
        ModuleRegistry.startTraversal()

    # Demote to bytecode, now that imports had a chance to be resolved, and
    # dependencies were handled.
    for module in ModuleRegistry.getDoneUserModules():
        if module.isPythonShlibModule():
            continue

        if module.mode == "bytecode":
            demoteCompiledModuleToBytecode(module)

    if _progress:
        info("PASS 2 ... :")

    # Second, "endless" pass.
    while not finished:
        finished = makeOptimizationPass(True)

    Graphs.endGraph()
Пример #18
0
def optimize():
    # This is somewhat complex with many cases, pylint: disable=R0912

    # We maintain this globally to make it accessible, pylint: disable=W0603
    global graph

    if Options.shouldCreateGraph():

        try:
            from graphviz import Digraph # pylint: disable=F0401,I0021
            graph = Digraph('G')
        except ImportError:
            warning("Cannot import graphviz module, no graphing capability.")

    while True:
        finished = True

        ModuleRegistry.startTraversal()

        while True:
            current_module = ModuleRegistry.nextModule()

            if current_module is None:
                break

            if _progress:
                printLine(
                    """\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
                        module_name = current_module.getFullName(),
                        remaining   = ModuleRegistry.remainingCount(),
                        memory      = MemoryUsage.getHumanReadableProcessMemoryUsage()
                    )
                )

            if current_module.isPythonShlibModule():
                optimizeShlibModule(current_module)
            else:
                changed = optimizePythonModule(current_module)

                if changed:
                    finished = False

        # Unregister collection traces from now unused code.
        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                for function in current_module.getUnusedFunctions():
                    VariableRegistry.updateFromCollection(
                        old_collection = function.constraint_collection,
                        new_collection = None
                    )

                    function.constraint_collection = None

        if VariableRegistry.considerCompletion():
            finished = False

        for current_module in ModuleRegistry.getDoneModules():
            if not current_module.isPythonShlibModule():
                optimizeVariables(current_module)

        if finished:
            break


    if graph is not None:
        graph.engine = "dot"
        graph.graph_attr["rankdir"] = "TB"
        graph.render("something.dot")

        printLine(graph.source)
Пример #19
0
def makeOptimizationPass():
    """Make a single pass for optimization, indication potential completion."""
    # Controls complex optimization, pylint: disable=too-many-branches

    finished = True

    ModuleRegistry.startTraversal()

    while True:
        current_module = ModuleRegistry.nextModule()

        if current_module is None:
            break

        if _progress:
            _traceProgress(current_module)

        # The tag set is global, so it can react to changes without context.
        # pylint: disable=global-statement
        global tag_set
        tag_set = TagSet()

        changed = optimizeModule(current_module)

        if changed:
            finished = False

    # Unregister collection traces from now unused code, dropping the trace
    # collections of functions no longer used.
    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            for function in current_module.getUnusedFunctions():
                Variables.updateVariablesFromCollection(
                    old_collection=function.trace_collection,
                    new_collection=None,
                    source_ref=function.getSourceReference(),
                )

                function.trace_collection = None

    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            if optimizeVariables(current_module):
                finished = False

            used_functions = current_module.getUsedFunctions()

            for unused_function in current_module.getUnusedFunctions():
                unused_function.trace_collection = None

            used_functions = tuple(
                function for function in current_module.getFunctions()
                if function in used_functions)

            current_module.setFunctions(used_functions)

    if Variables.complete:
        if optimizeLocalsDictsHandles():
            finished = False

    return finished
Пример #20
0
def optimize():
    Graphs.startGraph()

    # First pass.
    if _progress:
        info("PASS 1:")

    makeOptimizationPass(False)
    Variables.complete = True

    finished = makeOptimizationPass(False)

    if Options.isExperimental():
        new_roots = ModuleRegistry.root_modules.__class__()  # @UndefinedVariable

        for module in tuple(ModuleRegistry.getDoneModules()):
            ModuleRegistry.root_modules.remove(module)

            if module.isPythonShlibModule():
                continue

            text = module.asXmlText()
            open("out.xml", 'w').write(text)
            restored = restoreFromXML(text)
            retext = restored.asXmlText()
            open("out2.xml", 'w').write(retext)

            assert module.getOutputFilename() == restored.getOutputFilename(), \
               (module.getOutputFilename(),restored.getOutputFilename())

            # The variable versions give diffs.
            if False: # To manually enable, pylint: disable=W0125
                import difflib
                diff = difflib.unified_diff(
                    text.splitlines(),
                    retext.splitlines(),
                    "xml orig",
                    "xml reloaded"
                )
                for line in diff:
                    printLine(line)

            new_roots.add(restored)

        ModuleRegistry.root_modules = new_roots
        ModuleRegistry.startTraversal()

    # Demote to bytecode, now that imports had a chance to be resolved, and
    # dependencies were handled.
    for module in ModuleRegistry.getDoneUserModules():
        if module.isPythonShlibModule():
            continue

        if module.mode == "bytecode":
            demoteCompiledModuleToBytecode(module)

    if _progress:
        info("PASS 2 ... :")

    # Second, "endless" pass.
    while not finished:
        finished = makeOptimizationPass(True)

    Graphs.endGraph()
Пример #21
0
def copyDataFiles(dist_dir):
    """Copy the data files needed for standalone distribution.

    Args:
        dist_dir: The distribution folder under creation
    Notes:
        This is for data files only, not DLLs or even extension modules,
        those must be registered as entry points, and would not go through
        necessary handling if provided like this.
    """

    # Many details to deal with, pylint: disable=too-many-branches,too-many-locals

    for pattern, dest, arg in Options.getShallIncludeDataFiles():
        filenames = resolveShellPatternToFilenames(pattern)

        if not filenames:
            inclusion_logger.warning("No match data file to be included: %r" %
                                     pattern)

        for filename in filenames:
            file_reason = "specified data file %r on command line" % arg

            rel_path = dest

            if rel_path.endswith(("/", os.path.sep)):
                rel_path = os.path.join(rel_path, os.path.basename(filename))

            _handleDataFile(
                dist_dir,
                inclusion_logger,
                makeIncludedDataFile(filename, rel_path, file_reason),
            )

    for src, dest in Options.getShallIncludeDataDirs():
        filenames = getFileList(src)

        if not filenames:
            inclusion_logger.warning("No files in directory" % src)

        for filename in filenames:
            relative_filename = relpath(filename, src)

            file_reason = "specified data dir %r on command line" % src

            rel_path = os.path.join(dest, relative_filename)

            _handleDataFile(
                dist_dir,
                inclusion_logger,
                makeIncludedDataFile(filename, rel_path, file_reason),
            )

    # Cyclic dependency
    from nuitka import ModuleRegistry

    for module in ModuleRegistry.getDoneModules():
        for plugin, included_datafile in Plugins.considerDataFiles(module):
            _handleDataFile(dist_dir=dist_dir,
                            tracer=plugin,
                            included_datafile=included_datafile)

    for module in ModuleRegistry.getDoneModules():
        if module.isCompiledPythonPackage(
        ) or module.isUncompiledPythonPackage():
            package_name = module.getFullName()

            match, reason = package_name.matchesToShellPatterns(
                patterns=Options.getShallIncludePackageData())

            if match:
                package_directory = module.getCompileTimeDirectory()

                pkg_filenames = getFileList(
                    package_directory,
                    ignore_dirs=("__pycache__", ),
                    ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
                    getSharedLibrarySuffixes(),
                )

                if pkg_filenames:
                    file_reason = "package '%s' %s" % (package_name, reason)

                    for pkg_filename in pkg_filenames:
                        rel_path = os.path.join(
                            package_name.asPath(),
                            os.path.relpath(pkg_filename, package_directory),
                        )

                        _handleDataFile(
                            dist_dir,
                            inclusion_logger,
                            makeIncludedDataFile(pkg_filename, rel_path,
                                                 file_reason),
                        )
Пример #22
0
def makeOptimizationPass(initial_pass):
    """ Make a single pass for optimization, indication potential completion.

    """
    # Controls complex optimization, pylint: disable=too-many-branches

    finished = True

    ModuleRegistry.startTraversal()

    if _progress:
        if initial_pass:
            info("Initial optimization pass.")
        else:
            info("Next global optimization pass.")

    while True:
        current_module = ModuleRegistry.nextModule()

        if current_module is None:
            break

        if _progress:
            _traceProgress(current_module)

        # The tag set is global, so it can react to changes without context.
        # pylint: disable=global-statement
        global tag_set
        tag_set = TagSet()

        changed = optimizeModule(current_module)

        if changed:
            finished = False

    # Unregister collection traces from now unused code, dropping the trace
    # collections of functions no longer used.
    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            for function in current_module.getUnusedFunctions():
                Variables.updateVariablesFromCollection(
                    old_collection=function.trace_collection, new_collection=None
                )

                function.trace_collection = None

    for current_module in ModuleRegistry.getDoneModules():
        if current_module.isCompiledPythonModule():
            if optimizeVariables(current_module):
                finished = False

            used_functions = current_module.getUsedFunctions()

            for unused_function in current_module.getUnusedFunctions():
                unused_function.trace_collection = None

            used_functions = tuple(
                function
                for function in current_module.getFunctions()
                if function in used_functions
            )

            current_module.setFunctions(used_functions)

    if Variables.complete:
        if optimizeLocalsDictsHandles():
            finished = False

    return finished