Beispiel #1
0
def checkPluginPath(plugin_filename, module_package):
    plugin_filename = os.path.normpath(plugin_filename)

    if Options.isShowInclusion():
        recursion_logger.info("Checking top level plug-in path %s %s" %
                              (plugin_filename, module_package))

    plugin_info = considerFilename(module_filename=plugin_filename)

    if plugin_info is not None:
        # File or package makes a difference, handle that
        if os.path.isfile(plugin_info[0]) or Importing.isPackageDir(
                plugin_info[0]):
            checkPluginSinglePath(plugin_filename,
                                  module_package=module_package)
        elif os.path.isdir(plugin_info[0]):
            for sub_path, sub_filename in listDir(plugin_info[0]):
                assert sub_filename != "__init__.py"

                if Importing.isPackageDir(sub_path) or sub_path.endswith(
                        ".py"):
                    checkPluginSinglePath(sub_path, module_package=None)
        else:
            recursion_logger.warning("Failed to include module from %r." %
                                     plugin_info[0])
    else:
        recursion_logger.warning("Failed to recurse to directory %r." %
                                 plugin_filename)
Beispiel #2
0
def checkPluginPath(plugin_filename, module_package):
    debug(
        "Checking top level plug-in path %s %s",
        plugin_filename,
        module_package
    )

    plugin_info = considerFilename(
        module_package  = module_package,
        module_filename = plugin_filename
    )

    if plugin_info is not None:
        # File or package makes a difference, handle that
        if Utils.isFile(plugin_info[0]) or \
           Importing.isPackageDir(plugin_info[0]):
            _checkPluginPath(plugin_filename, module_package)
        elif Utils.isDir(plugin_info[0]):
            for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
                assert sub_filename != "__init__.py"

                if Importing.isPackageDir(sub_path) or \
                   sub_path.endswith(".py"):
                    _checkPluginPath(sub_path, None)
        else:
            warning("Failed to include module from '%s'.", plugin_info[0])
    else:
        warning("Failed to recurse to directory '%s'.", plugin_filename)
Beispiel #3
0
def checkPluginPath(plugin_filename, module_package):
    debug(
        "Checking top level plug-in path %s %s",
        plugin_filename,
        module_package
    )

    plugin_info = considerFilename(
        module_filename = plugin_filename
    )

    if plugin_info is not None:
        # File or package makes a difference, handle that
        if Utils.isFile(plugin_info[0]) or \
           Importing.isPackageDir(plugin_info[0]):
            _checkPluginPath(plugin_filename, module_package)
        elif Utils.isDir(plugin_info[0]):
            for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
                assert sub_filename != "__init__.py"

                if Importing.isPackageDir(sub_path) or \
                   sub_path.endswith(".py"):
                    _checkPluginPath(sub_path, None)
        else:
            warning("Failed to include module from '%s'.", plugin_info[0])
    else:
        warning("Failed to recurse to directory '%s'.", plugin_filename)
Beispiel #4
0
def considerUsedModules(module, signal_change):
    for (
            used_module_name,
            used_module_filename,
            finding,
            level,
            source_ref,
    ) in module.getUsedModules():
        if finding == "not-found":
            Importing.warnAbout(
                importing=module,
                source_ref=source_ref,
                module_name=used_module_name,
                level=level,
            )

        try:
            if used_module_filename is None:
                continue

            _module_name, module_kind = getModuleNameAndKindFromFilename(
                used_module_filename)

            decision, reason = decideRecursion(
                module_filename=used_module_filename,
                module_name=used_module_name,
                module_kind=module_kind,
            )

            if decision:
                _addParentPackageUsages(
                    using_module=module,
                    module_name=used_module_name,
                    signal_change=signal_change,
                    source_ref=source_ref,
                )

                used_module = recurseTo(
                    signal_change=signal_change,
                    module_name=used_module_name,
                    module_filename=used_module_filename,
                    module_kind=module_kind,
                    reason=reason,
                )

                addUsedModule(
                    module=used_module,
                    using_module=module,
                    usage_tag="import",
                    reason=reason,
                    source_ref=source_ref,
                )
        except NuitkaForbiddenImportEncounter as e:
            recursion_logger.sysexit(
                "Error, forbidden import of '%s' in module '%s' at '%s' encountered."
                %
                (e, module.getFullName().asString(), source_ref.getAsString()))

    Plugins.considerImplicitImports(module=module, signal_change=signal_change)
Beispiel #5
0
def buildMainModuleTree(filename, is_main):
    # Detect to be frozen modules if any, so we can consider to not follow
    # to them.

    if is_main:
        # TODO: Doesn't work for deeply nested packages at all.
        if Options.hasPythonFlagPackageMode():
            module_name = ModuleName(os.path.basename(filename) + ".__main__")
        else:
            module_name = ModuleName("__main__")
    else:
        module_name = Importing.getModuleNameAndKindFromFilename(filename)[0]

    module, _added = buildModule(
        module_name=module_name,
        module_filename=filename,
        source_code=None,
        is_top=True,
        is_main=is_main,
        is_extension=False,
        is_fake=False,
        hide_syntax_error=False,
    )

    if is_main and Options.isStandaloneMode():
        module.setEarlyModules(detectEarlyImports())

    # Main modules do not get added to the import cache, but plugins get to see it.
    if module.isMainModule():
        Plugins.onModuleDiscovered(module)
    else:
        addImportedModule(imported_module=module)

    return module
Beispiel #6
0
    def locateModule(module_name, module_package, source_ref):
        from nuitka.importing import Importing

        _module_package, module_filename, _finding = Importing.findModule(
            source_ref=source_ref, module_name=module_name, parent_package=module_package, level=-1, warn=True
        )

        return module_filename
Beispiel #7
0
    def locateModule(importing, module_name, module_package):
        from nuitka.importing import Importing

        _module_package, module_filename, _finding = Importing.findModule(
            importing=importing,
            module_name=module_name,
            parent_package=module_package,
            level=-1,
            warn=True)

        return module_filename
Beispiel #8
0
    def _considerImplicitImports(plugin, module):
        from nuitka.importing import Importing

        result = []

        def iterateModuleNames(value):
            for v in value:
                if type(v) in (tuple, list):
                    plugin.sysexit(
                        "Plugin %r needs to be change to only return modules names, not %r (for %s)"
                        % (plugin.plugin_name, v,
                           module.getFullName().asString()))

                if inspect.isgenerator(v):
                    for w in iterateModuleNames(v):
                        yield w

                    return

                if not checkModuleName(v):
                    plugin.sysexit(
                        "Plugin %r returned an invalid module name, not %r (for %s)"
                        % (plugin, v, module.getFullName().asString()))

                yield ModuleName(v)

        for full_name in iterateModuleNames(plugin.getImplicitImports(module)):
            try:
                _module_name, module_filename, _finding = Importing.locateModule(
                    module_name=full_name,
                    parent_package=None,
                    level=0,
                )
            except Exception:
                plugin.warning(
                    "Problem locating '%s' for implicit imports of '%s'." %
                    (module.getFullName(), full_name))
                raise

            if module_filename is None:
                if Options.isShowInclusion():
                    plugin.info(
                        "Implicit module '%s' suggested for '%s' not found." %
                        (full_name, module.getFullName()))

                continue

            result.append((full_name, module_filename))

        if result:
            plugin.info("Implicit dependencies of module '%s' added '%s'." %
                        (module.getFullName(), ",".join(r[0] for r in result)))

        return result
Beispiel #9
0
    def _considerImplicitImports(plugin, module):
        from nuitka.importing import Importing

        result = []

        for full_name in plugin.getImplicitImports(module):
            if type(full_name) in (tuple, list):
                raise NuitkaPluginError(
                    "Plugin %r needs to be change to only return modules names, not %r"
                    % (plugin, full_name)
                )

            full_name = ModuleName(full_name)

            try:
                _module_package, module_filename, _finding = Importing.findModule(
                    importing=module,
                    module_name=full_name,
                    parent_package=None,
                    level=-1,
                    warn=False,
                )

                module_filename = plugin.locateModule(
                    importing=module, module_name=full_name
                )
            except Exception:
                plugin.warning(
                    "Problem locating '%s' for implicit imports of '%s'."
                    % (module.getFullName(), full_name)
                )
                raise

            if module_filename is None:
                if Options.isShowInclusion():
                    plugin.info(
                        "Implicit module '%s' suggested for '%s' not found."
                        % (full_name, module.getFullName())
                    )

                continue

            result.append((full_name, module_filename))

        if result:
            plugin.info(
                "Implicit dependencies of module '%s' added '%s'."
                % (module.getFullName(), ",".join(r[0] for r in result))
            )

        return result
Beispiel #10
0
    def attemptRecursion(self):
        # Make sure the package is recursed to.
        from nuitka.importing import Recursion

        # Return the list of newly added modules.
        result = []

        if self.package_name is not None and self.package is None:
            package_package, package_filename, _finding = \
              Importing.findModule(
                source_ref     = self.getSourceReference(),
                module_name    = self.package_name,
                parent_package = None,
                level          = 1,
                warn           = Utils.python_version < 330
            )

            # TODO: Temporary, if we can't find the package for Python3.3 that
            # is semi-OK, maybe.
            if Utils.python_version >= 330 and not package_filename:
                return []

            imported_module, is_added = Recursion.recurseTo(
                module_package=package_package,
                module_filename=package_filename,
                module_relpath=Utils.relpath(package_filename),
                module_kind="py",
                reason="Containing package of recursed module.",
            )

            self.package = imported_module

            if is_added:
                result.append(imported_module)

        if self.package:
            from nuitka.ModuleRegistry import addUsedModule

            addUsedModule(self.package)

            #            print "Recursed to package", self.package_name
            result.extend(self.package.attemptRecursion())

        return result
Beispiel #11
0
    def attemptRecursion(self):
        # Make sure the package is recursed to.
        from nuitka.importing import Recursion

        # Return the list of newly added modules.
        result = []

        if self.package_name is not None and self.package is None:
            package_package, package_filename, _finding = \
              Importing.findModule(
                source_ref     = self.getSourceReference(),
                module_name    = self.package_name,
                parent_package = None,
                level          = 1,
                warn           = Utils.python_version < 330
            )

            # TODO: Temporary, if we can't find the package for Python3.3 that
            # is semi-OK, maybe.
            if Utils.python_version >= 330 and not package_filename:
                return []

            imported_module, is_added = Recursion.recurseTo(
                module_package  = package_package,
                module_filename = package_filename,
                module_relpath  = Utils.relpath(package_filename),
                module_kind     = "py",
                reason          = "Containing package of recursed module.",
            )

            self.package = imported_module

            if is_added:
                result.append(imported_module)

        if self.package:
            from nuitka.ModuleRegistry import addUsedModule

            addUsedModule(self.package)

#            print "Recursed to package", self.package_name
            result.extend(self.package.attemptRecursion())

        return result
Beispiel #12
0
    def locateModule(importing, module_name):
        """Provide a filename / -path for a to-be-imported module.

        Args:
            importing: module object that asked for it (tracing only)
            module_name: (str or ModuleName) full name of module
        Returns:
            filename for module
        """
        from nuitka.importing import Importing

        _module_package, module_filename, _finding = Importing.findModule(
            importing=importing,
            module_name=ModuleName(module_name),
            parent_package=None,
            level=-1,
            warn=False,
        )

        return module_filename
Beispiel #13
0
    def locateModule(importing, module_name, warn):
        """ Provide a filename / -path for a to-be-imported module.

        Args:
            importing: module object
            module_name: (str or ModuleName) full name of module
            warn: (bool) True if required module
        Returns:
            filename for module
        """
        from nuitka.importing import Importing

        _module_package, module_filename, _finding = Importing.findModule(
            importing=importing,
            module_name=ModuleName(module_name),
            parent_package=None,
            level=-1,
            warn=warn,
        )

        return module_filename
Beispiel #14
0
def _loadUncompiledModuleFromCache(module_name, is_package, source_code,
                                   source_ref):
    result = makeUncompiledPythonModule(
        module_name=module_name,
        filename=source_ref.getFilename(),
        bytecode=demoteSourceCodeToBytecode(
            module_name=module_name,
            source_code=source_code,
            filename=source_ref.getFilename(),
        ),
        user_provided=False,
        technical=False,
        is_package=is_package,
    )

    used_modules = OrderedSet()

    for used_module_name, line_number in getCachedImportedModulesNames(
            module_name=module_name, source_code=source_code):
        _module_name, module_filename, finding = Importing.locateModule(
            module_name=used_module_name,
            parent_package=None,
            level=0,
        )

        assert _module_name == used_module_name

        used_modules.add((
            used_module_name,
            module_filename,
            finding,
            0,
            source_ref.atLineNumber(line_number),
        ))

    # assert not is_package, (module_name, used_modules, result, result.getCompileTimeFilename())

    result.setUsedModules(used_modules)

    return result
Beispiel #15
0
def decideCompilationMode(is_top, module_name, for_pgo):
    """Decide the compilation mode for a module.

    module_name - The module to decide compilation mode for.
    for_pgo - consider PGO information or not
    """

    result = Plugins.decideCompilation(module_name)

    # Cannot change mode of __main__ to bytecode, that is not going
    # to work currently.
    if result == "bytecode" and is_top:
        plugins_logger.warning("""\
Ignoring plugin decision to compile top level package '%s'
as bytecode, the extension module entry point is technically
required to compiled.""" % module_name)
        result = "compiled"

    # Include all of standard library as bytecode, for now. We need to identify
    # which ones really need that.
    if not is_top:
        module_filename = Importing.locateModule(module_name=module_name,
                                                 parent_package=None,
                                                 level=0)[1]

        if module_filename is not None and isStandardLibraryPath(
                module_filename):
            result = "bytecode"

    # Plugins need to win over PGO, as they might know it better
    if result is None and not for_pgo:
        result = decideCompilationFromPGO(module_name=module_name)

    # Default if neither plugins nor PGO have expressed an opinion
    if result is None:
        result = "compiled"

    return result
Beispiel #16
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases
    # pylint: disable=too-many-branches,too-many-locals,too-many-statements

    assert package is None or type(package) is ModuleName
    assert filename is not None

    if is_main and os.path.isdir(filename):
        source_filename = os.path.join(filename, "__main__.py")

        if not os.path.isfile(source_filename):
            sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
                             (os.path.basename(sys.argv[0]), filename))
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if os.path.isfile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(filename=filename)

        if is_main:
            module_name = ModuleName("__main__")
        else:
            # Derive module name from filename.
            module_name = os.path.basename(filename)
            if is_shlib:
                module_name = module_name.split(".")[0]
            elif module_name.endswith(".py"):
                module_name = module_name[:-3]

            if "." in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" %
                    (module_name))

                sys.exit(2)

            module_name = ModuleName.makeModuleNameInPackage(
                module_name, package)

        if is_shlib:
            result = PythonShlibModule(module_name=module_name,
                                       source_ref=source_ref)
            source_code = None
        else:
            source_code = readSourceCodeFromFilename(
                module_name=module_name, source_filename=source_filename)

            if is_main:
                result = PythonMainModule(
                    main_added=main_added,
                    mode=decideCompilationMode(False, module_name, source_ref),
                    future_spec=None,
                    source_ref=source_ref,
                )

                checkPythonVersionFromCode(source_code)
            else:
                mode = decideCompilationMode(is_top, module_name, source_ref)

                if (mode == "bytecode" and not is_top
                        and hasCachedImportedModulesNames(
                            module_name, source_code)):

                    optimization_logger.info("%r is included as bytecode." %
                                             (module_name.asString()))
                    result = UncompiledPythonModule(
                        module_name=module_name,
                        filename=filename,
                        bytecode=demoteSourceCodeToBytecode(
                            module_name=module_name,
                            source_code=source_code,
                            filename=filename,
                        ),
                        source_ref=source_ref,
                        user_provided=False,
                        technical=False,
                    )

                    used_modules = OrderedSet()

                    for used_module_name in getCachedImportedModulesNames(
                            module_name=module_name, source_code=source_code):
                        (
                            _module_package,
                            module_filename,
                            _finding,
                        ) = Importing.findModule(
                            importing=result,
                            module_name=used_module_name,
                            parent_package=None,
                            level=-1,
                            warn=False,
                        )

                        used_modules.add((used_module_name,
                                          os.path.relpath(module_filename)))

                    result.setUsedModules(used_modules)

                    # Not used anymore
                    source_code = None
                else:
                    result = CompiledPythonModule(
                        module_name=module_name,
                        is_top=is_top,
                        mode=mode,
                        future_spec=None,
                        source_ref=source_ref,
                    )

    elif Importing.isPackageDir(filename):
        if is_top:
            module_name = splitPath(filename)[-1]
        else:
            module_name = os.path.basename(filename)

        module_name = ModuleName.makeModuleNameInPackage(module_name, package)

        source_filename = os.path.join(filename, "__init__.py")

        if not os.path.isfile(source_filename):
            source_ref, result = createNamespacePackage(
                module_name=module_name,
                is_top=is_top,
                module_relpath=filename)
            source_filename = None
            source_code = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename=os.path.abspath(source_filename))

            result = CompiledPythonPackage(
                module_name=module_name,
                is_top=is_top,
                mode=decideCompilationMode(is_top, module_name, source_ref),
                future_spec=None,
                source_ref=source_ref,
            )

            source_code = readSourceCodeFromFilename(
                module_name=module_name, source_filename=source_filename)
    else:
        sys.stderr.write("%s: can't open file '%s'.\n" %
                         (os.path.basename(sys.argv[0]), filename))
        sys.exit(2)

    return result, source_ref, source_code
Beispiel #17
0
def _checkPluginPath(plugin_filename, module_package):
    # Many branches, for the decision is very complex

    debug(
        "Checking detail plug-in path '%s' '%s':",
        plugin_filename,
        module_package
    )

    plugin_info = considerFilename(
        module_filename = plugin_filename
    )

    if plugin_info is not None:
        module, is_added = recurseTo(
            module_filename = plugin_info[0],
            module_relpath  = plugin_info[1],
            module_package  = module_package,
            module_kind     = "py",
            reason          = "Lives in plug-in directory."
        )

        if module:
            if not is_added:
                warning(
                    "Recursed to %s '%s' at '%s' twice.",
                    "package" if module.isCompiledPythonPackage() else "module",
                    module.getName(),
                    plugin_info[0]
                )

                if not isSameModulePath(module.getFilename(), plugin_info[0]):
                    warning(
                        "Duplicate ignored '%s'.",
                        plugin_info[1]
                    )

                    return

            debug(
                "Recursed to %s %s %s",
                module.getName(),
                module.getPackage(),
                module
            )

            ImportCache.addImportedModule(module)

            if module.isCompiledPythonPackage():
                package_filename = module.getFilename()

                if Utils.isDir(package_filename):
                    # Must be a namespace package.
                    assert python_version >= 330

                    package_dir = package_filename

                    # Only include it, if it contains actual modules, which will
                    # recurse to this one and find it again.
                else:
                    package_dir = Utils.dirname(package_filename)

                    # Real packages will always be included.
                    ModuleRegistry.addRootModule(module)

                debug(
                    "Package directory %s",
                    package_dir
                )

                for sub_path, sub_filename in Utils.listDir(package_dir):
                    if sub_filename in ("__init__.py", "__pycache__"):
                        continue

                    assert sub_path != plugin_filename

                    if Importing.isPackageDir(sub_path) or \
                       sub_path.endswith(".py"):
                        _checkPluginPath(sub_path, module.getFullName())

            elif module.isCompiledPythonModule():
                ModuleRegistry.addRootModule(module)

        else:
            warning("Failed to include module from '%s'.", plugin_info[0])
Beispiel #18
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases, pylint: disable=R0912

    assert package is None or type(package) is str
    assert filename is not None

    if is_main and Utils.isDir(filename):
        source_filename = Utils.joinpath(filename, "__main__.py")

        if not Utils.isFile(source_filename):
            sys.stderr.write(
                "%s: can't find '__main__' module in '%s'\n" % (
                    Utils.basename(sys.argv[0]),
                    filename
                )
            )
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if Utils.isFile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(
            filename = filename,
        )

        if is_main:
            module_name = "__main__"
        else:
            module_name = Utils.basename(filename)

            if module_name.endswith(".py"):
                module_name = module_name[:-3]

            if is_shlib:
                module_name = module_name.split('.')[0]

            if '.' in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" % (
                        module_name
                    )
                )

                sys.exit(2)

        if is_shlib:
            result = PythonShlibModule(
                name         = module_name,
                package_name = package,
                source_ref   = source_ref
            )
        elif is_main:
            result = PythonMainModule(
                main_added = main_added,
                source_ref = source_ref
            )
        else:
            result = PythonModule(
                name         = module_name,
                package_name = package,
                source_ref   = source_ref
            )
    elif Importing.isPackageDir(filename):
        if is_top:
            package_name = Utils.splitpath(filename)[-1]
        else:
            package_name = Utils.basename(filename)

        source_filename = Utils.joinpath(filename, "__init__.py")

        if not Utils.isFile(source_filename):
            source_ref, result = createNamespacePackage(
                package_name   = package_name,
                module_relpath = filename
            )
            source_filename = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename = Utils.abspath(source_filename),
            )

            result = PythonPackage(
                name         = package_name,
                package_name = package,
                source_ref   = source_ref
            )
    else:
        sys.stderr.write(
            "%s: can't open file '%s'.\n" % (
                Utils.basename(sys.argv[0]),
                filename
            )
        )
        sys.exit(2)

    if not Options.shallHaveStatementLines():
        source_ref = source_ref.atInternal()

    return result, source_ref, source_filename
Beispiel #19
0
def buildModule(
    module_name,
    module_filename,
    source_code,
    is_top,
    is_main,
    is_extension,
    is_fake,
    hide_syntax_error,
):
    # Many details to deal with, pylint: disable=too-many-branches,too-many-locals
    (
        main_added,
        is_package,
        is_namespace,
        source_ref,
        source_filename,
    ) = Importing.decideModuleSourceRef(
        filename=module_filename,
        module_name=module_name,
        is_main=is_main,
        is_fake=is_fake,
        logger=general,
    )

    if Options.hasPythonFlagPackageMode():
        if is_top and Options.shallMakeModule():
            optimization_logger.warning(
                "Python flag -m (package_mode) has no effect in module mode, it's only for executables."
            )
        elif is_main and not main_added:
            optimization_logger.warning(
                "Python flag -m (package_mode) only works on packages with '__main__.py'."
            )

    # Read source code if necessary. Might give a SyntaxError due to not being proper
    # encoded source.
    if source_filename is not None and not is_namespace and not is_extension:
        try:
            # For fake modules, source is provided directly.
            if source_code is None:
                source_code = readSourceCodeFromFilename(
                    module_name=module_name, source_filename=source_filename)
        except SyntaxError as e:
            # Avoid hiding our own syntax errors.
            if not hasattr(e, "generated_by_nuitka"):
                raise

            # Do not hide SyntaxError in main module.
            if not hide_syntax_error:
                raise

            module = _makeModuleBodyFromSyntaxError(
                exc=e,
                module_name=module_name,
                module_filename=module_filename)
            return module, True

        try:
            ast_tree = parseSourceCodeToAst(
                source_code=source_code,
                module_name=module_name,
                filename=source_filename,
                line_offset=0,
            )
        except (SyntaxError, IndentationError) as e:
            # Do not hide SyntaxError if asked not to.
            if not hide_syntax_error:
                raise

            module = _makeModuleBodyFromSyntaxError(
                exc=e,
                module_name=module_name,
                module_filename=module_filename)
            return module, True
        except CodeTooComplexCode:
            # Do not hide CodeTooComplexCode in main module.
            if is_main:
                raise

            module = _makeModuleBodyTooComplex(
                module_name=module_name,
                module_filename=module_filename,
                source_code=source_code,
                is_package=is_package,
            )
            return module, False
    else:
        ast_tree = None
        source_code = None

    module = _createModule(
        module_name=module_name,
        source_code=source_code,
        source_ref=source_ref,
        is_top=is_top,
        is_main=is_main,
        is_extension=is_extension,
        is_namespace=is_namespace,
        is_package=is_package,
        main_added=main_added,
    )

    if is_top:
        ModuleRegistry.addRootModule(module)

        OutputDirectories.setMainModule(module)

    if module.isCompiledPythonModule() and source_code is not None:
        createModuleTree(
            module=module,
            source_ref=source_ref,
            ast_tree=ast_tree,
            is_main=is_main,
        )

    return module, True
Beispiel #20
0
def main():
    """Main program flow of Nuitka

    At this point, options will be parsed already, Nuitka will be executing
    in the desired version of Python with desired flags, and we just get
    to execute the task assigned.

    We might be asked to only re-compile generated C, dump only an XML
    representation of the internal node tree after optimization, etc.
    """

    # Main has to fulfill many options, leading to many branches and statements
    # to deal with them.  pylint: disable=too-many-branches,too-many-statements
    if not Options.shallDumpBuiltTreeXML():
        general.info("Starting Python compilation.")

    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir=os.path.dirname(os.path.abspath(filename))
    )

    # Turn that source code into a node tree structure.
    try:
        main_module = _createNodeTree(filename=filename)
    except (SyntaxError, IndentationError) as e:
        handleSyntaxError(e)

    if Options.shallDumpBuiltTreeXML():
        # XML output only.
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    else:
        # Make the actual compilation.
        result, options = compileTree()

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCCompilerCall():
            if Options.isShowMemory():
                MemoryUsage.showMemoryTrace()

            sys.exit(0)

        executePostProcessing()

        if Options.shallMakeModule() and Options.shallCreatePyiFile():
            pyi_filename = OutputDirectories.getResultBasepath() + ".pyi"

            putTextFileContents(
                filename=pyi_filename,
                contents="""\
# This file was generated by Nuitka and describes the types of the
# created shared library.

# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.

# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.

%(imports)s

# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...

"""
                % {
                    "imports": "\n".join(
                        "import %s" % module_name for module_name in getImportedNames()
                    )
                },
            )

        if Options.isStandaloneMode():
            binary_filename = options["result_exe"]

            setMainEntryPoint(binary_filename)

            dist_dir = OutputDirectories.getStandaloneDirectoryPath()

            for module in ModuleRegistry.getDoneModules():
                addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))

            copyUsedDLLs(
                source_dir=OutputDirectories.getSourceDirectoryPath(),
                dist_dir=dist_dir,
                standalone_entry_points=getStandaloneEntryPoints(),
            )

            copyDataFiles(dist_dir=dist_dir)

            Plugins.onStandaloneDistributionFinished(dist_dir)

            if Options.isOnefileMode():
                packDistFolderToOnefile(dist_dir, binary_filename)

                if Options.isRemoveBuildDir():
                    general.info("Removing dist folder %r." % dist_dir)

                    removeDirectory(path=dist_dir, ignore_errors=False)
                else:
                    general.info(
                        "Keeping dist folder %r for inspection, no need to use it."
                        % dist_dir
                    )

        # Remove the source directory (now build directory too) if asked to.
        source_dir = OutputDirectories.getSourceDirectoryPath()

        if Options.isRemoveBuildDir():
            general.info("Removing build directory %r." % source_dir)

            removeDirectory(path=source_dir, ignore_errors=False)
            assert not os.path.exists(source_dir)
        else:
            general.info("Keeping build directory %r." % source_dir)

        final_filename = OutputDirectories.getResultFullpath(
            onefile=Options.isOnefileMode()
        )

        Plugins.onFinalResult(final_filename)

        general.info("Successfully created %r." % final_filename)

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            general.info("Launching %r." % final_filename)

            if Options.shallMakeModule():
                executeModule(
                    tree=main_module,
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
            else:
                executeMain(
                    binary_filename=final_filename,
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
Beispiel #21
0
def main():
    """ Main program flow of Nuitka

        At this point, options will be parsed already, Nuitka will be executing
        in the desired version of Python with desired flags, and we just get
        to execute the task assigned.

        We might be asked to only re-compile generated C++, dump only an XML
        representation of the internal node tree after optimization, etc.
    """

    # Main has to fulfill many options, leading to many branches and statements
    # to deal with them.  pylint: disable=too-many-branches
    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir=os.path.dirname(os.path.abspath(filename)))

    # Detect to be frozen modules if any, so we can consider to not recurse
    # to them.
    if Options.isStandaloneMode():
        for module in detectEarlyImports():
            ModuleRegistry.addUncompiledModule(module)

            if module.getName() == "site":
                origin_prefix_filename = os.path.join(
                    os.path.dirname(module.getCompileTimeFilename()),
                    "orig-prefix.txt")

                if os.path.isfile(origin_prefix_filename):
                    data_files.append((filename, "orig-prefix.txt"))

    # Turn that source code into a node tree structure.
    try:
        main_module = createNodeTree(filename=filename)
    except (SyntaxError, IndentationError) as e:
        # Syntax or indentation errors, output them to the user and abort. If
        # we are not in full compat, and user has not specified the Python
        # versions he wants, tell him about the potential version problem.
        error_message = SyntaxErrors.formatOutput(e)

        if not Options.isFullCompat() and \
           Options.getIntendedPythonVersion() is None:
            if python_version < 300:
                suggested_python_version_str = getSupportedPythonVersions()[-1]
            else:
                suggested_python_version_str = "2.7"

            error_message += """

Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of e.g. '--python-version=%s' option, if that's not the one expected.
""" % (python_version_str, suggested_python_version_str)

        sys.exit(error_message)

    if Options.shallDumpBuiltTreeXML():
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    elif Options.shallDisplayBuiltTree():
        displayTree(main_module)
    else:
        result, options = compileTree(main_module=main_module)

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCppCall():
            sys.exit(0)

        # Remove the source directory (now build directory too) if asked to.
        if Options.isRemoveBuildDir():
            removeDirectory(path=getSourceDirectoryPath(main_module),
                            ignore_errors=False)

        if Options.isStandaloneMode():
            binary_filename = options["result_name"] + ".exe"

            standalone_entry_points.insert(0, (None, binary_filename, None))

            dist_dir = getStandaloneDirectoryPath(main_module)

            for module in ModuleRegistry.getDoneUserModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module))

            for module in ModuleRegistry.getUncompiledModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module))

            copyUsedDLLs(dist_dir=dist_dir,
                         standalone_entry_points=standalone_entry_points)

            for module in ModuleRegistry.getDoneModules():
                data_files.extend(Plugins.considerDataFiles(module))

            for source_filename, target_filename in data_files:
                target_filename = os.path.join(
                    getStandaloneDirectoryPath(main_module), target_filename)

                makePath(os.path.dirname(target_filename))

                shutil.copy2(source_filename, target_filename)

        # Modules should not be executable, but Scons creates them like it, fix
        # it up here.
        if Utils.getOS() != "Windows" and Options.shallMakeModule():
            subprocess.call(("chmod", "-x", getResultFullpath(main_module)))

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            if Options.shallMakeModule():
                executeModule(
                    tree=main_module,
                    clean_path=Options.shallClearPythonPathEnvironment())
            else:
                executeMain(
                    binary_filename=getResultFullpath(main_module),
                    clean_path=Options.shallClearPythonPathEnvironment())
Beispiel #22
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915

    assert package is None or type(package) is str
    assert filename is not None

    if is_main and Utils.isDir(filename):
        source_filename = Utils.joinpath(filename, "__main__.py")

        if not Utils.isFile(source_filename):
            sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
                             (Utils.basename(sys.argv[0]), filename))
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if Utils.isFile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(filename=filename, )

        if is_main:
            module_name = "__main__"
        else:
            module_name = Utils.basename(filename)

            if module_name.endswith(".py"):
                module_name = module_name[:-3]

            if is_shlib:
                module_name = module_name.split('.')[0]

            if '.' in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" %
                    (module_name))

                sys.exit(2)

        if is_shlib:
            result = PythonShlibModule(name=module_name,
                                       package_name=package,
                                       source_ref=source_ref)
        elif is_main:
            result = PythonMainModule(main_added=main_added,
                                      source_ref=source_ref)
        else:
            if package is not None:
                full_name = package + "." + module_name
            else:
                full_name = module_name

            decision = Plugins.decideCompilation(full_name, source_ref)

            if decision == "compiled":
                result = CompiledPythonModule(name=module_name,
                                              package_name=package,
                                              source_ref=source_ref)
            else:
                source_code = readSourceCodeFromFilename(module_name, filename)

                source_code = Plugins.onFrozenModuleSourceCode(
                    module_name=full_name,
                    is_package=False,
                    source_code=source_code)

                bytecode = compile(source_code, filename, "exec")

                bytecode = Plugins.onFrozenModuleBytecode(
                    module_name=module_name,
                    is_package=False,
                    bytecode=bytecode)

                result = UncompiledPythonModule(name=module_name,
                                                package_name=package,
                                                bytecode=bytecode,
                                                filename=filename,
                                                user_provided=True,
                                                technical=False,
                                                source_ref=source_ref)

                # Don't read it anymore.
                source_filename = None

    elif Importing.isPackageDir(filename):
        if is_top:
            package_name = Utils.splitpath(filename)[-1]
        else:
            package_name = Utils.basename(filename)

        source_filename = Utils.joinpath(filename, "__init__.py")

        if not Utils.isFile(source_filename):
            source_ref, result = createNamespacePackage(
                package_name=package_name, module_relpath=filename)
            source_filename = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename=Utils.abspath(source_filename), )

            if package is not None:
                full_name = package + "." + package_name
            else:
                full_name = package_name

            decision = Plugins.decideCompilation(full_name, source_ref)

            if decision == "compiled":
                result = CompiledPythonPackage(name=package_name,
                                               package_name=package,
                                               source_ref=source_ref)
            else:
                bytecode = compile(source_code, filename, "exec")

                bytecode = Plugins.onFrozenModuleBytecode(
                    module_name=module_name,
                    is_package=False,
                    bytecode=bytecode)

                result = UncompiledPythonPackage(name=module_name,
                                                 package_name=package,
                                                 bytecode=bytecode,
                                                 filename=filename,
                                                 user_provided=True,
                                                 technical=False,
                                                 source_ref=source_ref)

                # Don't read it anymore.
                source_filename = None
    else:
        sys.stderr.write("%s: can't open file '%s'.\n" %
                         (Utils.basename(sys.argv[0]), filename))
        sys.exit(2)

    if not Options.shallHaveStatementLines():
        source_ref = source_ref.atInternal()

    return result, source_ref, source_filename
Beispiel #23
0
def checkPluginSinglePath(plugin_filename, module_package):
    # Many branches, for the decision is very complex, pylint: disable=too-many-branches

    # The importing wants these to be unique.
    plugin_filename = os.path.abspath(plugin_filename)

    if Options.isShowInclusion():
        recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
                              (plugin_filename, module_package))

    module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
        plugin_filename)

    module_name = ModuleName.makeModuleNameInPackage(module_name,
                                                     module_package)

    if module_kind == "extension" and not Options.isStandaloneMode():
        recursion_logger.warning(
            "Cannot include '%s' unless using at least standalone mode." %
            module_name.asString())

    if module_kind is not None:
        decision, reason = decideRecursion(
            module_filename=plugin_filename,
            module_name=module_name,
            module_kind=module_kind,
            extra_recursion=True,
        )

        if decision:
            module = recurseTo(
                signal_change=None,
                module_filename=plugin_filename,
                module_name=module_name,
                module_kind=module_kind,
                reason=reason,
            )

            if module:
                if Options.isShowInclusion():
                    recursion_logger.info("Included '%s' as '%s'." % (
                        module.getFullName(),
                        module,
                    ))

                ImportCache.addImportedModule(module)

                if module.isCompiledPythonPackage():
                    package_filename = module.getFilename()

                    if os.path.isdir(package_filename):
                        # Must be a namespace package.
                        assert python_version >= 0x300

                        package_dir = package_filename

                        # Only include it, if it contains actual modules, which will
                        # recurse to this one and find it again.
                    else:
                        package_dir = os.path.dirname(package_filename)

                        # Real packages will always be included.
                        ModuleRegistry.addRootModule(module)

                    if Options.isShowInclusion():
                        recursion_logger.info("Package directory '%s'." %
                                              package_dir)

                    for sub_path, sub_filename in listDir(package_dir):
                        if sub_filename in ("__init__.py", "__pycache__"):
                            continue

                        assert sub_path != plugin_filename

                        if Importing.isPackageDir(
                                sub_path) and not os.path.exists(sub_path +
                                                                 ".py"):
                            checkPluginSinglePath(
                                sub_path, module_package=module.getFullName())
                        elif sub_path.endswith(".py"):
                            checkPluginSinglePath(
                                sub_path, module_package=module.getFullName())

                elif module.isCompiledPythonModule():
                    ModuleRegistry.addRootModule(module)
                elif module.isPythonExtensionModule():
                    if Options.isStandaloneMode():
                        ModuleRegistry.addRootModule(module)

            else:
                recursion_logger.warning(
                    "Failed to include module from '%s'." % plugin_filename)
Beispiel #24
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements

    assert package is None or type(package) is str
    assert filename is not None

    if is_main and os.path.isdir(filename):
        source_filename = os.path.join(filename, "__main__.py")

        if not os.path.isfile(source_filename):
            sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
                             (os.path.basename(sys.argv[0]), filename))
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if os.path.isfile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(filename=filename, )

        if is_main:
            module_name = "__main__"
        else:
            module_name = os.path.basename(filename)

            if module_name.endswith(".py"):
                module_name = module_name[:-3]

            if is_shlib:
                module_name = module_name.split('.')[0]

            if '.' in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" %
                    (module_name))

                sys.exit(2)

        if is_shlib:
            result = PythonShlibModule(name=module_name,
                                       package_name=package,
                                       source_ref=source_ref)
        elif is_main:
            result = PythonMainModule(main_added=main_added,
                                      mode=Plugins.decideCompilation(
                                          "__main__", source_ref),
                                      future_spec=None,
                                      source_ref=source_ref)
        else:
            if package is not None:
                full_name = package + '.' + module_name
            else:
                full_name = module_name

            result = CompiledPythonModule(name=module_name,
                                          package_name=package,
                                          mode=Plugins.decideCompilation(
                                              full_name, source_ref),
                                          future_spec=None,
                                          source_ref=source_ref)

    elif Importing.isPackageDir(filename):
        if is_top:
            module_name = splitPath(filename)[-1]
        else:
            module_name = os.path.basename(filename)

        source_filename = os.path.join(filename, "__init__.py")

        if not os.path.isfile(source_filename):
            source_ref, result = createNamespacePackage(
                module_name=module_name,
                package_name=package,
                module_relpath=filename)
            source_filename = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename=os.path.abspath(source_filename), )

            if package is not None:
                full_name = package + '.' + module_name
            else:
                full_name = module_name

            result = CompiledPythonPackage(name=module_name,
                                           package_name=package,
                                           mode=Plugins.decideCompilation(
                                               full_name, source_ref),
                                           future_spec=None,
                                           source_ref=source_ref)

            assert result.getFullName() == full_name, result
    else:
        sys.stderr.write("%s: can't open file '%s'.\n" %
                         (os.path.basename(sys.argv[0]), filename))
        sys.exit(2)

    if not Options.shallHaveStatementLines():
        source_ref = source_ref.atInternal()

    return result, source_ref, source_filename
Beispiel #25
0
def main():
    """ Main program flow of Nuitka

        At this point, options will be parsed already, Nuitka will be executing
        in the desired version of Python with desired flags, and we just get
        to execute the task assigned.

        We might be asked to only re-compile generated C, dump only an XML
        representation of the internal node tree after optimization, etc.
    """

    # Main has to fulfill many options, leading to many branches and statements
    # to deal with them.  pylint: disable=too-many-branches
    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir = os.path.dirname(os.path.abspath(filename))
    )

    # Detect to be frozen modules if any, so we can consider to not recurse
    # to them.
    if Options.isStandaloneMode():
        for module in detectEarlyImports():
            ModuleRegistry.addUncompiledModule(module)

            if module.getName() == "site":
                origin_prefix_filename = os.path.join(
                    os.path.dirname(module.getCompileTimeFilename()),
                    "orig-prefix.txt"
                )

                if os.path.isfile(origin_prefix_filename):
                    data_files.append(
                        (filename, "orig-prefix.txt")
                    )

    # Turn that source code into a node tree structure.
    try:
        main_module = createNodeTree(
            filename = filename
        )
    except (SyntaxError, IndentationError) as e:
        handleSyntaxError(e)

    if Options.shallDumpBuiltTreeXML():
        # XML output only.
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    else:
        # Make the actual compilation.
        result, options = compileTree(
            main_module = main_module
        )

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCCompilerCall():
            if Options.isShowMemory():
                MemoryUsage.showMemoryTrace()

            sys.exit(0)

        if Options.isStandaloneMode():
            binary_filename = options["result_exe"]

            standalone_entry_points.insert(
                0,
                (binary_filename, binary_filename, None)
            )

            dist_dir = getStandaloneDirectoryPath(main_module)

            for module in ModuleRegistry.getDoneUserModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module)
                )

            for module in ModuleRegistry.getUncompiledModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module)
                )

            copyUsedDLLs(
                source_dir              = getSourceDirectoryPath(main_module),
                dist_dir                = dist_dir,
                standalone_entry_points = standalone_entry_points
            )

            for module in ModuleRegistry.getDoneModules():
                data_files.extend(
                    Plugins.considerDataFiles(module)
                )

            for source_filename, target_filename in data_files:
                target_filename = os.path.join(
                    getStandaloneDirectoryPath(main_module),
                    target_filename
                )

                makePath(os.path.dirname(target_filename))

                shutil.copy2(
                    source_filename,
                    target_filename
                )

        # Remove the source directory (now build directory too) if asked to.
        if Options.isRemoveBuildDir():
            removeDirectory(
                path          = getSourceDirectoryPath(main_module),
                ignore_errors = False
            )

        # Modules should not be executable, but Scons creates them like it, fix
        # it up here. TODO: Move inside scons file and avoid subprocess call.
        if Utils.getOS() != "Windows" and Options.shallMakeModule():
            subprocess.call(
                (
                    "chmod",
                    "-x",
                    getResultFullpath(main_module)
                )
            )

        if Options.shallMakeModule() and Options.shallCreatePyiFile():
            pyi_filename = getResultBasepath(main_module) + ".pyi"

            with open(pyi_filename, 'w') as pyi_file:
                pyi_file.write(
                    """\
# This file was generated by Nuitka and describes the types of the
# created shared library.

# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.

# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.

%(imports)s

# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...

""" % {
                        "imports" : '\n'.join(
                            "import %s" % module_name
                            for module_name in
                            getImportedNames()
                        )

                    }
                )


        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            if Options.shallMakeModule():
                executeModule(
                    tree       = main_module,
                    clean_path = Options.shallClearPythonPathEnvironment()
                )
            else:
                executeMain(
                    binary_filename = getResultFullpath(main_module),
                    clean_path      = Options.shallClearPythonPathEnvironment()
                )
Beispiel #26
0
def main():
    """ Main program flow of Nuitka

        At this point, options will be parsed already, Nuitka will be executing
        in the desired version of Python with desired flags, and we just get
        to execute the task assigned.

        We might be asked to only re-compile generated C++, dump only an XML
        representation of the internal node tree after optimization, etc.
    """

    # Main has to fullfil many options, leading to many branches and statements
    # to deal with them.  pylint: disable=R0912

    positional_args = Options.getPositionalArgs()
    assert len(positional_args) > 0

    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir = Utils.dirname(Utils.abspath(filename))
    )

    # Detect to be frozen modules if any, so we can consider to not recurse
    # to them.
    if Options.isStandaloneMode():
        for module in detectEarlyImports():
            ModuleRegistry.addUncompiledModule(module)

            if module.getName() == "site":
                origin_prefix_filename = Utils.joinpath(
                    Utils.dirname(module.getCompileTimeFilename()),
                    "orig-prefix.txt"
                )

                if Utils.isFile(origin_prefix_filename):
                    data_files.append(
                        (filename, "orig-prefix.txt")
                    )

    # Turn that source code into a node tree structure.
    try:
        main_module = createNodeTree(
            filename = filename
        )
    except (SyntaxError, IndentationError) as e:
        # Syntax or indentation errors, output them to the user and abort.
        sys.exit(
            SyntaxErrors.formatOutput(e)
        )

    if Options.shallDumpBuiltTreeXML():
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    elif Options.shallDisplayBuiltTree():
        displayTree(main_module)
    else:
        result, options = compileTree(
            main_module = main_module
        )

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCppCall():
            sys.exit(0)

        # Remove the source directory (now build directory too) if asked to.
        if Options.isRemoveBuildDir():
            shutil.rmtree(
                getSourceDirectoryPath(main_module)
            )

        if Options.isStandaloneMode():
            binary_filename = options["result_name"] + ".exe"

            standalone_entry_points.insert(
                0,
                (binary_filename, None)
            )

            dist_dir = getStandaloneDirectoryPath(main_module)

            for module in ModuleRegistry.getDoneUserModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module)
                )

            if Utils.getOS() == "NetBSD":
                warning("Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported.")

            copyUsedDLLs(
                dist_dir                = dist_dir,
                standalone_entry_points = standalone_entry_points
            )

            for source_filename, target_filename in data_files:
                shutil.copy2(
                    source_filename,
                    Utils.joinpath(
                        getStandaloneDirectoryPath(main_module),
                        target_filename
                    )
                )

        # Modules should not be executable, but Scons creates them like it, fix
        # it up here.
        if Utils.getOS() != "Windows" and Options.shallMakeModule():
            subprocess.call(
                (
                    "chmod",
                    "-x",
                    getResultFullpath(main_module)
                )
            )

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            if Options.shallMakeModule():
                executeModule(
                    tree       = main_module,
                    clean_path = Options.shallClearPythonPathEnvironment()
                )
            else:
                executeMain(
                    binary_filename = getResultFullpath(main_module),
                    clean_path      = Options.shallClearPythonPathEnvironment()
                )
Beispiel #27
0
def _createNodeTree(filename):
    """Create a node tree.

    Turn that source code into a node tree structure. If recursion into
    imported modules is available, more trees will be available during
    optimization, or immediately through recursed directory paths.

    """

    # Many cases to deal with, pylint: disable=too-many-branches

    # First, build the raw node tree from the source code.
    main_module = Building.buildMainModuleTree(
        filename=filename,
        is_main=not Options.shallMakeModule(),
    )

    # First remove old object files and old generated files, old binary or
    # module, and standalone mode program directory if any, they can only do
    # harm.
    source_dir = OutputDirectories.getSourceDirectoryPath()

    if not Options.shallOnlyExecCCompilerCall():
        SconsInterface.cleanSconsDirectory(source_dir)

    # Prepare the ".dist" directory, throwing away what was there before.
    if Options.isStandaloneMode():
        standalone_dir = OutputDirectories.getStandaloneDirectoryPath(bundle=False)
        removeDirectory(path=standalone_dir, ignore_errors=True)

        if Options.shallCreateAppBundle():
            removeDirectory(
                path=changeFilenameExtension(standalone_dir, ".app"), ignore_errors=True
            )

    # Delete result file, to avoid confusion with previous build and to
    # avoid locking issues after the build.
    deleteFile(
        path=OutputDirectories.getResultFullpath(onefile=False), must_exist=False
    )
    if Options.isOnefileMode():
        deleteFile(
            path=OutputDirectories.getResultFullpath(onefile=True), must_exist=False
        )

    # Second, do it for the directories given.
    for plugin_filename in Options.getShallFollowExtra():
        Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)

    for pattern in Options.getShallFollowExtraFilePatterns():
        Recursion.checkPluginFilenamePattern(pattern=pattern)

    for package_name in Options.getMustIncludePackages():
        package_name, package_directory, kind = Importing.locateModule(
            module_name=ModuleName(package_name),
            parent_package=None,
            level=0,
        )

        if kind != "absolute":
            inclusion_logger.sysexit(
                "Error, failed to locate package %r you asked to include."
                % package_name.asString()
            )

        Recursion.checkPluginPath(
            plugin_filename=package_directory,
            module_package=package_name.getPackageName(),
        )

    for module_name in Options.getMustIncludeModules():
        module_name, module_filename, kind = Importing.locateModule(
            module_name=ModuleName(module_name),
            parent_package=None,
            level=0,
        )

        if kind != "absolute":
            inclusion_logger.sysexit(
                "Error, failed to locate module '%s' you asked to include."
                % module_name.asString()
            )

        Recursion.checkPluginSinglePath(
            plugin_filename=module_filename, module_package=module_name.getPackageName()
        )

    # Allow plugins to add more modules based on the initial set being complete.
    Plugins.onModuleInitialSet()

    # Then optimize the tree and potentially recursed modules.
    optimizeModules(main_module.getOutputFilename())

    # Allow plugins to comment on final module set.
    Plugins.onModuleCompleteSet()

    if Options.isExperimental("check_xml_persistence"):
        for module in ModuleRegistry.getRootModules():
            if module.isMainModule():
                return module

        assert False
    else:
        # Main module might change behind our back, look it up again.
        return main_module
Beispiel #28
0
def main():
    """Main program flow of Nuitka

    At this point, options will be parsed already, Nuitka will be executing
    in the desired version of Python with desired flags, and we just get
    to execute the task assigned.

    We might be asked to only re-compile generated C, dump only an XML
    representation of the internal node tree after optimization, etc.
    """

    # Main has to fulfill many options, leading to many branches and statements
    # to deal with them.  pylint: disable=too-many-branches,too-many-statements

    # In case we are in a PGO run, we read its information first, so it becomes
    # available for later parts.
    pgo_filename = getPythonPgoInput()
    if pgo_filename is not None:
        readPGOInputFile(pgo_filename)

    if not Options.shallDumpBuiltTreeXML():
        general.info(
            "Starting Python compilation with Nuitka %r on Python %r commercial %r."
            % (getNuitkaVersion(), python_version_str, getCommercialVersion())
        )

    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir=os.path.dirname(os.path.abspath(filename))
    )

    addIncludedDataFilesFromFileOptions()

    # Turn that source code into a node tree structure.
    try:
        main_module = _createNodeTree(filename=filename)
    except (SyntaxError, IndentationError) as e:
        handleSyntaxError(e)

    addIncludedDataFilesFromPackageOptions()

    if Options.shallDumpBuiltTreeXML():
        # XML output only.
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    else:
        # Make the actual compilation.
        result, options = compileTree()

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        # Relaunch in case of Python PGO input to be produced.
        if Options.shallCreatePgoInput():
            # Will not return.
            pgo_filename = OutputDirectories.getPgoRunInputFilename()
            general.info(
                "Restarting compilation using collected information from '%s'."
                % pgo_filename
            )
            reExecuteNuitka(pgo_filename=pgo_filename)

        if Options.shallNotDoExecCCompilerCall():
            if Options.isShowMemory():
                MemoryUsage.showMemoryTrace()

            sys.exit(0)

        executePostProcessing()

        copyDataFiles()

        if Options.isStandaloneMode():
            binary_filename = options["result_exe"]

            setMainEntryPoint(binary_filename)

            dist_dir = OutputDirectories.getStandaloneDirectoryPath()

            for module in ModuleRegistry.getDoneModules():
                addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))

            copyDllsUsed(
                source_dir=OutputDirectories.getSourceDirectoryPath(),
                dist_dir=dist_dir,
                standalone_entry_points=getStandaloneEntryPoints(),
            )

            Plugins.onStandaloneDistributionFinished(dist_dir)

            if Options.isOnefileMode():
                packDistFolderToOnefile(dist_dir, binary_filename)

                if Options.isRemoveBuildDir():
                    general.info("Removing dist folder %r." % dist_dir)

                    removeDirectory(path=dist_dir, ignore_errors=False)
                else:
                    general.info(
                        "Keeping dist folder %r for inspection, no need to use it."
                        % dist_dir
                    )

        # Remove the source directory (now build directory too) if asked to.
        source_dir = OutputDirectories.getSourceDirectoryPath()

        if Options.isRemoveBuildDir():
            general.info("Removing build directory %r." % source_dir)

            removeDirectory(path=source_dir, ignore_errors=False)
            assert not os.path.exists(source_dir)
        else:
            general.info("Keeping build directory %r." % source_dir)

        final_filename = OutputDirectories.getResultFullpath(
            onefile=Options.isOnefileMode()
        )

        if Options.isStandaloneMode() and isMacOS():
            general.info(
                "Created binary that runs on macOS %s (%s) or higher."
                % (options["macos_min_version"], options["macos_target_arch"])
            )

        Plugins.onFinalResult(final_filename)

        general.info("Successfully created %r." % final_filename)

        report_filename = Options.getCompilationReportFilename()

        if report_filename:
            writeCompilationReport(report_filename)

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            run_filename = OutputDirectories.getResultRunFilename(
                onefile=Options.isOnefileMode()
            )

            general.info("Launching %r." % run_filename)

            if Options.shallMakeModule():
                executeModule(
                    tree=main_module,
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
            else:
                executeMain(
                    binary_filename=run_filename,
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
Beispiel #29
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915

    assert package is None or type(package) is str
    assert filename is not None

    if is_main and Utils.isDir(filename):
        source_filename = Utils.joinpath(filename, "__main__.py")

        if not Utils.isFile(source_filename):
            sys.stderr.write(
                "%s: can't find '__main__' module in '%s'\n" % (
                    Utils.basename(sys.argv[0]),
                    filename
                )
            )
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if Utils.isFile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(
            filename = filename,
        )

        if is_main:
            module_name = "__main__"
        else:
            module_name = Utils.basename(filename)

            if module_name.endswith(".py"):
                module_name = module_name[:-3]

            if is_shlib:
                module_name = module_name.split('.')[0]

            if '.' in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" % (
                        module_name
                    )
                )

                sys.exit(2)

        if is_shlib:
            result = PythonShlibModule(
                name         = module_name,
                package_name = package,
                source_ref   = source_ref
            )
        elif is_main:
            result = PythonMainModule(
                main_added = main_added,
                source_ref = source_ref
            )
        else:
            if package is not None:
                full_name = package + "." + module_name
            else:
                full_name = module_name

            decision = Plugins.decideCompilation(full_name, source_ref)

            if decision == "compiled":
                result = CompiledPythonModule(
                    name         = module_name,
                    package_name = package,
                    source_ref   = source_ref
                )
            else:
                source_code = readSourceCodeFromFilename(module_name, filename)

                source_code = Plugins.onFrozenModuleSourceCode(
                    module_name = full_name,
                    is_package  = False,
                    source_code = source_code
                )

                bytecode = compile(source_code, filename, "exec")

                bytecode = Plugins.onFrozenModuleBytecode(
                    module_name = module_name,
                    is_package  = False,
                    bytecode    = bytecode
                )

                result = UncompiledPythonModule(
                    name         = module_name,
                    package_name = package,
                    bytecode     = bytecode,
                    filename     = filename,
                    user_provided = True,
                    technical    = False,
                    source_ref   = source_ref
                )

                # Don't read it anymore.
                source_filename = None

    elif Importing.isPackageDir(filename):
        if is_top:
            package_name = Utils.splitpath(filename)[-1]
        else:
            package_name = Utils.basename(filename)

        source_filename = Utils.joinpath(filename, "__init__.py")

        if not Utils.isFile(source_filename):
            source_ref, result = createNamespacePackage(
                package_name   = package_name,
                module_relpath = filename
            )
            source_filename = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename = Utils.abspath(source_filename),
            )

            if package is not None:
                full_name = package + "." + package_name
            else:
                full_name = package_name

            decision = Plugins.decideCompilation(full_name, source_ref)

            if decision == "compiled":
                result = CompiledPythonPackage(
                    name         = package_name,
                    package_name = package,
                    source_ref   = source_ref
                )
            else:
                bytecode = compile(source_code, filename, "exec")

                bytecode = Plugins.onFrozenModuleBytecode(
                    module_name = module_name,
                    is_package  = False,
                    bytecode    = bytecode
                )

                result = UncompiledPythonPackage(
                    name         = module_name,
                    package_name = package,
                    bytecode     = bytecode,
                    filename     = filename,
                    user_provided = True,
                    technical    = False,
                    source_ref   = source_ref
                )

                # Don't read it anymore.
                source_filename = None
    else:
        sys.stderr.write(
            "%s: can't open file '%s'.\n" % (
                Utils.basename(sys.argv[0]),
                filename
            )
        )
        sys.exit(2)

    if not Options.shallHaveStatementLines():
        source_ref = source_ref.atInternal()

    return result, source_ref, source_filename
Beispiel #30
0
def createNodeTree(filename):
    """ Create a node tree.

    Turn that source code into a node tree structure. If recursion into
    imported modules is available, more trees will be available during
    optimization, or immediately through recursed directory paths.

    """

    # First, build the raw node tree from the source code.
    main_module = Building.buildModuleTree(
        filename = filename,
        package  = None,
        is_top   = True,
        is_main  = not Options.shallMakeModule()
    )
    ModuleRegistry.addRootModule(main_module)

    # First remove old object files and old generated files, old binary or
    # module, and standalone mode program directory if any, they can only do
    # harm.
    source_dir = getSourceDirectoryPath(main_module)

    if not Options.shallOnlyExecCCompilerCall():
        cleanSourceDirectory(source_dir)

    # Prepare the ".dist" directory, throwing away what was there before.
    if Options.isStandaloneMode():
        standalone_dir = getStandaloneDirectoryPath(main_module)
        removeDirectory(
            path          = standalone_dir,
            ignore_errors = True
        )
        makePath(standalone_dir)

    deleteFile(
        path       = getResultFullpath(main_module),
        must_exist = False
    )

    # Second, do it for the directories given.
    for plugin_filename in Options.getShallFollowExtra():
        Recursion.checkPluginPath(
            plugin_filename = plugin_filename,
            module_package  = None
        )

    for pattern in Options.getShallFollowExtraFilePatterns():
        Recursion.checkPluginFilenamePattern(
            pattern = pattern
        )

    for package_name in Options.getMustIncludePackages():
        package_package, package_directory, kind = Importing.findModule(
            importing      = None,
            module_name    = package_name,
            parent_package = None,
            level          = 0,
            warn           = False
        )

        if kind != "absolute":
            sys.exit("Error, failed to locate package %r." % package_name)

        Recursion.checkPluginPath(
            plugin_filename = package_directory,
            module_package  = package_package
        )

    for module_name in Options.getMustIncludeModules():
        module_package, module_filename, kind = Importing.findModule(
            importing      = None,
            module_name    = module_name,
            parent_package = None,
            level          = 0,
            warn           = False
        )

        if kind != "absolute":
            sys.exit("Error, failed to locate module %r." % module_name)

        Recursion.checkPluginSinglePath(
            plugin_filename = module_filename,
            module_package  = module_package
        )

    # Then optimize the tree and potentially recursed modules.
    Optimization.optimize(main_module.getOutputFilename())

    if Options.isExperimental("check_xml_persistence"):
        for module in ModuleRegistry.getRootModules():
            if module.isMainModule():
                return module

        assert False
    else:
        # Main module might change behind our back, look it up again.
        return main_module
Beispiel #31
0
def createNodeTree(filename):
    """ Create a node tree.

    Turn that source code into a node tree structure. If recursion into
    imported modules is available, more trees will be available during
    optimization, or immediately through recursed directory paths.

    """

    # First, build the raw node tree from the source code.
    main_module = Building.buildModuleTree(
        filename=filename,
        package=None,
        is_top=True,
        is_main=not Options.shallMakeModule(),
    )
    ModuleRegistry.addRootModule(main_module)

    # First remove old object files and old generated files, old binary or
    # module, and standalone mode program directory if any, they can only do
    # harm.
    source_dir = getSourceDirectoryPath(main_module)

    if not Options.shallOnlyExecCCompilerCall():
        cleanSourceDirectory(source_dir)

    # Prepare the ".dist" directory, throwing away what was there before.
    if Options.isStandaloneMode():
        standalone_dir = getStandaloneDirectoryPath(main_module)
        removeDirectory(path=standalone_dir, ignore_errors=True)
        makePath(standalone_dir)

    deleteFile(path=getResultFullpath(main_module), must_exist=False)

    # Second, do it for the directories given.
    for plugin_filename in Options.getShallFollowExtra():
        Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)

    for pattern in Options.getShallFollowExtraFilePatterns():
        Recursion.checkPluginFilenamePattern(pattern=pattern)

    for package_name in Options.getMustIncludePackages():
        package_package, package_directory, kind = Importing.findModule(
            importing=None,
            module_name=package_name,
            parent_package=None,
            level=0,
            warn=False,
        )

        if kind != "absolute":
            sys.exit("Error, failed to locate package %r." % package_name)

        Recursion.checkPluginPath(
            plugin_filename=package_directory, module_package=package_package
        )

    for module_name in Options.getMustIncludeModules():
        module_package, module_filename, kind = Importing.findModule(
            importing=None,
            module_name=module_name,
            parent_package=None,
            level=0,
            warn=False,
        )

        if kind != "absolute":
            sys.exit("Error, failed to locate module %r." % module_name)

        Recursion.checkPluginSinglePath(
            plugin_filename=module_filename, module_package=module_package
        )

    # Then optimize the tree and potentially recursed modules.
    Optimization.optimize(main_module.getOutputFilename())

    if Options.isExperimental("check_xml_persistence"):
        for module in ModuleRegistry.getRootModules():
            if module.isMainModule():
                return module

        assert False
    else:
        # Main module might change behind our back, look it up again.
        return main_module
Beispiel #32
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases, pylint: disable=too-many-branches

    assert package is None or type(package) is ModuleName
    assert filename is not None

    if is_main and os.path.isdir(filename):
        source_filename = os.path.join(filename, "__main__.py")

        if not os.path.isfile(source_filename):
            sys.stderr.write(
                "%s: can't find '__main__' module in '%s'\n"
                % (os.path.basename(sys.argv[0]), filename)
            )
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if os.path.isfile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(filename=filename)

        if is_main:
            module_name = ModuleName("__main__")
        else:
            # Derive module name from filename.
            module_name = os.path.basename(filename)
            if is_shlib:
                module_name = module_name.split(".")[0]
            elif module_name.endswith(".py"):
                module_name = module_name[:-3]

            if "." in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" % (module_name)
                )

                sys.exit(2)

            module_name = ModuleName.makeModuleNameInPackage(module_name, package)

        if is_shlib:
            result = PythonShlibModule(module_name=module_name, source_ref=source_ref)
        elif is_main:
            result = PythonMainModule(
                main_added=main_added,
                mode=decideCompilationMode(False, module_name, source_ref),
                future_spec=None,
                source_ref=source_ref,
            )
        else:
            result = CompiledPythonModule(
                module_name=module_name,
                is_top=is_top,
                mode=decideCompilationMode(is_top, module_name, source_ref),
                future_spec=None,
                source_ref=source_ref,
            )

    elif Importing.isPackageDir(filename):
        if is_top:
            module_name = splitPath(filename)[-1]
        else:
            module_name = os.path.basename(filename)

        module_name = ModuleName.makeModuleNameInPackage(module_name, package)

        source_filename = os.path.join(filename, "__init__.py")

        if not os.path.isfile(source_filename):
            source_ref, result = createNamespacePackage(
                module_name=module_name, is_top=is_top, module_relpath=filename
            )
            source_filename = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename=os.path.abspath(source_filename)
            )

            result = CompiledPythonPackage(
                module_name=module_name,
                is_top=is_top,
                mode=decideCompilationMode(is_top, module_name, source_ref),
                future_spec=None,
                source_ref=source_ref,
            )
    else:
        sys.stderr.write(
            "%s: can't open file '%s'.\n" % (os.path.basename(sys.argv[0]), filename)
        )
        sys.exit(2)

    return result, source_ref, source_filename
Beispiel #33
0
def main():
    """ Main program flow of Nuitka

        At this point, options will be parsed already, Nuitka will be executing
        in the desired version of Python with desired flags, and we just get
        to execute the task assigned.

        We might be asked to only re-compile generated C, dump only an XML
        representation of the internal node tree after optimization, etc.
    """

    # Main has to fulfill many options, leading to many branches and statements
    # to deal with them.  pylint: disable=too-many-branches
    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir=os.path.dirname(os.path.abspath(filename))
    )

    # Detect to be frozen modules if any, so we can consider to not recurse
    # to them.
    if Options.isStandaloneMode():
        for module in detectEarlyImports():
            ModuleRegistry.addUncompiledModule(module)

    # Turn that source code into a node tree structure.
    try:
        main_module = createNodeTree(filename=filename)
    except (SyntaxError, IndentationError) as e:
        handleSyntaxError(e)

    if Options.shallDumpBuiltTreeXML():
        # XML output only.
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    else:
        # Make the actual compilation.
        result, options = compileTree(main_module=main_module)

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCCompilerCall():
            if Options.isShowMemory():
                MemoryUsage.showMemoryTrace()

            sys.exit(0)

        executePostProcessing(getResultFullpath(main_module))

        if Options.isStandaloneMode():
            binary_filename = options["result_exe"]

            standalone_entry_points.insert(0, (binary_filename, binary_filename, None))

            dist_dir = getStandaloneDirectoryPath(main_module)

            for module in ModuleRegistry.getDoneUserModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module)
                )

            copyUsedDLLs(
                source_dir=getSourceDirectoryPath(main_module),
                dist_dir=dist_dir,
                standalone_entry_points=standalone_entry_points,
            )

            data_files = []
            for module in ModuleRegistry.getDoneModules():
                data_files.extend(Plugins.considerDataFiles(module))

            copyDataFiles(dist_dir=dist_dir, data_files=data_files)

            Plugins.onStandaloneDistributionFinished(dist_dir)

        # Remove the source directory (now build directory too) if asked to.
        if Options.isRemoveBuildDir():
            removeDirectory(
                path=getSourceDirectoryPath(main_module), ignore_errors=False
            )

        if Options.shallMakeModule() and Options.shallCreatePyiFile():
            pyi_filename = getResultBasepath(main_module) + ".pyi"

            with open(pyi_filename, "w") as pyi_file:
                pyi_file.write(
                    """\
# This file was generated by Nuitka and describes the types of the
# created shared library.

# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.

# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.

%(imports)s

# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...

"""
                    % {
                        "imports": "\n".join(
                            "import %s" % module_name
                            for module_name in getImportedNames()
                        )
                    }
                )

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            if Options.shallMakeModule():
                executeModule(
                    tree=main_module,
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
            else:
                executeMain(
                    binary_filename=getResultFullpath(main_module),
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
Beispiel #34
0
def _checkPluginPath(plugin_filename, module_package):
    # Many branches, for the decision is very complex, pylint: disable=R0912

    debug(
        "Checking detail plug-in path '%s' '%s':",
        plugin_filename,
        module_package
    )

    plugin_info = considerFilename(
        module_package  = module_package,
        module_filename = plugin_filename
    )

    if plugin_info is not None:
        module, is_added = recurseTo(
            module_filename = plugin_info[0],
            module_relpath  = plugin_info[1],
            module_package  = module_package,
            module_kind     = "py",
            reason          = "Lives in plug-in directory."
        )

        if module:
            if not is_added:
                warning(
                    "Recursed to %s '%s' at '%s' twice.",
                    "package" if module.isPythonPackage() else "module",
                    module.getName(),
                    plugin_info[0]
                )

                if not isSameModulePath(module.getFilename(), plugin_info[0]):
                    warning(
                        "Duplicate ignored '%s'.",
                        plugin_info[1]
                    )

                    return

            debug(
                "Recursed to %s %s %s",
                module.getName(),
                module.getPackage(),
                module
            )

            if module.isPythonPackage():
                package_filename = module.getFilename()

                if Utils.isDir(package_filename):
                    # Must be a namespace package.
                    assert Utils.python_version >= 330

                    package_dir = package_filename

                    # Only include it, if it contains actual modules, which will
                    # recurse to this one and find it again.
                    useful = False
                else:
                    package_dir = Utils.dirname(package_filename)

                    # Real packages will always be included.
                    useful = True

                debug(
                    "Package directory %s",
                    package_dir
                )


                for sub_path, sub_filename in Utils.listDir(package_dir):
                    if sub_filename in ("__init__.py", "__pycache__"):
                        continue

                    assert sub_path != plugin_filename

                    if Importing.isPackageDir(sub_path) or \
                       sub_path.endswith(".py"):
                        _checkPluginPath(sub_path, module.getFullName())
            else:
                # Modules should always be included.
                useful = True

            if useful:
                ModuleRegistry.addRootModule(module)

        else:
            warning("Failed to include module from '%s'.", plugin_info[0])
Beispiel #35
0
def checkPluginSinglePath(plugin_filename, module_package):
    # Many branches, for the decision is very complex, pylint: disable=too-many-branches

    debug("Checking detail plug-in path '%s' '%s':", plugin_filename, module_package)

    module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
        plugin_filename
    )

    if module_kind is not None:
        decision, reason = decideRecursion(
            module_filename=plugin_filename,
            module_name=module_name,
            module_package=module_package,
            module_kind=module_kind,
            extra_recursion=True,
        )

        if decision:
            module_relpath = relpath(plugin_filename)

            module, is_added = recurseTo(
                module_filename=plugin_filename,
                module_relpath=module_relpath,
                module_package=module_package,
                module_kind=module_kind,
                reason=reason,
            )

            if module:
                if not is_added:
                    warning(
                        "Recursed to %s '%s' at '%s' twice.",
                        "package" if module.isCompiledPythonPackage() else "module",
                        module.getName(),
                        plugin_filename,
                    )

                    if not isSameModulePath(module.getFilename(), plugin_filename):
                        warning(
                            "Duplicate '%s' of '%s' ignored .",
                            plugin_filename,
                            module.getFilename(),
                        )

                        return

                debug(
                    "Recursed to %s %s %s",
                    module.getName(),
                    module.getPackage(),
                    module,
                )

                ImportCache.addImportedModule(module)

                if module.isCompiledPythonPackage():
                    package_filename = module.getFilename()

                    if os.path.isdir(package_filename):
                        # Must be a namespace package.
                        assert python_version >= 300

                        package_dir = package_filename

                        # Only include it, if it contains actual modules, which will
                        # recurse to this one and find it again.
                    else:
                        package_dir = os.path.dirname(package_filename)

                        # Real packages will always be included.
                        ModuleRegistry.addRootModule(module)

                    debug("Package directory %s", package_dir)

                    for sub_path, sub_filename in listDir(package_dir):
                        if sub_filename in ("__init__.py", "__pycache__"):
                            continue

                        assert sub_path != plugin_filename

                        if Importing.isPackageDir(sub_path) or sub_path.endswith(".py"):
                            checkPluginSinglePath(sub_path, module.getFullName())

                elif module.isCompiledPythonModule():
                    ModuleRegistry.addRootModule(module)

            else:
                warning("Failed to include module from '%s'.", plugin_filename)
Beispiel #36
0
def _checkPluginPath(plugin_filename, module_package):
    # Many branches, for the decision is very complex, pylint: disable=too-many-branches

    debug("Checking detail plug-in path '%s' '%s':", plugin_filename,
          module_package)

    module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
        plugin_filename)

    if module_kind is not None:
        decision, reason = decideRecursion(module_filename=plugin_filename,
                                           module_name=module_name,
                                           module_package=module_package,
                                           module_kind=module_kind,
                                           extra_recursion=True)

        if decision:
            module_relpath = relpath(plugin_filename)

            module, is_added = recurseTo(module_filename=plugin_filename,
                                         module_relpath=module_relpath,
                                         module_package=module_package,
                                         module_kind="py",
                                         reason=reason)

            if module:
                if not is_added:
                    warning(
                        "Recursed to %s '%s' at '%s' twice.", "package"
                        if module.isCompiledPythonPackage() else "module",
                        module.getName(), plugin_filename)

                    if not isSameModulePath(module.getFilename(),
                                            plugin_filename):
                        warning("Duplicate '%s' of '%s' ignored .",
                                plugin_filename, module.getFilename())

                        return

                debug("Recursed to %s %s %s", module.getName(),
                      module.getPackage(), module)

                ImportCache.addImportedModule(module)

                if module.isCompiledPythonPackage():
                    package_filename = module.getFilename()

                    if os.path.isdir(package_filename):
                        # Must be a namespace package.
                        assert python_version >= 330

                        package_dir = package_filename

                        # Only include it, if it contains actual modules, which will
                        # recurse to this one and find it again.
                    else:
                        package_dir = os.path.dirname(package_filename)

                        # Real packages will always be included.
                        ModuleRegistry.addRootModule(module)

                    debug("Package directory %s", package_dir)

                    for sub_path, sub_filename in listDir(package_dir):
                        if sub_filename in ("__init__.py", "__pycache__"):
                            continue

                        assert sub_path != plugin_filename

                        if Importing.isPackageDir(sub_path) or \
                           sub_path.endswith(".py"):
                            _checkPluginPath(sub_path, module.getFullName())

                elif module.isCompiledPythonModule():
                    ModuleRegistry.addRootModule(module)

            else:
                warning("Failed to include module from '%s'.", plugin_filename)
Beispiel #37
0
def checkPluginSinglePath(plugin_filename, module_package):
    # Many branches, for the decision is very complex, pylint: disable=too-many-branches

    if Options.isShowInclusion():
        recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
                              (plugin_filename, module_package))

    module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
        plugin_filename)

    module_name = ModuleName.makeModuleNameInPackage(module_name,
                                                     module_package)

    if module_kind is not None:
        decision, reason = decideRecursion(
            module_filename=plugin_filename,
            module_name=module_name,
            module_kind=module_kind,
            extra_recursion=True,
        )

        if decision:
            module_relpath = relpath(plugin_filename)

            module, is_added = recurseTo(
                module_filename=plugin_filename,
                module_relpath=module_relpath,
                module_package=module_package,
                module_kind=module_kind,
                reason=reason,
            )

            if module:
                if not is_added:
                    recursion_logger.warning(
                        "Recursed to %s '%s' at '%s' twice." % (
                            "package"
                            if module.isCompiledPythonPackage() else "module",
                            module.getName(),
                            plugin_filename,
                        ))

                    if not isSameModulePath(module.getFilename(),
                                            plugin_filename):
                        recursion_logger.warning(
                            "Duplicate '%s' of '%s' ignored ." % (
                                plugin_filename,
                                module.getFilename(),
                            ))

                        return

                if Options.isShowInclusion():
                    recursion_logger.info("Recursed to '%s' %s" % (
                        module.getFullName(),
                        module,
                    ))

                ImportCache.addImportedModule(module)

                if module.isCompiledPythonPackage():
                    package_filename = module.getFilename()

                    if os.path.isdir(package_filename):
                        # Must be a namespace package.
                        assert python_version >= 0x300

                        package_dir = package_filename

                        # Only include it, if it contains actual modules, which will
                        # recurse to this one and find it again.
                    else:
                        package_dir = os.path.dirname(package_filename)

                        # Real packages will always be included.
                        ModuleRegistry.addRootModule(module)

                    if Options.isShowInclusion():
                        recursion_logger.info("Package directory '%s'." %
                                              package_dir)

                    for sub_path, sub_filename in listDir(package_dir):
                        if sub_filename in ("__init__.py", "__pycache__"):
                            continue

                        assert sub_path != plugin_filename

                        if Importing.isPackageDir(
                                sub_path) and not os.path.exists(sub_path +
                                                                 ".py"):
                            checkPluginSinglePath(
                                sub_path, module_package=module.getFullName())
                        elif sub_path.endswith(".py"):
                            checkPluginSinglePath(
                                sub_path, module_package=module.getFullName())

                elif module.isCompiledPythonModule():
                    ModuleRegistry.addRootModule(module)
                elif module.isPythonShlibModule():
                    if Options.isStandaloneMode():
                        ModuleRegistry.addRootModule(module)

            else:
                recursion_logger.warning(
                    "Failed to include module from '%s'." % plugin_filename)