Beispiel #1
0
    def _attemptRecursion(self, trace_collection):
        found = False

        parent_module = self.getParentModule()

        if parent_module.isCompiledPythonPackage():
            parent_package = parent_module.getFullName()
        else:
            parent_package = self.getParentModule().getPackage()

        module_package, module_filename, self.finding = findModule(
            importing      = self,
            module_name    = self.getModuleName(),
            parent_package = parent_package,
            level          = self.getLevel(),
            warn           = True
        )

        if module_filename is not None:
            self.imported_module = self._consider(
                trace_collection = trace_collection,
                module_filename  = module_filename,
                module_package   = module_package
            )

            if self.imported_module is not None:
                found = self.imported_module.getFullName()

                self.found_modules = []

                import_list = self.getImportList()

                if import_list and \
                   self.imported_module.isCompiledPythonPackage():
                    for import_item in import_list:
                        if import_item == '*':
                            continue

                        module_package, module_filename, _finding = findModule(
                            importing      = self,
                            module_name    = import_item,
                            parent_package = found,
                            level          = -1,
                            warn           = False
                        )

                        if module_filename is not None:
                            sub_imported_module = self._consider(
                                trace_collection = trace_collection,
                                module_filename  = module_filename,
                                module_package   = module_package
                            )

                            if sub_imported_module is not None:
                                self.found_modules.append(
                                    sub_imported_module.getFullName()
                                )

        return found
Beispiel #2
0
    def _attemptRecursion(self, constraint_collection):
        found = False

        parent_module = self.getParentModule()

        if parent_module.isCompiledPythonPackage():
            parent_package = parent_module.getFullName()
        else:
            parent_package = self.getParentModule().getPackage()

        module_package, module_filename, _finding = findModule(
            importing      = self,
            module_name    = self.getModuleName(),
            parent_package = parent_package,
            level          = self.getLevel(),
            warn           = True
        )

        if module_filename is not None:
            imported_module = self._consider(
                constraint_collection = constraint_collection,
                module_filename       = module_filename,
                module_package        = module_package
            )

            if imported_module is not None:
                found = imported_module.getFullName()

                self.found_modules = []

                import_list = self.getImportList()

                if import_list and imported_module.isCompiledPythonPackage():
                    for import_item in import_list:
                        if import_item == '*':
                            continue

                        module_package, module_filename, _finding = findModule(
                            importing      = self,
                            module_name    = import_item,
                            parent_package = imported_module.getFullName(),
                            level          = -1,
                            warn           = False
                        )

                        if module_filename is not None:
                            sub_imported_module = self._consider(
                                constraint_collection = constraint_collection,
                                module_filename       = module_filename,
                                module_package        = module_package
                            )

                            if sub_imported_module is not None:
                                self.found_modules.append(sub_imported_module.getFullName())

            return found
Beispiel #3
0
    def _attemptRecursion(self, constraint_collection):
        assert self.getModule() is None

        parent_module = self.getParentModule()

        if parent_module.isPythonPackage():
            parent_package = parent_module.getFullName()
        else:
            parent_package = self.getParentModule().getPackage()

        module_package, module_filename, _finding = findModule(
            source_ref     = self.source_ref,
            module_name    = self.getModuleName(),
            parent_package = parent_package,
            level          = self.getLevel(),
            warn           = True
        )

        if module_filename is not None:
            imported_module = self._consider(
                constraint_collection = constraint_collection,
                module_filename       = module_filename,
                module_package        = module_package
            )

            if imported_module is not None:
                self.setModule(imported_module)
                self.found_modules = []

                import_list = self.getImportList()

                if import_list and imported_module.isPythonPackage():
                    for import_item in import_list:
                        if import_item == '*':
                            continue

                        module_package, module_filename, _finding = findModule(
                            source_ref     = self.source_ref,
                            module_name    = import_item,
                            parent_package = imported_module.getFullName(),
                            level          = -1,
                            warn           = False
                        )

                        if module_filename is not None:
                            sub_imported_module = self._consider(
                                constraint_collection = constraint_collection,
                                module_filename       = module_filename,
                                module_package        = module_package
                            )

                            if sub_imported_module is not None:
                                self.found_modules.append(sub_imported_module)
Beispiel #4
0
    def attemptRecursion(self):
        # Make sure the package is recursed to.

        # Return the list of newly added modules.
        result = []

        if self.package_name is not None and self.package is None:
            self.package = getModuleByName(self.package_name)

        if self.package_name is not None and self.package is None:
            package_package, package_filename, finding = findModule(
                importing=self,
                module_name=self.package_name,
                parent_package=None,
                level=1,
                warn=python_version < 330)

            # TODO: Temporary, if we can't find the package for Python3.3 that
            # is semi-OK, maybe.
            if python_version >= 330 and not package_filename:
                return []

            if self.package_name == "uniconvertor.app.modules":
                return []

            assert package_filename is not None, (self.package_name, finding)

            _package_name, package_kind = getModuleNameAndKindFromFilename(
                package_filename)
            # assert _package_name == self.package_name, (package_filename, _package_name, self.package_name)

            decision, _reason = decideRecursion(
                module_filename=package_filename,
                module_name=self.package_name,
                module_package=package_package,
                module_kind=package_kind)

            if decision is not None:
                self.package, is_added = recurseTo(
                    module_package=package_package,
                    module_filename=package_filename,
                    module_relpath=relpath(package_filename),
                    module_kind="py",
                    reason="Containing package of recursed module '%s'." %
                    self.getFullName(),
                )

                if is_added:
                    result.append(self.package)

        if self.package:
            from nuitka.ModuleRegistry import addUsedModule

            addUsedModule(self.package)

            #            print "Recursed to package", self.package_name
            result.extend(self.package.attemptRecursion())

        return result
Beispiel #5
0
    def attemptRecursion(self):
        # Make sure the package is recursed to.

        # Return the list of newly added modules.
        result = []

        if self.package_name is not None and self.package is None:
            package_package, package_filename, _finding = findModule(
                importing      = self,
                module_name    = self.package_name,
                parent_package = None,
                level          = 1,
                warn           = python_version < 330
            )

            # TODO: Temporary, if we can't find the package for Python3.3 that
            # is semi-OK, maybe.
            if python_version >= 330 and not package_filename:
                return []

            if self.package_name == "uniconvertor.app.modules":
                return []

            assert package_filename is not None, self.package_name

            _package_name, package_kind = getModuleNameAndKindFromFilename(package_filename)
            # assert _package_name == self.package_name, (package_filename, _package_name, self.package_name)

            decision, _reason = decideRecursion(
                module_filename = package_filename,
                module_name     = self.package_name,
                module_package  = package_package,
                module_kind     = package_kind
            )

            if decision is not None:
                imported_module, is_added = recurseTo(
                    module_package  = package_package,
                    module_filename = package_filename,
                    module_relpath  = Utils.relpath(package_filename),
                    module_kind     = "py",
                    reason          = "Containing package of recursed module '%s'." % self.getFullName(),
                )

                self.package = imported_module

                if is_added:
                    result.append(imported_module)

        if self.package:
            from nuitka.ModuleRegistry import addUsedModule

            addUsedModule(self.package)

#            print "Recursed to package", self.package_name
            result.extend(self.package.attemptRecursion())

        return result
Beispiel #6
0
def getScanDirectories(package_name, original_dir):
    cache_key = package_name, original_dir

    if cache_key in _scan_dir_cache:
        return _scan_dir_cache[cache_key]

    scan_dirs = [sys.prefix]

    if package_name is not None:
        from nuitka.importing.Importing import findModule

        package_dir = findModule(None, package_name, None, 0, False)[1]

        if os.path.isdir(package_dir):
            scan_dirs.append(package_dir)
            scan_dirs.extend(getSubDirectories(package_dir))

    if original_dir is not None:
        scan_dirs.append(original_dir)
        scan_dirs.extend(getSubDirectories(original_dir))

    for path_dir in os.environ["PATH"].split(";"):
        if not os.path.isdir(path_dir):
            continue

        if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
            continue
        if areSamePaths(path_dir,
                        os.path.join(os.environ["SYSTEMROOT"], "System32")):
            continue
        if areSamePaths(path_dir,
                        os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
            continue

        scan_dirs.append(path_dir)

    result = []

    # Remove directories that hold no DLLs.
    for scan_dir in scan_dirs:
        sys.stdout.flush()

        # These are useless, but plenty.
        if os.path.basename(scan_dir) == "__pycache__":
            continue

        # No DLLs, no use.
        if not any(entry[1].lower().endswith(".dll")
                   for entry in listDir(scan_dir)):
            continue

        result.append(scan_dir)

    _scan_dir_cache[cache_key] = result
    return result
Beispiel #7
0
    def _buildPackage(self, build_lib):
        # Nuitka wants the main package by filename, probably we should stop
        # needing that.
        from nuitka.importing.Importing import findModule, setMainScriptDirectory
        from nuitka.utils.Execution import getExecutablePath

        old_dir = os.getcwd()
        os.chdir(build_lib)

        # Search in the build directory preferrably.
        setMainScriptDirectory('.')

        package, main_filename, finding = findModule(
            importing=None,
            module_name=self.main_package,
            parent_package=None,
            level=0,
            warn=False)

        # Check expectations, e.g. do not compile built-in modules.
        assert finding == "absolute", finding
        assert package is None, package

        recurse_packages = self.compile_packages.copy()
        if os.path.isdir(self.main_package):
            # Include all python files in wheel
            for root, dirs, files in os.walk(self.main_package):
                for fn in fnmatch.filter(files, '*.py'):
                    module = os.path.join(
                        root, fn)[:-3] if fn != '__init__.py' else root
                    pypath = ".".join(module.split(os.sep))
                    recurse_packages.append(pypath)

        command = [
            sys.executable,
            "-m",
            "nuitka",
            "--module",
            "--plugin-enable=pylint-warnings",
            "--output-dir=%s" % build_lib,
            "--recurse-dir=%s" % self.main_package,
            "--recurse-not-to=*.tests",
            "--show-modules",
            "--remove-output",
            main_filename,
        ] + ["--recurse-to=%s" % p for p in recurse_packages]

        subprocess.check_call(command)
        os.chdir(old_dir)

        self.build_lib = build_lib
Beispiel #8
0
def getPackageSpecificDLLDirectories(package_name):
    scan_dirs = OrderedSet()

    if package_name is not None:
        from nuitka.importing.Importing import findModule

        package_dir = findModule(None, package_name, None, 0, False)[1]

        if os.path.isdir(package_dir):
            scan_dirs.add(package_dir)
            scan_dirs.update(
                getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))

        scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))

    return scan_dirs
Beispiel #9
0
    def _buildPackage(self, build_lib):
        # Nuitka wants the main package by filename, probably we should stop
        # needing that.
        from nuitka.importing.Importing import findModule, setMainScriptDirectory
        from nuitka.utils.Execution import getExecutablePath

        old_dir = os.getcwd()
        os.chdir(build_lib)

        # Search in the build directory preferrably.
        setMainScriptDirectory('.')

        package, main_filename, finding = findModule(
            importing=None,
            module_name=self.main_package,
            parent_package=None,
            level=0,
            warn=False)

        # Check expectations, e.g. do not compile built-in modules.
        assert finding == "absolute", finding
        assert package is None, package

        nuitka_binary = getExecutablePath("nuitka")
        if nuitka_binary is None:
            sys.exit("Error, cannot find nuitka binary in PATH.")

        command = [
            sys.executable,
            nuitka_binary,
            "--module",
            "--plugin-enable=pylint-warnings",
            "--output-dir=%s" % build_lib,
            "--recurse-to={%s}" % ','.join(self.compile_packages),
            "--recurse-dir=%s" % self.main_package,
            "--recurse-not-to=*.tests",
            "--show-modules",
            "--remove-output",
            main_filename,
        ]

        subprocess.check_call(command)
        os.chdir(old_dir)

        self.build_lib = build_lib
Beispiel #10
0
    def _attemptRecursion(self, trace_collection, module_name):
        # Complex stuff, pylint: disable=too-many-branches

        parent_module = self.getParentModule()

        if parent_module.isCompiledPythonPackage():
            parent_package = parent_module.getFullName()
        else:
            parent_package = self.getParentModule().getPackage()

        level = self.getLevel()

        if level is None:
            level = 0 if parent_module.getFutureSpec().isAbsoluteImport() else -1
        elif not level.isCompileTimeConstant():
            return
        else:
            level = level.getCompileTimeConstant()

        # TODO: Catch this as a static error maybe.
        if type(level) not in (int, long):
            return

        module_package, module_filename, self.finding = findModule(
            importing=self,
            module_name=module_name,
            parent_package=parent_package,
            level=level,
            warn=True,
        )

        if module_filename is not None:
            imported_module = self._consider(
                trace_collection=trace_collection,
                module_filename=module_filename,
                module_package=module_package,
            )

            if imported_module is not None:
                self.imported_module_desc = (
                    imported_module.getFullName(),
                    imported_module.getFilename(),
                )

                import_list = self.getFromList()

                if import_list is not None:
                    if import_list.isCompileTimeConstant():
                        import_list = import_list.getCompileTimeConstant()

                    if type(import_list) not in (tuple, list):
                        import_list = None

                if import_list and imported_module.isCompiledPythonPackage():
                    for import_item in import_list:
                        if import_item == "*":
                            continue

                        module_package, module_filename, _finding = findModule(
                            importing=self,
                            module_name=import_item,
                            parent_package=imported_module.getFullName(),
                            level=-1,  # Relative import, so child is used.
                            warn=False,
                        )

                        if module_filename is not None:
                            sub_imported_module = self._consider(
                                trace_collection=trace_collection,
                                module_filename=module_filename,
                                module_package=module_package,
                            )

                            if sub_imported_module is not None:
                                self.import_list_modules_desc.append(
                                    (
                                        sub_imported_module.getFullName(),
                                        sub_imported_module.getFilename(),
                                    )
                                )
        else:
            while "." in module_name:
                module_name = ".".join(module_name.split(".")[:-1])

                module_package, module_filename, _finding = findModule(
                    importing=self,
                    module_name=module_name,
                    parent_package=parent_package,
                    level=level,
                    warn=True,
                )

                if module_filename is not None:
                    package_module = self._consider(
                        trace_collection=trace_collection,
                        module_filename=module_filename,
                        module_package=module_package,
                    )

                    if package_module is not None:
                        if self.package_modules_desc is None:
                            self.package_modules_desc = []

                        self.package_modules_desc.append(
                            (package_module.getFullName(), package_module.getFilename())
                        )
Beispiel #11
0
    def _build(self, build_lib):
        # High complexity, pylint: disable=too-many-branches,too-many-locals

        # Nuitka wants the main package by filename, probably we should stop
        # needing that.
        from nuitka.importing.Importing import findModule, setMainScriptDirectory
        from nuitka.utils.ModuleNames import ModuleName
        from nuitka.__past__ import (  # pylint: disable=I0021,redefined-builtin
            Iterable,
            unicode,
        )

        old_dir = os.getcwd()
        os.chdir(build_lib)

        # Search in the build directory preferably.
        setMainScriptDirectory(".")

        to_builds = self._find_to_build()
        for to_build in to_builds:
            package, main_filename, finding = findModule(
                importing=None,
                module_name=ModuleName(to_build.module_name),
                parent_package=None,
                level=0,
                warn=False,
            )

            # Check expectations, e.g. do not compile built-in modules.
            assert finding == "absolute", finding

            if package is not None:
                output_dir = os.path.join(build_lib, package)
            else:
                output_dir = build_lib

            command = [
                sys.executable,
                "-m",
                "nuitka",
                "--module",
                "--plugin-enable=pylint-warnings",
                "--output-dir=%s" % output_dir,
                "--nofollow-import-to=*.tests",
                "--show-modules",
                "--remove-output",
            ]

            if type(to_build) is PyPackage:
                command += [
                    "--include-package=%s" % package_name.replace("/", ".")
                    for package_name in to_build.related_packages
                ]

            else:  # type(to_build) is PyModule
                command += [
                    "--include-module=%s" % module_name
                    for module_name in to_build.related_modules
                ]

            # Process any extra options from setuptools
            if "nuitka" in self.distribution.command_options:
                for option, value in self.distribution.command_options[
                    "nuitka"
                ].items():
                    option = "--" + option.lstrip("-")
                    if value is None:
                        command.append(option)
                    elif isinstance(value, bool):
                        option = "--" + ("no" if not value else "") + option.lstrip("-")
                        command.append(option)
                    elif isinstance(value, Iterable) and not isinstance(
                        value, (unicode, bytes, str)
                    ):
                        for val in value:
                            command.append("%s=%s" % (option, val))
                    else:
                        command.append("%s=%s" % (option, value))

            command.append(main_filename)

            # added for clarity
            my_print("Building: %s" % to_build, style="yellow")
            subprocess.check_call(command, cwd=build_lib)

            for root, _, filenames in os.walk(build_lib):
                for filename in filenames:
                    fullpath = os.path.join(root, filename)

                    if fullpath.lower().endswith((".py", ".pyw", ".pyc", ".pyo")):
                        os.unlink(fullpath)

            os.chdir(old_dir)

            self.build_lib = build_lib
Beispiel #12
0
    def _attemptRecursion(self, trace_collection, module_name):
        # Complex stuff, pylint: disable=too-many-branches

        parent_module = self.getParentModule()

        parent_package = parent_module.getFullName()
        if not parent_module.isCompiledPythonPackage():
            parent_package = parent_package.getPackageName()

        level = self.subnode_level

        if level is None:
            level = 0 if parent_module.getFutureSpec().isAbsoluteImport(
            ) else -1
        elif not level.isCompileTimeConstant():
            return
        else:
            level = level.getCompileTimeConstant()

        # TODO: Catch this as a static error maybe.
        if type(level) not in (int, long):
            return

        module_package, self.module_filename, self.finding = findModule(
            importing=self,
            module_name=ModuleName(module_name),
            parent_package=parent_package,
            level=level,
            warn=True,
        )

        if self.module_filename is not None:
            imported_module = self._consider(
                trace_collection=trace_collection,
                module_filename=self.module_filename,
                module_package=module_package,
            )

            if imported_module is not None:
                self.imported_module_desc = (
                    imported_module.getFullName(),
                    imported_module.getFilename(),
                )

                import_list = self.subnode_fromlist

                if import_list is not None:
                    if import_list.isCompileTimeConstant():
                        import_list = import_list.getCompileTimeConstant()

                    if type(import_list) not in (tuple, list):
                        import_list = None

                if import_list and imported_module.isCompiledPythonPackage():
                    for import_item in import_list:
                        if import_item == "*":
                            continue

                        module_package, module_filename, _finding = findModule(
                            importing=self,
                            module_name=ModuleName(import_item),
                            parent_package=imported_module.getFullName(),
                            level=-1,  # Relative import, so child is used.
                            warn=False,
                        )

                        if module_filename is not None:
                            sub_imported_module = self._consider(
                                trace_collection=trace_collection,
                                module_filename=module_filename,
                                module_package=module_package,
                            )

                            if sub_imported_module is not None:
                                self.import_list_modules_desc.append((
                                    sub_imported_module.getFullName(),
                                    sub_imported_module.getFilename(),
                                ))
        else:
            module_name = ModuleName(module_name)

            while True:
                module_name = module_name.getPackageName()

                if module_name is None:
                    break

                module_package, module_filename, _finding = findModule(
                    importing=self,
                    module_name=module_name,
                    parent_package=parent_package,
                    level=level,
                    warn=True,
                )

                if module_filename is not None:
                    package_module = self._consider(
                        trace_collection=trace_collection,
                        module_filename=module_filename,
                        module_package=module_package,
                    )

                    if package_module is not None:
                        if self.package_modules_desc is None:
                            self.package_modules_desc = []

                        self.package_modules_desc.append(
                            (package_module.getFullName(),
                             package_module.getFilename()))
Beispiel #13
0
def _detectBinaryPathDLLsWindows(is_main_executable, source_dir, original_dir,
                                 binary_filename, package_name):
    # This is complex, as it also includes the caching mechanism
    # pylint: disable=too-many-locals

    result = set()

    cache_filename = _getCacheFilename(is_main_executable, source_dir,
                                       original_dir, binary_filename)

    if os.path.exists(cache_filename
                      ) and not Options.shallNotUseDependsExeCachedResults():
        for line in open(cache_filename):
            line = line.strip()

            result.add(line)

        return result

    # User query should only happen once if at all.
    with _withLock():
        depends_exe = getDependsExePath()

    scan_dirs = [sys.prefix]

    if package_name is not None:
        from nuitka.importing.Importing import findModule

        package_dir = findModule(None, package_name, None, 0, False)[1]

        if os.path.isdir(package_dir):
            scan_dirs.append(package_dir)
            scan_dirs.extend(getSubDirectories(package_dir))

    if original_dir is not None:
        scan_dirs.append(original_dir)
        scan_dirs.extend(getSubDirectories(original_dir))

    with _withLock():
        # The search order by default prefers the system directory, where a
        # wrong "PythonXX.dll" might be living.
        with open(binary_filename + ".dwp", 'w') as dwp_file:
            dwp_file.write(
                """\
KnownDLLs
%(original_dirs)s
SysPath
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""" % {
                    "original_dirs":
                    '\n'.join("UserDir %s" % dirname for dirname in scan_dirs
                              if not os.path.basename(dirname) == "__pycache__"
                              if any(entry[1].lower().endswith(".dll")
                                     for entry in listDir(dirname))),
                })

        # Starting the process while locked, so file handles are not duplicated.
        depends_exe_process = subprocess.Popen(
            (depends_exe, "-c", "-ot%s" % binary_filename + ".depends",
             "-d:%s" % binary_filename + ".dwp", "-f1", "-pa1", "-ps1",
             binary_filename))

    # TODO: Exit code should be checked.
    depends_exe_process.wait()

    # Opening the result under lock, so it is not getting locked by new processes.
    with _withLock():
        _parseDependsExeOutput(binary_filename + ".depends", result)

    deleteFile(binary_filename + ".depends", must_exist=True)
    deleteFile(binary_filename + ".dwp", must_exist=True)

    if not Options.shallNotStoreDependsExeCachedResults():
        with open(cache_filename, 'w') as cache_file:
            for dll_filename in result:
                print(dll_filename, file=cache_file)

    return result
Beispiel #14
0
    def _buildPackage(self, build_lib):
        # High complexity, pylint: disable=too-many-branches,too-many-locals

        # Nuitka wants the main package by filename, probably we should stop
        # needing that.
        from nuitka.importing.Importing import findModule, setMainScriptDirectory

        old_dir = os.getcwd()
        os.chdir(build_lib)

        # Search in the build directory preferably.
        setMainScriptDirectory(".")

        package, main_filename, finding = findModule(
            importing=None,
            module_name=self.main_package,
            parent_package=None,
            level=0,
            warn=False,
        )

        # Check expectations, e.g. do not compile built-in modules.
        assert finding == "absolute", finding
        assert package is None, package

        # If there are other files left over in the wheel after python scripts
        # are compiled, we'll keep the folder structure with the files in the wheel
        keep_resources = False

        python_files = []
        if os.path.isdir(main_filename):
            # Include all python files in wheel
            for root, dirs, files in os.walk(main_filename):
                if "__pycache__" in dirs:
                    dirs.remove("__pycache__")
                    shutil.rmtree(os.path.join(root, "__pycache__"))

                for fn in files:
                    if fn.lower().endswith((".py", ".pyc", ".pyo")):
                        # These files will definitely be deleted once nuitka
                        # has compiled the main_package
                        python_files.append(os.path.join(root, fn))
                    else:
                        keep_resources = True

        output_dir = build_lib

        command = [
            sys.executable,
            "-m",
            "nuitka",
            "--module",
            "--plugin-enable=pylint-warnings",
            "--output-dir=%s" % output_dir,
            "--include-package=%s" % self.main_package,
            "--nofollow-import-to=*.tests",
            "--show-modules",
            "--remove-output",
        ]

        # Process any extra options from setuptools
        if "nuitka" in self.distribution.command_options:
            for option, details in self.distribution.command_options["nuitka"].items():
                option = "--" + option.lstrip("-")
                _source, value = details
                if value is None:
                    command.append(option)
                elif isinstance(value, collections.Iterable) and not isinstance(
                    value, (unicode, bytes, str)
                ):
                    for val in value:
                        command.append("%s=%s" % (option, val))
                else:
                    command.append("%s=%s" % (option, value))

        command.append(main_filename)

        subprocess.check_call(command, cwd=build_lib)
        os.chdir(old_dir)

        self.build_lib = build_lib

        if keep_resources:
            # Delete the individual source files
            for fn in python_files:
                os.unlink(fn)
        else:
            # Delete the entire source copy of the module
            shutil.rmtree(os.path.join(self.build_lib, self.main_package))
Beispiel #15
0
def _detectBinaryPathDLLsWindowsPE(is_main_executable, source_dir,
                                   original_dir, binary_filename,
                                   package_name):
    # This is complex, as it also includes the caching mechanism
    # pylint: disable=too-many-branches,too-many-locals

    result = set()

    cache_filename = _getCacheFilename(is_main_executable, source_dir,
                                       original_dir, binary_filename)

    if (os.path.exists(cache_filename)
            and not Options.shallNotUseDependsExeCachedResults()):
        for line in getFileContentByLine(cache_filename):
            line = line.strip()

            result.add(line)

        return result

    scan_dirs = [sys.prefix]

    if package_name is not None:
        from nuitka.importing.Importing import findModule

        package_dir = findModule(None, package_name, None, 0, False)[1]

        if os.path.isdir(package_dir):
            scan_dirs.append(package_dir)
            scan_dirs.extend(getSubDirectories(package_dir))

    if os.path.isdir(original_dir):
        scan_dirs.append(original_dir)
        scan_dirs.extend(getSubDirectories(original_dir))

    if Options.isExperimental("use_pefile_fullrecurse"):
        try:
            scan_dirs.extend(getSubDirectories(get_python_lib()))
        except OSError:
            print(
                "Cannot recurse into site-packages for dependencies. Path not found."
            )
    else:
        # Fix for missing pywin32 inclusion when using pythonwin library, no way to detect that automagically
        # Since there are more than one dependencies on pywintypes37.dll, let's include this anyway
        # In recursive mode, using dirname(original_dir) won't always work, hence get_python_lib
        try:
            scan_dirs.append(os.path.join(get_python_lib(),
                                          "pywin32_system32"))
        except OSError:
            pass

    # Add native system directory based on pe file architecture and os architecture
    # Python 32: system32 = syswow64 = 32 bits systemdirectory
    # Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory
    binary_file_is_64bit = _isPE64(binary_filename)
    python_is_64bit = getArchitecture() == "x86_64"
    if binary_file_is_64bit is not python_is_64bit:
        print("Warning: Using Python x64=%s with x64=%s binary dependencies" %
              (binary_file_is_64bit, python_is_64bit))

    if binary_file_is_64bit:
        # This is actually not useful as of today since we don't compile 32 bits on 64 bits
        if python_is_64bit:
            scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
                                          "System32"))
        else:
            scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
                                          "SysWOW64"))
    else:
        scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "System32"))

    if Options.isExperimental("use_pefile_recurse"):
        # Recursive one level scanning of all .pyd and .dll in the original_dir too
        # This shall fix a massive list of missing dependencies that may come with included libraries which themselves
        # need to be scanned for inclusions
        for root, _, filenames in os.walk(original_dir):
            for optional_libary in filenames:
                if optional_libary.endswith(
                        ".dll") or optional_libary.endswith(".pyd"):
                    _parsePEFileOutput(os.path.join(root, optional_libary),
                                       scan_dirs, result)

    _parsePEFileOutput(binary_filename, scan_dirs, result)

    if not Options.shallNotStoreDependsExeCachedResults():
        with open(cache_filename, "w") as cache_file:
            for dll_filename in result:
                print(dll_filename, file=cache_file)

    return result
Beispiel #16
0
    def _attemptRecursion(self, trace_collection, module_name):
        # Complex stuff, pylint: disable=too-many-branches

        parent_module = self.getParentModule()

        if parent_module.isCompiledPythonPackage():
            parent_package = parent_module.getFullName()
        else:
            parent_package = self.getParentModule().getPackage()

        level = self.getLevel()

        if level is None:
            level = 0 if parent_module.getFutureSpec().isAbsoluteImport() else -1
        elif not level.isCompileTimeConstant():
            return
        else:
            level = level.getCompileTimeConstant()

        # TODO: Catch this as a static error maybe.
        if type(level) not in (int, long):
            return

        module_package, module_filename, self.finding = findModule(
            importing      = self,
            module_name    = module_name,
            parent_package = parent_package,
            level          = level,
            warn           = True
        )

        if module_filename is not None:
            self.imported_module = self._consider(
                trace_collection = trace_collection,
                module_filename  = module_filename,
                module_package   = module_package
            )

            if self.imported_module is not None:
                import_list = self.getFromList()

                if import_list is not None:
                    if import_list.isCompileTimeConstant():
                        import_list = import_list.getCompileTimeConstant()

                    if type(import_list) not in (tuple, list):
                        import_list = None

                if import_list and \
                   self.imported_module.isCompiledPythonPackage():
                    for import_item in import_list:
                        if import_item == '*':
                            continue

                        module_package, module_filename, _finding = findModule(
                            importing      = self,
                            module_name    = import_item,
                            parent_package = self.imported_module.getFullName(),
                            level          = -1, # Relative import, so child is used.
                            warn           = False
                        )

                        if module_filename is not None:
                            sub_imported_module = self._consider(
                                trace_collection = trace_collection,
                                module_filename  = module_filename,
                                module_package   = module_package
                            )

                            if sub_imported_module is not None:
                                self.import_list_modules.append(
                                    sub_imported_module.getFullName()
                                )
        else:
            while "." in module_name:
                module_name = ".".join(module_name.split(".")[:-1])

                module_package, module_filename, _finding = findModule(
                    importing      = self,
                    module_name    = module_name,
                    parent_package = parent_package,
                    level          = level,
                    warn           = True
                )

                if module_filename is not None:
                    package_module = self._consider(
                        trace_collection = trace_collection,
                        module_filename  = module_filename,
                        module_package   = module_package
                    )

                    if package_module is not None:
                        if self.package_modules is None:
                            self.package_modules = []

                        self.package_modules.append(package_module)
Beispiel #17
0
def getScanDirectories(package_name, original_dir):
    # Many cases, pylint: disable=too-many-branches

    cache_key = package_name, original_dir

    if cache_key in _scan_dir_cache:
        return _scan_dir_cache[cache_key]

    scan_dirs = [sys.prefix]

    if package_name is not None:
        from nuitka.importing.Importing import findModule

        package_dir = findModule(None, package_name, None, 0, False)[1]

        if os.path.isdir(package_dir):
            scan_dirs.append(package_dir)
            scan_dirs.extend(getSubDirectories(package_dir))

    if original_dir is not None:
        scan_dirs.append(original_dir)
        scan_dirs.extend(getSubDirectories(original_dir))

    if (Utils.isWin32Windows() and package_name is not None
            and package_name.isBelowNamespace("win32com")):
        pywin32_dir = getPyWin32Dir()

        if pywin32_dir is not None:
            scan_dirs.append(pywin32_dir)

    for path_dir in os.environ["PATH"].split(";"):
        if not os.path.isdir(path_dir):
            continue

        if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
            continue
        if areSamePaths(path_dir,
                        os.path.join(os.environ["SYSTEMROOT"], "System32")):
            continue
        if areSamePaths(path_dir,
                        os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
            continue

        scan_dirs.append(path_dir)

    result = []

    # Remove directories that hold no DLLs.
    for scan_dir in scan_dirs:
        sys.stdout.flush()

        # These are useless, but plenty.
        if os.path.basename(scan_dir) == "__pycache__":
            continue

        scan_dir = getDirectoryRealPath(scan_dir)

        # No DLLs, no use.
        if not any(entry[1].lower().endswith(".dll")
                   for entry in listDir(scan_dir)):
            continue

        result.append(os.path.realpath(scan_dir))

    _scan_dir_cache[cache_key] = result
    return result
Beispiel #18
0
    def _buildPackage(self, build_lib):
        # Nuitka wants the main package by filename, probably we should stop
        # needing that, pylint: disable=too-many-locals
        from nuitka.importing.Importing import findModule, setMainScriptDirectory

        old_dir = os.getcwd()
        os.chdir(build_lib)

        # Search in the build directory preferably.
        setMainScriptDirectory('.')

        package, main_filename, finding = findModule(
            importing      = None,
            module_name    = self.main_package,
            parent_package = None,
            level          = 0,
            warn           = False
        )

        # Check expectations, e.g. do not compile built-in modules.
        assert finding == "absolute", finding
        assert package is None, package

        # If there are other files left over in the wheel after python scripts
        # are compiled, we'll keep the folder structure with the files in the wheel
        keep_resources = False

        python_files = []
        if os.path.isdir(main_filename):
            # Include all python files in wheel
            for root, dirs, files in os.walk(main_filename):
                if "__pycache__" in dirs:
                    dirs.remove("__pycache__")
                    shutil.rmtree(os.path.join(root, "__pycache__"))

                for fn in files:
                    if fn.lower().endswith((".py", ".pyc", ".pyo")):
                        # These files will definitely be deleted once nuitka
                        # has compiled the main_package
                        python_files.append(os.path.join(root, fn))
                    else:
                        keep_resources = True

        output_dir = build_lib

        command = [
            sys.executable,
            "-m", "nuitka",
            "--module",
            "--plugin-enable=pylint-warnings",
            "--output-dir=%s" % output_dir,
            "--recurse-dir=%s" % self.main_package,
            "--recurse-to=%s" % self.main_package,
            "--recurse-not-to=*.tests",
            "--show-modules",
            "--remove-output",
            main_filename,
        ]

        subprocess.check_call(
            command
        )
        os.chdir(old_dir)

        self.build_lib = build_lib

        if keep_resources:
            # Delete the individual source files
            for fn in python_files:
                os.unlink(fn)
        else:
            # Delete the entire source copy of the module
            shutil.rmtree(os.path.join(self.build_lib, self.main_package))