예제 #1
0
파일: PluginBase.py 프로젝트: Xzarh/Nuitka
    def considerExtraDlls(self, dist_dir, module):
        full_name = module.getFullName()

        if full_name in ("PyQt4", "PyQt5"):
            if not Options.isExperimental():
               return

            qt_version = int(full_name[-1])

            plugin_dir, = self.getPyQtPluginDirs(qt_version)

            shutil.copytree(
                plugin_dir,
                Utils.joinpath(
                    dist_dir,
                    "qt-plugins"
                )
            )

            return [
                (filename, full_name)
                for filename in
                Utils.getFileList(plugin_dir)
            ]

        return ()
예제 #2
0
def checkPluginFilenamePattern(pattern):
    import sys, glob

    debug(
        "Checking plug-in pattern '%s':",
        pattern,
    )

    if Utils.isDir(pattern):
        sys.exit("Error, pattern cannot be a directory name.")

    found = False

    for filename in glob.iglob(pattern):
        if filename.endswith(".pyc"):
            continue

        if not Utils.isFile(filename):
            continue

        found = True
        _checkPluginPath(filename, None)

    if not found:
        warning("Didn't match any files against pattern '%s'." % pattern)
예제 #3
0
        def getPackageDirCandidates(element):
            yield Utils.joinpath(element, package_name), False

            # Hack for PyWin32. TODO: Move this __path__ extensions to
            # plug-in decisions.
            if package_name == "win32com":
                yield Utils.joinpath(element, "win32comext"), True
예제 #4
0
def detectPthImportedPackages():
    if not hasattr(sys.modules["site"], "getsitepackages"):
        return ()

    pth_imports = set()

    for prefix in sys.modules["site"].getsitepackages():
        if not Utils.isDir(prefix):
            continue

        for path, filename in Utils.listDir(prefix):
            if filename.endswith(".pth"):
                try:
                    for line in open(path, "rU"):
                        if line.startswith("import "):
                            if ';' in line:
                                line = line[:line.find(';')]

                            for part in line[7:].split(','):
                                pth_imports.add(part.strip())
                except OSError:
                    warning("Python installation problem, cannot read file '%s'.")


    return tuple(sorted(pth_imports))
예제 #5
0
    def considerExtraDlls(self, dist_dir, module):
        full_name = module.getFullName()

        if full_name in ("PyQt4", "PyQt5"):
            qt_version = int(full_name[-1])

            plugin_dir, = self.getPyQtPluginDirs(qt_version)

            target_plugin_dir = Utils.joinpath(
                dist_dir,
                full_name,
                "qt-plugins"
            )

            shutil.copytree(
                plugin_dir,
                target_plugin_dir
            )

            info("Copying all Qt plug-ins to '%s'." % target_plugin_dir)

            return [
                (filename, full_name)
                for filename in
                Utils.getFileList(target_plugin_dir)
            ]

        return ()
예제 #6
0
def detectBinaryDLLs(binary_filename, package_name):
    """ Detect the DLLs used by a binary.

        Using "ldd" (Linux), "depends.exe" (Windows), or "otool" (MacOS) the list
        of used DLLs is retrieved.
    """


    if Utils.getOS() in ("Linux", "NetBSD"):
        # TODO: FreeBSD may work the same way, not tested.

        return _detectBinaryPathDLLsLinuxBSD(
            binary_filename = binary_filename
        )
    elif Utils.getOS() == "Windows":
        return _detectBinaryPathDLLsWindows(
            binary_filename = binary_filename,
            package_name    = package_name
        )
    elif Utils.getOS() == "Darwin":
        return _detectBinaryPathDLLsMacOS(
            binary_filename = binary_filename
        )
    else:
        # Support your platform above.
        assert False, Utils.getOS()
예제 #7
0
def checkPluginPath(plugin_filename, module_package):
    debug(
        "Checking top level plug-in path %s %s",
        plugin_filename,
        module_package
    )

    plugin_info = considerFilename(
        module_package  = module_package,
        module_filename = plugin_filename
    )

    if plugin_info is not None:
        # File or package makes a difference, handle that
        if Utils.isFile(plugin_info[0]) or \
           Importing.isPackageDir(plugin_info[0]):
            _checkPluginPath(plugin_filename, module_package)
        elif Utils.isDir(plugin_info[0]):
            for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
                assert sub_filename != "__init__.py"

                if Importing.isPackageDir(sub_path) or \
                   sub_path.endswith(".py"):
                    _checkPluginPath(sub_path, None)
        else:
            warning("Failed to include module from '%s'.", plugin_info[0])
    else:
        warning("Failed to recurse to directory '%s'.", plugin_filename)
예제 #8
0
def getResultBasepath(main_module):
    assert main_module.isCompiledPythonModule()

    if Options.isStandaloneMode():
        return Utils.joinpath(
            getStandaloneDirectoryPath(main_module), Utils.basename(getTreeFilenameWithSuffix(main_module, ""))
        )
    else:
        return Options.getOutputPath(path=Utils.basename(getTreeFilenameWithSuffix(main_module, "")))
예제 #9
0
def isPreloadedPackagePath(path):
    path = Utils.normcase(path)

    for paths in getPreloadedPackagePaths().values():
        for element in paths:
            if Utils.normcase(element) == path:
                return True

    return False
예제 #10
0
    def _consider(self, constraint_collection, module_filename, module_package):
        assert module_package is None or \
              (type(module_package) is str and module_package != "")

        module_filename = Utils.normpath(module_filename)

        module_name, module_kind = getModuleNameAndKindFromFilename(module_filename)

        if module_kind is not None:
            decision, reason = decideRecursion(
                module_filename = module_filename,
                module_name     = module_name,
                module_package  = module_package,
                module_kind     = module_kind
            )

            if decision:
                module_relpath = Utils.relpath(module_filename)

                imported_module, added_flag = recurseTo(
                    module_package  = module_package,
                    module_filename = module_filename,
                    module_relpath  = module_relpath,
                    module_kind     = module_kind,
                    reason          = reason
                )

                if added_flag:
                    constraint_collection.signalChange(
                        "new_code",
                        imported_module.getSourceReference(),
                        "Recursed to module."
                    )

                return imported_module
            elif decision is None and module_kind == "py":
                if module_package is None:
                    module_fullpath = module_name
                else:
                    module_fullpath = module_package + '.' + module_name

                if module_filename not in self._warned_about and \
                   module_fullpath not in getModuleWhiteList():
                    self._warned_about.add(module_filename)

                    warning(
                        """\
Not recursing to '%(full_path)s' (%(filename)s), please specify \
--recurse-none (do not warn), \
--recurse-all (recurse to all), \
--recurse-not-to=%(full_path)s (ignore it), \
--recurse-to=%(full_path)s (recurse to it) to change.""" % {
                            "full_path" : module_fullpath,
                            "filename"  : module_filename
                        }
                    )
예제 #11
0
def getSconsBinaryCall():
    """ Return a way to execute Scons.

        Using potentially in-line copy if no system Scons is available
        or if we are on Windows.
    """
    if Utils.getOS() != "Windows" and Utils.isFile("/usr/bin/scons"):
        return ["/usr/bin/scons"]
    else:
        return [getPython2ExePath(), Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")]
예제 #12
0
def scanStandardLibraryPath(stdlib_dir):
    # There is a lot of black-listing here, done in branches, so there
    # is many of them, but that's acceptable, pylint: disable=R0912

    for root, dirs, filenames in os.walk(stdlib_dir):
        import_path = root[len(stdlib_dir):].strip("/\\")
        import_path = import_path.replace('\\', '.').replace('/','.')

        if import_path == "":
            if "site-packages" in dirs:
                dirs.remove("site-packages")
            if "dist-packages" in dirs:
                dirs.remove("dist-packages")
            if "test" in dirs:
                dirs.remove("test")
            if "idlelib" in dirs:
                dirs.remove("idlelib")
            if "turtledemo" in dirs:
                dirs.remove("turtledemo")

        if import_path in ("tkinter", "importlib", "ctypes"):
            if "test" in dirs:
                dirs.remove("test")

        if import_path == "lib2to3":
            if "tests" in dirs:
                dirs.remove("tests")

        if Utils.python_version >= 340 and Utils.getOS() == "Windows":
            if import_path == "multiprocessing":
                filenames.remove("popen_fork.py")
                filenames.remove("popen_forkserver.py")
                filenames.remove("popen_spawn_posix.py")

        if Utils.getOS() == "NetBSD":
            if import_path == "xml.sax":
                filenames.remove("expatreader.py")

        for filename in filenames:
            if filename.endswith(".py") and filename not in ignore_modules:
                module_name = filename[:-3]
                if import_path == "":
                    yield module_name
                else:
                    yield import_path + '.' + module_name

        if Utils.python_version >= 300:
            if "__pycache__" in dirs:
                dirs.remove("__pycache__")

        for dirname in dirs:
            if import_path == "":
                yield dirname
            else:
                yield import_path + '.' + dirname
예제 #13
0
def compileTree(main_module):
    source_dir = getSourceDirectoryPath(main_module)

    if not Options.shallOnlyExecCppCall():
        # Now build the target language code for the whole tree.
        makeSourceDirectory(
            main_module = main_module
        )

        if Options.isStandaloneMode():
            for module in detectLateImports():
                ModuleRegistry.addUncompiledModule(module)

        frozen_code = generateBytecodeFrozenCode()

        if frozen_code is not None:

            writeSourceCode(
                filename    = Utils.joinpath(
                    source_dir,
                    "__frozen.cpp"
                ),
                source_code = frozen_code
            )

        writeBinaryData(
            filename    = Utils.joinpath(source_dir, "__constants.bin"),
            binary_data = ConstantCodes.stream_data.getBytes()
        )
    else:
        source_dir = getSourceDirectoryPath(main_module)

        if not Utils.isFile(Utils.joinpath(source_dir, "__helpers.hpp")):
            sys.exit("Error, no previous build directory exists.")

    if Options.isShowProgress() or Options.isShowMemory():
        Tracing.printLine(
            "Total memory usage before running scons: {memory}:".format(
                memory = Utils.getHumanReadableProcessMemoryUsage()
            )
        )

    if Options.isShowMemory():
        InstanceCounters.printStats()

    if Options.shallNotDoExecCppCall():
        return True, {}

    # Run the Scons to build things.
    result, options = runScons(
        main_module = main_module,
        quiet       = not Options.isShowScons()
    )

    return result, options
예제 #14
0
    def getFilenames(module):
        base_filename = Utils.joinpath(
            source_dir, "module." + module.getFullName() if not module.isInternalModule() else module.getFullName()
        )

        # Note: Could detect if the file system is cases sensitive in source_dir
        # or not, but that's probably not worth the effort. False positives do
        # no harm at all.
        collision_filename = Utils.normcase(base_filename)

        return base_filename, collision_filename
예제 #15
0
def createNodeTree(filename):
    """ Create a node tree.

    Turn that source code into a node tree structure. If recursion into
    imported modules is available, more trees will be available during
    optimization, or immediately through recursed directory paths.

    """

    # First, build the raw node tree from the source code.
    main_module = Building.buildModuleTree(
        filename = filename,
        package  = None,
        is_top   = True,
        is_main  = not Options.shallMakeModule()
    )
    ModuleRegistry.addRootModule(main_module)

    # First remove old object files and old generated files, old binary or
    # module, and standalone mode program directory if any, they can only do
    # harm.
    source_dir = getSourceDirectoryPath(main_module)

    if not Options.shallOnlyExecCppCall():
        cleanSourceDirectory(source_dir)

    # Prepare the ".dist" directory, throwing away what was there before.
    if Options.isStandaloneMode():
        standalone_dir = getStandaloneDirectoryPath(main_module)
        shutil.rmtree(standalone_dir, ignore_errors = True)
        Utils.makePath(standalone_dir)

    Utils.deleteFile(
        path       = getResultFullpath(main_module),
        must_exist = False
    )

    # Second, do it for the directories given.
    for plugin_filename in Options.getShallFollowExtra():
        Recursion.checkPluginPath(
            plugin_filename = plugin_filename,
            module_package  = None
        )

    for pattern in Options.getShallFollowExtraFilePatterns():
        Recursion.checkPluginFilenamePattern(
            pattern = pattern
        )


    # Then optimize the tree and potentially recursed modules.
    Optimization.optimize()

    return main_module
예제 #16
0
def isPackageDir(dirname):
    """ Decide if a directory is a package.

        Before Python3.3 it's required to have a "__init__.py" file, but then
        it became impossible to decide, and for extra fun, there is also the
        extra packages provided via "*.pth" file tricks by "site.py" loading.
    """

    return Utils.isDir(dirname) and (
        Utils.python_version >= 330
        or Utils.isFile(Utils.joinpath(dirname, "__init__.py"))
        or isPreloadedPackagePath(dirname)
    )
예제 #17
0
def _setupSconsEnvironment():
    """ Setup the scons execution environment.

    For the scons inline copy on Windows needs to find the library, using
    the "SCONS_LIB_DIR" environment variable "NUITKA_SCONS". And for the
    target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
    Python DLL lives, in case it needs to be copied, and then the
    "NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.
    """

    # For Python2, avoid unicode working directory.
    if Utils.isWin32Windows() and python_version < 300:
        if os.getcwd() != os.getcwdu():
            os.chdir(getWindowsShortPathName(os.getcwdu()))

    if Utils.isWin32Windows():
        # On Win32, we use the Python.DLL path for some things. We pass it
        # via environment variable
        os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()

    os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable

    # Remove environment variables that can only harm if we have to switch
    # major Python versions, these cannot help Python2 to execute scons, this
    # is a bit of noise, but helpful.
    if python_version >= 300:
        if "PYTHONPATH" in os.environ:
            old_pythonpath = os.environ["PYTHONPATH"]
            del os.environ["PYTHONPATH"]
        else:
            old_pythonpath = None

        if "PYTHONHOME" in os.environ:
            old_pythonhome = os.environ["PYTHONHOME"]
            del os.environ["PYTHONHOME"]
        else:
            old_pythonhome = None

    yield

    if python_version >= 300:
        if old_pythonpath is not None:
            os.environ["PYTHONPATH"] = old_pythonpath

        if old_pythonhome is not None:
            os.environ["PYTHONHOME"] = old_pythonhome

    if Utils.isWin32Windows():
        del os.environ["NUITKA_PYTHON_DLL_PATH"]

    del os.environ["NUITKA_PYTHON_EXE_PATH"]
예제 #18
0
    def __init__(self, *args):
        QtGui.QDialog.__init__(self, *args)

        ui_dir = Utils.dirname(__file__)
        ui_filename = Utils.joinpath(ui_dir, "dialogs", "InspectPythonTree.ui")

        uic.loadUi(ui_filename, self)

        self.treeview_nodes.setSelectionMode(self.treeview_nodes.SingleSelection)

        self.displayed = None
        self.source_code = None
        self.model = None
        self.moving = None
예제 #19
0
def getSconsBinaryCall():
    """ Return a way to execute Scons.

        Using potentially in-line copy if no system Scons is available
        or if we are on Windows.
    """
    if Utils.getOS() != "Windows":
        scons_path = Execution.getExecutablePath("scons")

        if scons_path is not None:
            return [scons_path]

    return [
        getPython2ExePath(),
        Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")
    ]
예제 #20
0
def _detectedPrecompiledFile(filename, module_name, result, is_late):
    if filename.endswith(b".pyc"):
        if Utils.isFile(filename[:-1]):
            return _detectedSourceFile(
                filename    = filename[:-1],
                module_name = module_name,
                result      = result,
                is_late     = is_late
            )

    if Utils.python_version >= 300:
        filename = filename.decode("utf-8")

    if module_name in module_names:
        return

    debug(
        "Freezing module '%s' (from '%s').",
        module_name,
        filename
    )

    result.append(
        (
            module_name,
            loadCodeObjectData(
                precompiled_filename = filename
            ),
            "__init__" in filename,
            filename,
            is_late
        ),
    )

    module_names.add(module_name)
예제 #21
0
def detectEarlyImports():
    if Options.freezeAllStdlib():
        stdlib_modules = set()

        # Scan the standard library paths (multiple in case of virtualenv.
        for stdlib_dir in getStandardLibraryPaths():
            for module_name in scanStandardLibraryPath(stdlib_dir):
                stdlib_modules.add(module_name)

        import_code = "imports = " + repr(sorted(stdlib_modules)) + '\n'\
                      "for imp in imports:\n" \
                      "    try:\n" \
                      "        __import__(imp)\n" \
                      "    except ImportError:\n" \
                      "        pass\n"
    else:
        # TODO: Should recursively include all of encodings module.
        import_code = "import encodings.utf_8;import encodings.ascii;import encodings.idna;"

        if Utils.getOS() == "Windows":
            import_code += "import encodings.mbcs;import encodings.cp437;"

        # String method hex depends on it.
        if Utils.python_version < 300:
            import_code += "import encodings.hex_codec;"

        import_code += "import locale;"

    result = _detectImports(import_code, False)
    debug("Finished detecting early imports.")

    return result
예제 #22
0
def _normalizeModuleFilename(filename):
    if python_version >= 300:
        filename = filename.replace("__pycache__", "")

        suffix = ".cpython-%d.pyc" % (python_version // 10)

        if filename.endswith(suffix):
            filename = filename[:-len(suffix)] + ".py"
    else:
        if filename.endswith(".pyc"):
            filename = filename[:-3] + ".py"

    if Utils.basename(filename) == "__init__.py":
        filename = Utils.dirname(filename)

    return filename
예제 #23
0
파일: Building.py 프로젝트: Xzarh/Nuitka
def buildModuleTree(filename, package, is_top, is_main):
    module, source_ref, source_filename = decideModuleTree(
        filename = filename,
        package  = package,
        is_top   = is_top,
        is_main  = is_main,
        is_shlib = False
    )

    addImportedModule(
        module_relpath  = Utils.relpath(filename),
        imported_module = module
    )

    # If there is source code associated (not the case for namespace packages of
    # Python3.3 or higher, then read it.
    if source_filename is not None:
        createModuleTree(
            module      = module,
            source_ref  = source_ref,
            source_code = readSourceCodeFromFilename(source_filename),
            is_main     = is_main
        )

    return module
예제 #24
0
def buildSconsCommand(quiet, options):
    scons_command = getSconsBinaryCall()

    if quiet:
        scons_command.append("--quiet")

    scons_command += [
        # The scons file
        "-f",
        Utils.joinpath(getSconsDataPath(), "SingleExe.scons"),

        # Parallel compilation.
        "--jobs",
        str(Options.getJobLimit()),

        # Do not warn about deprecation from Scons
        "--warn=no-deprecated",

        # Don't load "site_scons" at all.
        "--no-site-dir",
    ]

    if Options.isShowScons():
        scons_command.append("--debug=explain")

    # Option values to provide to scons. Find these in the caller.
    for key, value in options.items():
        scons_command += [key + '=' + value]

    return scons_command
예제 #25
0
def _getPython2ExePathWindows():
    # Shortcuts for the default installation directories, to avoid going to
    # registry at all unless necessary. Any Python2 will do for Scons, so it
    # can be avoided.

    if os.path.isfile(r"c:\Python27\python.exe"):
        return r"c:\Python27\python.exe"
    elif os.path.isfile(r"c:\Python26\python.exe"):
        return r"c:\Python26\python.exe"

    # Windows only code, pylint: disable=E0602,F0401,I0021
    try:
        import _winreg as winreg
    except ImportError:
        import winreg  # lint:ok

    for search in ("2.7", "2.6"):
        for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
            for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
                try:
                    key = winreg.OpenKey(
                        hkey_branch,
                        r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
                        0,
                        winreg.KEY_READ | arch_key
                    )

                    return Utils.joinpath(
                        winreg.QueryValue(key, ""),
                        "python.exe"
                    )
                except WindowsError:  # @UndefinedVariable
                    pass
예제 #26
0
def getPython2ExePath():
    """ Find a way to call any Python2. Scons needs it."""
    if python_version < 300:
        return sys.executable
    elif Utils.getOS() == "Windows":
        python_exe = _getPython2ExePathWindows()

        if python_exe is not None:
            return python_exe
        else:
            sys.exit("""\
Error, while Nuitka is fully Python3 compatible, it needs to find a
Python2 executable under C:\\Python26 or C:\\Python27 to execute
scons which is not yet Python3 compatible.""")

    candidate = Execution.getExecutablePath("python2.7")
    if candidate is None:
        candidate = Execution.getExecutablePath("python2.6")

        if candidate is None:
            candidate = Execution.getExecutablePath("python2")

    # Our weakest bet is that there is no "python3" named "python", but we
    # take it, as on some systems it's true.
    if candidate is None:
        candidate = "python"

    return candidate
예제 #27
0
def _detectedPrecompiledFile(filename, module_name, result, user_provided,
                             technical):
    if filename.endswith(".pyc"):
        if Utils.isFile(filename[:-1]):
            return _detectedSourceFile(
                filename      = filename[:-1],
                module_name   = module_name,
                result        = result,
                user_provided = user_provided,
                technical     = technical
            )

    if module_name in module_names:
        return

    debug(
        "Freezing module '%s' (from '%s').",
        module_name,
        filename
    )

    result.append(
        makeUncompiledPythonModule(
            module_name   = module_name,
            bytecode      = loadCodeObjectData(
                precompiled_filename = filename
            ),
            is_package    = "__init__" in filename,
            filename      = filename,
            user_provided = user_provided,
            technical     = technical
        )
    )

    module_names.add(module_name)
예제 #28
0
def _detectedSourceFile(filename, module_name, result, is_late):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(
            filename    = filename,
            module_name = "_collections_abc",
            result      = result,
            is_late     = is_late
        )

    source_code = readSourceCodeFromFilename(module_name, filename)

    if Utils.python_version >= 300:
        filename = filename.decode("utf-8")

    if module_name == "site":
        if source_code.startswith("def ") or source_code.startswith("class "):
            source_code = '\n' + source_code

        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep,
            Utils.basename(filename),
            source_code
        )

    debug(
        "Freezing module '%s' (from '%s').",
        module_name,
        filename
    )

    result.append(
        (
            module_name,
            marshal.dumps(
                compile(source_code, filename, "exec")
            ),
            Utils.basename(filename) == "__init__.py",
            filename,
            is_late
        )
    )

    module_names.add(module_name)
예제 #29
0
    def considerImplicitImports(self, module, signal_change):
        """ Consider module imports.

            You will most likely want to look at "module.getFullName()" to get
            the fully qualified module or package name.

            You do not want to overload this method, but rather the things it
            calls, as the "signal_change" part of this API is not to be cared
            about. Most prominently "getImplicitImports()".
        """

        for full_name in self.getImplicitImports(module.getFullName()):
            module_name = full_name.split('.')[-1]
            module_package = '.'.join(full_name.split('.')[:-1]) or None

            module_filename = self.locateModule(
                source_ref     = module.getSourceReference(),
                module_name    = module_name,
                module_package = module_package,
            )

            if module_filename is None:
                sys.exit(
                    "Error, implicit module '%s' expected by '%s' not found" % (
                        module_name,
                        module.getFullName()
                    )
                )
            elif Utils.isDir(module_filename):
                module_kind = "py"
            elif module_filename.endswith(".py"):
                module_kind = "py"
            elif module_filename.endswith(".so") or \
                 module_filename.endswith(".pyd"):
                module_kind = "shlib"
            else:
                assert False, module_filename

            # TODO: This should get back to plug-ins, they should be allowed to
            # preempt or override the decision.
            decision, reason = self.decideRecursion(
                module_filename = module_filename,
                module_name     = module_name,
                module_package  = module_package,
                module_kind     = module_kind
            )

            if decision:
                self.recurseTo(
                    module_package  = module_package,
                    module_filename = module_filename,
                    module_kind     = module_kind,
                    reason          = reason,
                    signal_change   = signal_change
                )

            full_name = module.getFullName()
            if full_name in post_modules:
                addUsedModule(post_modules[full_name])
예제 #30
0
def getSourceDirectoryPath(main_module):
    assert main_module.isCompiledPythonModule()

    return Options.getOutputPath(
        path = Utils.basename(
            getTreeFilenameWithSuffix(main_module, ".build")
        )
    )
예제 #31
0
def getStandaloneDirectoryPath(main_module):
    return Options.getOutputPath(
        path = Utils.basename(
            getTreeFilenameWithSuffix(main_module, ".dist")
        )
    )
예제 #32
0
 def getOutputFilename(self):
     if self.main_added:
         return Utils.dirname(self.getFilename())
     else:
         return PythonModule.getOutputFilename(self)
예제 #33
0
파일: Standalone.py 프로젝트: luzpaz/Nuitka
def detectEarlyImports():
    encoding_names = [
        filename[:-3] for _path, filename in listDir(
            os.path.dirname(sys.modules["encodings"].__file__))
        if filename.endswith(".py") if "__init__" not in filename
    ]

    if Utils.getOS() != "Windows":
        for encoding_name in ("mbcs", "cp65001", "oem"):
            if encoding_name in encoding_names:
                encoding_names.remove(encoding_name)

    import_code = ';'.join("import encodings.%s" % encoding_name
                           for encoding_name in encoding_names)

    import_code += ";import locale;"

    # For Python3 we patch inspect without knowing if it is used.
    if python_version >= 300:
        import_code += "import inspect;"

    result = _detectImports(command=import_code,
                            user_provided=False,
                            technical=True)

    if Options.shallFreezeAllStdlib():
        stdlib_modules = set()

        # Scan the standard library paths (multiple in case of virtualenv.
        for stdlib_dir in getStandardLibraryPaths():
            for module_name in scanStandardLibraryPath(stdlib_dir):
                stdlib_modules.add(module_name)

        # Put here ones that should be imported first.
        first_ones = ("Tkinter", )

        # We have to fight zombie modules in this, some things, e.g. Tkinter
        # on newer Python 2.7, comes back after failure without a second error
        # being raised, leading to other issues. So we fight it after each
        # module that was tried, and prevent re-try by adding a meta path
        # based loader that will never load it again, and remove it from the
        # "sys.modules" over and over, once it sneaks back. The root cause is
        # that extension modules sometimes only raise an error when first
        # imported, not the second time around.
        # Otherwise this just makes imports of everything so we can see where
        # it comes from and what it requires.

        import_code = """
imports = %r

failed = set()

class ImportBlocker(object):
    def find_module(self, fullname, path = None):
        if fullname in failed:
            return self

        return None

    def load_module(self, name):
        raise ImportError("%%s has failed before" %% name)

sys.meta_path.insert(0, ImportBlocker())

for imp in imports:
    try:
        __import__(imp)
    except (ImportError, SyntaxError):
        failed.add(imp)

    for fail in failed:
        if fail in sys.modules:
            del sys.modules[fail]
""" % sorted(stdlib_modules, key=lambda name: (name not in first_ones, name))

        early_names = [module.getFullName() for module in result]

        result += [
            module for module in _detectImports(
                command=import_code, user_provided=False, technical=False)
            if module.getFullName() not in early_names
        ]

    debug("Finished detecting early imports.")

    return result
예제 #34
0
Copyright (C) 2016 Kay Hayen."""

import logging
import sys
from optparse import SUPPRESS_HELP, OptionGroup, OptionParser

from nuitka.PythonVersions import (
    getSupportedPythonVersions,
    getSupportedPythonVersionStr,
    python_version_str
)
from nuitka.utils import Utils

# Indicator if we were called as "nuitka-run" in which case we assume some
# other defaults and work a bit different with parameters.
is_nuitka_run = Utils.basename(sys.argv[0]).lower().startswith("nuitka-run")

def getVersion():
    return version_string.split()[1][1:]

def getYear():
    return int(version_string.split()[4])

if not is_nuitka_run:
    usage = "usage: %prog [--module] [--execute] [options] main_module.py"
else:
    usage = "usage: %prog [options] main_module.py"

parser = OptionParser(
    usage   = usage,
    version = getVersion()
예제 #35
0
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
    # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915

    assert package is None or type(package) is str
    assert filename is not None

    if is_main and Utils.isDir(filename):
        source_filename = Utils.joinpath(filename, "__main__.py")

        if not Utils.isFile(source_filename):
            sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
                             (Utils.basename(sys.argv[0]), filename))
            sys.exit(2)

        filename = source_filename

        main_added = True
    else:
        main_added = False

    if Utils.isFile(filename):
        source_filename = filename

        source_ref = SourceCodeReferences.fromFilename(filename=filename, )

        if is_main:
            module_name = "__main__"
        else:
            module_name = Utils.basename(filename)

            if module_name.endswith(".py"):
                module_name = module_name[:-3]

            if is_shlib:
                module_name = module_name.split('.')[0]

            if '.' in module_name:
                sys.stderr.write(
                    "Error, '%s' is not a proper python module name.\n" %
                    (module_name))

                sys.exit(2)

        if is_shlib:
            result = PythonShlibModule(name=module_name,
                                       package_name=package,
                                       source_ref=source_ref)
        elif is_main:
            result = PythonMainModule(main_added=main_added,
                                      mode=Plugins.decideCompilation(
                                          "__main__", source_ref),
                                      source_ref=source_ref)
        else:
            if package is not None:
                full_name = package + '.' + module_name
            else:
                full_name = module_name

            result = CompiledPythonModule(name=module_name,
                                          package_name=package,
                                          mode=Plugins.decideCompilation(
                                              full_name, source_ref),
                                          source_ref=source_ref)

    elif Importing.isPackageDir(filename):
        if is_top:
            package_name = Utils.splitpath(filename)[-1]
        else:
            package_name = Utils.basename(filename)

        source_filename = Utils.joinpath(filename, "__init__.py")

        if not Utils.isFile(source_filename):
            source_ref, result = createNamespacePackage(
                package_name=package_name, module_relpath=filename)
            source_filename = None
        else:
            source_ref = SourceCodeReferences.fromFilename(
                filename=Utils.abspath(source_filename), )

            if package is not None:
                full_name = package + '.' + package_name
            else:
                full_name = package_name

            result = CompiledPythonPackage(name=package_name,
                                           package_name=package,
                                           mode=Plugins.decideCompilation(
                                               full_name, source_ref),
                                           source_ref=source_ref)
    else:
        sys.stderr.write("%s: can't open file '%s'.\n" %
                         (Utils.basename(sys.argv[0]), filename))
        sys.exit(2)

    if not Options.shallHaveStatementLines():
        source_ref = source_ref.atInternal()

    return result, source_ref, source_filename
예제 #36
0
def _checkPluginPath(plugin_filename, module_package):
    # Many branches, for the decision is very complex, pylint: disable=R0912

    debug("Checking detail plug-in path '%s' '%s':", plugin_filename,
          module_package)

    module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
        plugin_filename)

    if module_kind is not None:
        decision, _reason = decideRecursion(module_filename=plugin_filename,
                                            module_name=module_name,
                                            module_package=module_package,
                                            module_kind=module_kind)

        if decision:
            module_relpath = Utils.relpath(plugin_filename)

            module, is_added = recurseTo(module_filename=plugin_filename,
                                         module_relpath=module_relpath,
                                         module_package=module_package,
                                         module_kind="py",
                                         reason="Lives in plug-in directory.")

            if module:
                if not is_added:
                    warning(
                        "Recursed to %s '%s' at '%s' twice.", "package"
                        if module.isCompiledPythonPackage() else "module",
                        module.getName(), plugin_filename)

                    if not isSameModulePath(module.getFilename(),
                                            plugin_filename):
                        warning("Duplicate ignored '%s'.", plugin_filename)

                        return

                debug("Recursed to %s %s %s", module.getName(),
                      module.getPackage(), module)

                ImportCache.addImportedModule(module)

                if module.isCompiledPythonPackage():
                    package_filename = module.getFilename()

                    if Utils.isDir(package_filename):
                        # Must be a namespace package.
                        assert python_version >= 330

                        package_dir = package_filename

                        # Only include it, if it contains actual modules, which will
                        # recurse to this one and find it again.
                    else:
                        package_dir = Utils.dirname(package_filename)

                        # Real packages will always be included.
                        ModuleRegistry.addRootModule(module)

                    debug("Package directory %s", package_dir)

                    for sub_path, sub_filename in Utils.listDir(package_dir):
                        if sub_filename in ("__init__.py", "__pycache__"):
                            continue

                        assert sub_path != plugin_filename

                        if Importing.isPackageDir(sub_path) or \
                           sub_path.endswith(".py"):
                            _checkPluginPath(sub_path, module.getFullName())

                elif module.isCompiledPythonModule():
                    ModuleRegistry.addRootModule(module)

            else:
                warning("Failed to include module from '%s'.", plugin_filename)
예제 #37
0
 def isRelevant():
     return Utils.getOS() == "Windows" and not Options.shallMakeModule()
예제 #38
0
def parseArgs():
    # singleton with many cases, pylint: disable=global-statement,too-many-branches
    global is_nuitka_run, options, positional_args, extra_args

    is_nuitka_run, options, positional_args, extra_args = parseOptions()

    if options.verbose:
        logging.getLogger().setLevel(logging.DEBUG)
    else:
        logging.getLogger().setLevel(logging.INFO)

    # Standalone mode implies an executable, not importing "site" module, which is
    # only for this machine, recursing to all modules, and even including the
    # standard library.
    if options.is_standalone:
        if not options.executable:
            sys.exit(
                """\
Error, conflicting options, cannot make standalone module, only executable."""
            )

        options.recurse_all = True

        if Utils.getOS() == "NetBSD":
            logging.warning(
                "Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported."
            )

    for any_case_module in getShallFollowModules():
        if any_case_module.startswith("."):
            bad = True
        else:
            for char in "/\\:":
                if char in any_case_module:
                    bad = True
                    break
            else:
                bad = False

        if bad:
            sys.exit(
                """\
Error, '--follow-import-to' takes only module names, not directory path '%s'."""
                % any_case_module
            )

    for no_case_module in getShallFollowInNoCase():
        if no_case_module.startswith("."):
            bad = True
        else:
            for char in "/\\:":
                if char in no_case_module:
                    bad = True
                    break
            else:
                bad = False

        if bad:
            sys.exit(
                """\
Error, '--nofollow-import-to' takes only module names, not directory path '%s'."""
                % no_case_module
            )

    scons_python = getPythonPathForScons()

    if scons_python is not None and not os.path.exists(scons_python):
        sys.exit("Error, no such Python binary '%s'." % scons_python)

    if options.output_filename is not None and (
        isStandaloneMode() or shallMakeModule()
    ):
        sys.exit(
            """\
Error, can only specify output filename for acceleration mode, not for module
mode where filenames are mandatory, and not for standalone where there is a
sane default used inside the dist folder."""
        )

    for icon_path in [getIconPath()]:
        if icon_path is not None and not os.path.exists(icon_path):
            sys.exit(
                """\
Error, icon path "%s" does not exist."""
                % icon_path
            )
예제 #39
0
파일: Variables.py 프로젝트: wsniper/Nuitka
    def getCodeName(self):
        var_name = self.variable_name
        var_name = var_name.replace('.', '$')
        var_name = Utils.encodeNonAscii(var_name)

        return var_name
예제 #40
0
파일: Options.py 프로젝트: szabadkai/Nuitka
def parseArgs():
    # singleton with many cases, pylint: disable=global-statement,too-many-branches,too-many-statements
    global options, positional_args, extra_args

    # First, isolate the first non-option arguments.
    if is_nuitka_run:
        count = 0

        for count, arg in enumerate(sys.argv):
            if count == 0:
                continue

            if arg[0] != '-':
                break

            # Treat "--" as a terminator.
            if arg == "--":
                count += 1
                break

        if count > 0:
            extra_args = sys.argv[count + 1:]
            sys.argv = sys.argv[0:count + 1]

    options, positional_args = parser.parse_args()

    if shallListPlugins():
        from nuitka.plugins.Plugins import listPlugins
        listPlugins()

    if not positional_args:
        parser.print_help()

        sys.exit("""
    Error, need positional argument with python module or main program.""")

    if not options.immediate_execution and len(positional_args) > 1:
        parser.print_help()

        sys.exit("""
    Error, need only one positional argument unless "--run" is specified to
    pass them to the compiled program execution.""")

    if options.verbose:
        logging.getLogger().setLevel(logging.DEBUG)
    else:
        logging.getLogger().setLevel(logging.INFO)

    # Standalone mode implies an executable, not importing "site" module, which is
    # only for this machine, recursing to all modules, and even including the
    # standard library.
    if options.is_standalone:
        if not options.executable:
            sys.exit("""\
Error, conflicting options, cannot make standalone module, only executable.""")

        options.recurse_all = True

        if Utils.getOS() == "NetBSD":
            logging.warning(
                "Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported."
            )

    for any_case_module in getShallFollowModules():
        if any_case_module.startswith('.'):
            bad = True
        else:
            for char in "/\\:":
                if char in any_case_module:
                    bad = True
                    break
            else:
                bad = False

        if bad:
            sys.exit(
                """\
Error, '--recurse-to' takes only module names, not directory path '%s'.""" % \
                any_case_module
            )

    for no_case_module in getShallFollowInNoCase():
        if no_case_module.startswith('.'):
            bad = True
        else:
            for char in "/\\:":
                if char in no_case_module:
                    bad = True
                    break
            else:
                bad = False

        if bad:
            sys.exit(
                """\
Error, '--recurse-not-to' takes only module names, not directory path '%s'.""" % \
                no_case_module
            )

    scons_python = getPythonPathForScons()

    if scons_python is not None and not os.path.exists(scons_python):
        sys.exit("Error, no such Python2 binary '%s'." % scons_python)
예제 #41
0
파일: Options.py 프로젝트: szabadkai/Nuitka
    # Defaults to what you run Nuitka with (currently %s)""" % (
    #        getSupportedPythonVersionStr(),
    #        python_version_str
    #     )
)

if os.name == "nt":
    parser.add_option("--python-arch",
                      action="store",
                      dest="python_arch",
                      choices=("x86", "x86_64"),
                      default=None,
                      help="""\
Architecture of Python to use. One of "x86" or "x86_64".
Defaults to what you run Nuitka with (currently "%s").""" %
                      (Utils.getArchitecture()))

parser.add_option("--python-debug",
                  "--python-dbg",
                  action="store_true",
                  dest="python_debug",
                  default=None,
                  help="""\
Use debug version or not. Default uses what you are using to run Nuitka, most
likely a non-debug version.""")

parser.add_option("--python-flag",
                  action="append",
                  dest="python_flags",
                  default=[],
                  help="""\
예제 #42
0
def main():
    """ Main program flow of Nuitka

        At this point, options will be parsed already, Nuitka will be executing
        in the desired version of Python with desired flags, and we just get
        to execute the task assigned.

        We might be asked to only re-compile generated C++, dump only an XML
        representation of the internal node tree after optimization, etc.
    """

    # Main has to fullfil many options, leading to many branches and statements
    # to deal with them.  pylint: disable=R0912

    positional_args = Options.getPositionalArgs()
    assert len(positional_args) > 0

    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir = Utils.dirname(Utils.abspath(filename))
    )

    # Detect to be frozen modules if any, so we can consider to not recurse
    # to them.
    if Options.isStandaloneMode():
        for module in detectEarlyImports():
            ModuleRegistry.addUncompiledModule(module)

            if module.getName() == "site":
                origin_prefix_filename = Utils.joinpath(
                    Utils.dirname(module.getCompileTimeFilename()),
                    "orig-prefix.txt"
                )

                if Utils.isFile(origin_prefix_filename):
                    data_files.append(
                        (filename, "orig-prefix.txt")
                    )

    # Turn that source code into a node tree structure.
    try:
        main_module = createNodeTree(
            filename = filename
        )
    except (SyntaxError, IndentationError) as e:
        # Syntax or indentation errors, output them to the user and abort.
        sys.exit(
            SyntaxErrors.formatOutput(e)
        )

    if Options.shallDumpBuiltTreeXML():
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    elif Options.shallDisplayBuiltTree():
        displayTree(main_module)
    else:
        result, options = compileTree(
            main_module = main_module
        )

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCppCall():
            sys.exit(0)

        # Remove the source directory (now build directory too) if asked to.
        if Options.isRemoveBuildDir():
            shutil.rmtree(
                getSourceDirectoryPath(main_module)
            )

        if Options.isStandaloneMode():
            binary_filename = options["result_name"] + ".exe"

            standalone_entry_points.insert(
                0,
                (binary_filename, None)
            )

            dist_dir = getStandaloneDirectoryPath(main_module)

            for module in ModuleRegistry.getDoneUserModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module)
                )

            for module in ModuleRegistry.getUncompiledModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module)
                )

            copyUsedDLLs(
                dist_dir                = dist_dir,
                standalone_entry_points = standalone_entry_points
            )


            for module in ModuleRegistry.getDoneModules():
                data_files.extend(
                    Plugins.considerDataFiles(module)
                )

            for source_filename, target_filename in data_files:
                target_filename = Utils.joinpath(
                    getStandaloneDirectoryPath(main_module),
                    target_filename
                )

                Utils.makePath(Utils.dirname(target_filename))

                shutil.copy2(
                    source_filename,
                    target_filename
                )

        # Modules should not be executable, but Scons creates them like it, fix
        # it up here.
        if Utils.getOS() != "Windows" and Options.shallMakeModule():
            subprocess.call(
                (
                    "chmod",
                    "-x",
                    getResultFullpath(main_module)
                )
            )

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            if Options.shallMakeModule():
                executeModule(
                    tree       = main_module,
                    clean_path = Options.shallClearPythonPathEnvironment()
                )
            else:
                executeMain(
                    binary_filename = getResultFullpath(main_module),
                    clean_path      = Options.shallClearPythonPathEnvironment()
                )
예제 #43
0
def createNodeTree(filename):
    """ Create a node tree.

    Turn that source code into a node tree structure. If recursion into
    imported modules is available, more trees will be available during
    optimization, or immediately through recursed directory paths.

    """

    # First, build the raw node tree from the source code.
    main_module = Building.buildModuleTree(
        filename = filename,
        package  = None,
        is_top   = True,
        is_main  = not Options.shallMakeModule()
    )
    ModuleRegistry.addRootModule(main_module)

    # First remove old object files and old generated files, old binary or
    # module, and standalone mode program directory if any, they can only do
    # harm.
    source_dir = getSourceDirectoryPath(main_module)

    if not Options.shallOnlyExecCppCall():
        cleanSourceDirectory(source_dir)

    # Prepare the ".dist" directory, throwing away what was there before.
    if Options.isStandaloneMode():
        standalone_dir = getStandaloneDirectoryPath(main_module)
        shutil.rmtree(standalone_dir, ignore_errors = True)
        Utils.makePath(standalone_dir)

    Utils.deleteFile(
        path       = getResultFullpath(main_module),
        must_exist = False
    )

    # Second, do it for the directories given.
    for plugin_filename in Options.getShallFollowExtra():
        Recursion.checkPluginPath(
            plugin_filename = plugin_filename,
            module_package  = None
        )

    for pattern in Options.getShallFollowExtraFilePatterns():
        Recursion.checkPluginFilenamePattern(
            pattern = pattern
        )


    # Then optimize the tree and potentially recursed modules.
    Optimization.optimize()

    if Options.isExperimental():
        for module in ModuleRegistry.getRootModules():
            if module.isMainModule():
                return module

        assert False
    else:
        # Main module might change behind our back, look it up again.
        return main_module
예제 #44
0
def scanStandardLibraryPath(stdlib_dir):
    # There is a lot of filtering here, done in branches, so there # is many of
    # them, but that's acceptable, pylint: disable=too-many-branches,too-many-statements

    for root, dirs, filenames in os.walk(stdlib_dir):
        import_path = root[len(stdlib_dir):].strip("/\\")
        import_path = import_path.replace("\\", ".").replace("/", ".")

        if import_path == "":
            if "site-packages" in dirs:
                dirs.remove("site-packages")
            if "dist-packages" in dirs:
                dirs.remove("dist-packages")
            if "test" in dirs:
                dirs.remove("test")
            if "idlelib" in dirs:
                dirs.remove("idlelib")
            if "turtledemo" in dirs:
                dirs.remove("turtledemo")

            if "ensurepip" in filenames:
                filenames.remove("ensurepip")
            if "ensurepip" in dirs:
                dirs.remove("ensurepip")

            # Ignore "lib-dynload" and "lib-tk" and alike.
            dirs[:] = [
                dirname for dirname in dirs if not dirname.startswith("lib-")
                if dirname != "Tools" if not dirname.startswith("plat-")
            ]

        if import_path in (
                "tkinter",
                "importlib",
                "ctypes",
                "unittest",
                "sqlite3",
                "distutils",
                "email",
                "bsddb",
        ):
            if "test" in dirs:
                dirs.remove("test")

        if import_path == "distutils.command":
            # Misbehaving and crashing while importing the world.
            if "bdist_conda.py" in filenames:
                filenames.remove("bdist_conda.py")

        if import_path in ("lib2to3", "json", "distutils"):
            if "tests" in dirs:
                dirs.remove("tests")

        if import_path == "asyncio":
            if "test_utils.py" in filenames:
                filenames.remove("test_utils.py")

        if python_version >= 0x340 and Utils.isWin32Windows():
            if import_path == "multiprocessing":
                filenames.remove("popen_fork.py")
                filenames.remove("popen_forkserver.py")
                filenames.remove("popen_spawn_posix.py")

        if python_version >= 0x300 and Utils.isPosixWindows():
            if import_path == "curses":
                filenames.remove("has_key.py")

        if Utils.getOS() == "NetBSD":
            if import_path == "xml.sax":
                filenames.remove("expatreader.py")

        for filename in filenames:
            if filename.endswith(
                    ".py") and filename not in _excluded_stdlib_modules:
                module_name = filename[:-3]
                if import_path == "":
                    yield module_name
                else:
                    yield import_path + "." + module_name

        if python_version >= 0x300:
            if "__pycache__" in dirs:
                dirs.remove("__pycache__")

        for dirname in dirs:
            if import_path == "":
                yield dirname
            else:
                yield import_path + "." + dirname
예제 #45
0
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points):
    # This is terribly complex, because we check the list of used DLLs
    # trying to avoid duplicates, and detecting errors with them not
    # being binary identical, so we can report them. And then of course
    # we also need to handle OS specifics.
    # pylint: disable=too-many-branches,too-many-locals,too-many-statements

    used_dlls = detectUsedDLLs(
        source_dir=source_dir,
        standalone_entry_points=standalone_entry_points,
        use_cache=not Options.shallNotUseDependsExeCachedResults()
        and not Options.getWindowsDependencyTool() == "depends.exe",
        update_cache=not Options.shallNotStoreDependsExeCachedResults()
        and not Options.getWindowsDependencyTool() == "depends.exe",
    )

    removed_dlls = set()
    warned_about = set()

    # Fist make checks and remove some.
    for dll_filename1, sources1 in tuple(iterItems(used_dlls)):
        if dll_filename1 in removed_dlls:
            continue

        for dll_filename2, sources2 in tuple(iterItems(used_dlls)):
            if dll_filename1 == dll_filename2:
                continue

            if dll_filename2 in removed_dlls:
                continue

            # Colliding basenames are an issue to us.
            if os.path.basename(dll_filename1) != os.path.basename(
                    dll_filename2):
                continue

            # May already have been removed earlier
            if dll_filename1 not in used_dlls:
                continue

            if dll_filename2 not in used_dlls:
                continue

            dll_name = os.path.basename(dll_filename1)

            if Options.isShowInclusion():
                inclusion_logger.info(
                    """Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2))

            # Check that if a DLL has the same name, if it's identical, then it's easy.
            if haveSameFileContents(dll_filename1, dll_filename2):
                del used_dlls[dll_filename2]
                removed_dlls.add(dll_filename2)

                continue

            # For Win32 we can check out file versions.
            if Utils.isWin32Windows():
                dll_version1 = getWindowsDLLVersion(dll_filename1)
                dll_version2 = getWindowsDLLVersion(dll_filename2)

                if dll_version2 < dll_version1:
                    del used_dlls[dll_filename2]
                    removed_dlls.add(dll_filename2)

                    solved = True
                elif dll_version1 < dll_version2:
                    del used_dlls[dll_filename1]
                    removed_dlls.add(dll_filename1)

                    solved = True
                else:
                    solved = False

                if solved:
                    if dll_name not in warned_about and dll_name not in ms_runtime_dlls:
                        warned_about.add(dll_name)

                        inclusion_logger.warning(
                            "Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best."
                            % dll_name)

                    continue

            # So we have conflicting DLLs, in which case we do report the fact.
            inclusion_logger.warning("""\
Ignoring non-identical DLLs for '%s'.
%s used by:
   %s
different from
%s used by
   %s""" % (
                dll_name,
                dll_filename1,
                "\n   ".join(sources1),
                dll_filename2,
                "\n   ".join(sources2),
            ))

            del used_dlls[dll_filename2]
            removed_dlls.add(dll_filename2)

    dll_map = []

    for dll_filename, sources in iterItems(used_dlls):
        dll_name = os.path.basename(dll_filename)

        target_path = os.path.join(dist_dir, dll_name)

        shutil.copyfile(dll_filename, target_path)

        dll_map.append((dll_filename, dll_name))

        if Options.isShowInclusion():
            inclusion_logger.info(
                "Included used shared library '%s' (used by %s)." %
                (dll_filename, ", ".join(sources)))

    if Utils.getOS() == "Darwin":
        # For macOS, the binary and the DLLs needs to be changed to reflect
        # the relative DLL location in the ".dist" folder.
        for standalone_entry_point in standalone_entry_points:
            fixupBinaryDLLPathsMacOS(
                binary_filename=standalone_entry_point.dest_path,
                dll_map=dll_map,
                original_location=standalone_entry_point.source_path,
            )

        for original_path, dll_filename in dll_map:
            fixupBinaryDLLPathsMacOS(
                binary_filename=os.path.join(dist_dir, dll_filename),
                dll_map=dll_map,
                original_location=original_path,
            )

        # Remove code signature from CPython installed library
        candidate = os.path.join(
            dist_dir,
            "Python",
        )

        if os.path.exists(candidate):
            removeMacOSCodeSignature(candidate)

    # Remove rpath settings.
    if Utils.getOS() in ("Linux", "Darwin"):
        # For Linux, the "rpath" of libraries may be an issue and must be
        # removed.
        if Utils.getOS() == "Darwin":
            start = 0
        else:
            start = 1

        for standalone_entry_point in standalone_entry_points[start:]:
            removeSharedLibraryRPATH(standalone_entry_point.dest_path)

        for _original_path, dll_filename in dll_map:
            removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))

    if Utils.isWin32Windows():
        if python_version < 0x300:
            # For Win32, we might have to remove SXS paths
            for standalone_entry_point in standalone_entry_points[1:]:
                removeSxsFromDLL(standalone_entry_point.dest_path)

            for _original_path, dll_filename in dll_map:
                removeSxsFromDLL(os.path.join(dist_dir, dll_filename))
예제 #46
0
# Indicator if we were called as "nuitka-run" in which case we assume some
# other defaults and work a bit different with parameters.
is_nuitka_run = os.path.basename(sys.argv[0]).lower().endswith("-run")

if not is_nuitka_run:
    usage = "usage: %prog [--module] [--run] [options] main_module.py"
else:
    usage = "usage: %prog [options] main_module.py"

parser = OptionParser(
    usage=usage,
    version="\n".join((
        getNuitkaVersion(),
        "Python: " + sys.version.split("\n")[0],
        "Executable: " + sys.executable,
        "OS: " + Utils.getOS(),
        "Arch: " + Utils.getArchitecture(),
    )),
)

parser.add_option(
    "--module",
    action="store_false",
    dest="executable",
    default=True,
    help="""\
Create an extension module executable instead of a program. Defaults to off.""",
)

parser.add_option(
    "--standalone",
예제 #47
0
def main():
    """ Main program flow of Nuitka

        At this point, options will be parsed already, Nuitka will be executing
        in the desired version of Python with desired flags, and we just get
        to execute the task assigned.

        We might be asked to only re-compile generated C, dump only an XML
        representation of the internal node tree after optimization, etc.
    """

    # Main has to fulfill many options, leading to many branches and statements
    # to deal with them.  pylint: disable=too-many-branches
    filename = Options.getPositionalArgs()[0]

    # Inform the importing layer about the main script directory, so it can use
    # it when attempting to follow imports.
    Importing.setMainScriptDirectory(
        main_dir=os.path.dirname(os.path.abspath(filename)))

    # Detect to be frozen modules if any, so we can consider to not recurse
    # to them.
    if Options.isStandaloneMode():
        for module in detectEarlyImports():
            ModuleRegistry.addUncompiledModule(module)

            if module.getName() == "site":
                origin_prefix_filename = os.path.join(
                    os.path.dirname(module.getCompileTimeFilename()),
                    "orig-prefix.txt")

                if os.path.isfile(origin_prefix_filename):
                    data_files.append((filename, "orig-prefix.txt"))

    # Turn that source code into a node tree structure.
    try:
        main_module = createNodeTree(filename=filename)
    except (SyntaxError, IndentationError) as e:
        handleSyntaxError(e)

    if Options.shallDumpBuiltTreeXML():
        # XML output only.
        for module in ModuleRegistry.getDoneModules():
            dumpTreeXML(module)
    else:
        # Make the actual compilation.
        result, options = compileTree(main_module=main_module)

        # Exit if compilation failed.
        if not result:
            sys.exit(1)

        if Options.shallNotDoExecCCompilerCall():
            if Options.isShowMemory():
                MemoryUsage.showMemoryTrace()

            sys.exit(0)

        if Options.isStandaloneMode():
            binary_filename = options["result_exe"]

            standalone_entry_points.insert(
                0, (binary_filename, binary_filename, None))

            dist_dir = getStandaloneDirectoryPath(main_module)

            for module in ModuleRegistry.getDoneUserModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module))

            for module in ModuleRegistry.getUncompiledModules():
                standalone_entry_points.extend(
                    Plugins.considerExtraDlls(dist_dir, module))

            copyUsedDLLs(
                source_dir=getSourceDirectoryPath(main_module),
                dist_dir=dist_dir,
                standalone_entry_points=standalone_entry_points,
            )

            for module in ModuleRegistry.getDoneModules():
                data_files.extend(Plugins.considerDataFiles(module))

            for source_filename, target_filename in data_files:
                target_filename = os.path.join(
                    getStandaloneDirectoryPath(main_module), target_filename)

                makePath(os.path.dirname(target_filename))

                shutil.copy2(source_filename, target_filename)

        # Remove the source directory (now build directory too) if asked to.
        if Options.isRemoveBuildDir():
            removeDirectory(path=getSourceDirectoryPath(main_module),
                            ignore_errors=False)

        # Modules should not be executable, but Scons creates them like it, fix
        # it up here. TODO: Move inside scons file and avoid subprocess call.
        if Utils.getOS() != "Windows" and Options.shallMakeModule():
            subprocess.call(("chmod", "-x", getResultFullpath(main_module)))

        if Options.shallMakeModule() and Options.shallCreatePyiFile():
            pyi_filename = getResultBasepath(main_module) + ".pyi"

            with open(pyi_filename, "w") as pyi_file:
                pyi_file.write(
                    """\
# This file was generated by Nuitka and describes the types of the
# created shared library.

# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.

# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.

%(imports)s

# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...

""" % {
                        "imports":
                        "\n".join("import %s" % module_name
                                  for module_name in getImportedNames())
                    })

        # Execute the module immediately if option was given.
        if Options.shallExecuteImmediately():
            if Options.shallMakeModule():
                executeModule(
                    tree=main_module,
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
            else:
                executeMain(
                    binary_filename=getResultFullpath(main_module),
                    clean_path=Options.shallClearPythonPathEnvironment(),
                )
예제 #48
0
파일: Standalone.py 프로젝트: luzpaz/Nuitka
                                technical=technical)
        elif kind == "shlib":
            _detectedShlibFile(filename=filename, module_name=module_name)
        else:
            assert False, kind

    return result


# Some modules we want to blacklist.
ignore_modules = [
    "__main__.py",
    "__init__.py",
    "antigravity.py",
]
if Utils.getOS() != "Windows":
    ignore_modules.append("wintypes.py")
    ignore_modules.append("cp65001.py")


def scanStandardLibraryPath(stdlib_dir):
    # There is a lot of black-listing here, done in branches, so there
    # is many of them, but that's acceptable, pylint: disable=too-many-branches

    for root, dirs, filenames in os.walk(stdlib_dir):
        import_path = root[len(stdlib_dir):].strip("/\\")
        import_path = import_path.replace('\\', '.').replace('/', '.')

        if import_path == "":
            if "site-packages" in dirs:
                dirs.remove("site-packages")
예제 #49
0
def runScons(main_module, quiet):
    # Scons gets transported many details, that we express as variables, and
    # have checks for them, leading to many branches and statements,
    # pylint: disable=too-many-branches,too-many-statements

    options = {
        "name":
        os.path.basename(getTreeFilenameWithSuffix(main_module, "")),
        "result_name":
        getResultBasepath(main_module),
        "source_dir":
        getSourceDirectoryPath(main_module),
        "debug_mode":
        _asBoolStr(Options.isDebug()),
        "python_debug":
        _asBoolStr(Options.isPythonDebug()),
        "unstripped_mode":
        _asBoolStr(Options.isUnstripped()),
        "module_mode":
        _asBoolStr(Options.shallMakeModule()),
        "full_compat":
        _asBoolStr(Options.isFullCompat()),
        "experimental":
        ",".join(Options.getExperimentalIndications()),
        "trace_mode":
        _asBoolStr(Options.shallTraceExecution()),
        "python_version":
        python_version_str,
        "target_arch":
        Utils.getArchitecture(),
        "python_prefix":
        sys.prefix,
        "nuitka_src":
        SconsInterface.getSconsDataPath(),
        "nuitka_cache":
        getCacheDir(),
        "module_count":
        "%d" % (1 + len(ModuleRegistry.getDoneUserModules()) +
                len(ModuleRegistry.getUncompiledNonTechnicalModules())),
    }

    if not Options.shallMakeModule():
        options["result_exe"] = getResultFullpath(main_module)

    # Ask Scons to cache on Windows, except where the directory is thrown
    # away. On non-Windows you can should use ccache instead.
    if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows":
        options["cache_mode"] = "true"

    if Options.isLto():
        options["lto_mode"] = "true"

    # For AnaConda default to trying static lib python library, which
    # normally is just not available or if it is even unusable.
    if "Anaconda" in sys.version:
        options["static_libpython"] = "true"

    if Options.shallDisableConsoleWindow():
        options["win_disable_console"] = "true"

    if Options.isStandaloneMode():
        options["standalone_mode"] = "true"

    if (not Options.isStandaloneMode() and not Options.shallMakeModule()
            and isUninstalledPython()):
        options["uninstalled_python"] = "true"

    if ModuleRegistry.getUncompiledTechnicalModules():
        options["frozen_modules"] = str(
            len(ModuleRegistry.getUncompiledTechnicalModules()))

    if Options.isShowScons():
        options["show_scons"] = "true"

    if Options.isMingw64():
        options["mingw_mode"] = "true"

    if Options.getMsvcVersion():
        msvc_version = Options.getMsvcVersion()

        msvc_version = msvc_version.replace("exp", "Exp")
        if "." not in msvc_version:
            msvc_version += ".0"

        options["msvc_version"] = msvc_version

    if Options.isClang():
        options["clang_mode"] = "true"

    if Options.getIconPath():
        options["icon_path"] = Options.getIconPath()

    if Options.isProfile():
        options["profile_mode"] = "true"

    if "no_warnings" in getPythonFlags():
        options["no_python_warnings"] = "true"

    if "no_asserts" in getPythonFlags():
        options["python_sysflag_optimize"] = "true"

    if python_version < 300 and sys.flags.py3k_warning:
        options["python_sysflag_py3k_warning"] = "true"

    if python_version < 300 and (sys.flags.division_warning
                                 or sys.flags.py3k_warning):
        options["python_sysflag_division_warning"] = "true"

    if sys.flags.bytes_warning:
        options["python_sysflag_bytes_warning"] = "true"

    if int(
            os.environ.get("NUITKA_SITE_FLAG", "no_site"
                           in Options.getPythonFlags())):
        options["python_sysflag_no_site"] = "true"

    if "trace_imports" in Options.getPythonFlags():
        options["python_sysflag_verbose"] = "true"

    if python_version < 300 and sys.flags.unicode:
        options["python_sysflag_unicode"] = "true"

    if python_version >= 370 and sys.flags.utf8_mode:
        options["python_sysflag_utf8"] = "true"

    abiflags = getPythonABI()
    if abiflags:
        options["abiflags"] = abiflags

    return SconsInterface.runScons(options, quiet), options
예제 #50
0
 def getCompileTimeFilename(self):
     return Utils.abspath(self.getSourceReference().getFilename())
예제 #51
0
파일: Standalone.py 프로젝트: luzpaz/Nuitka
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points):
    # This is terribly complex, because we check the list of used DLLs
    # trying to avoid duplicates, and detecting errors with them not
    # being binary identical, so we can report them. And then of course
    # we also need to handle OS specifics.
    # pylint: disable=too-many-branches,too-many-locals

    used_dlls = detectUsedDLLs(source_dir, standalone_entry_points)

    removed_dlls = set()

    # Fist make checks and remove some.
    for dll_filename1, sources1 in tuple(iterItems(used_dlls)):
        if dll_filename1 in removed_dlls:
            continue

        for dll_filename2, sources2 in tuple(iterItems(used_dlls)):
            if dll_filename1 == dll_filename2:
                continue

            if dll_filename2 in removed_dlls:
                continue

            # Colliding basenames are an issue to us.
            if os.path.basename(dll_filename1) != \
               os.path.basename(dll_filename2):
                continue

            # May already have been removed earlier
            if dll_filename1 not in used_dlls:
                continue

            if dll_filename2 not in used_dlls:
                continue

            dll_name = os.path.basename(dll_filename1)

            if Options.isShowInclusion():
                info("""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (
                    dll_name,
                    dll_filename1,
                    dll_filename2,
                ))

            # Check that if a DLL has the same name, if it's identical,
            # happens at least for OSC and Fedora 20.
            import filecmp
            if filecmp.cmp(dll_filename1, dll_filename2):
                del used_dlls[dll_filename2]
                removed_dlls.add(dll_filename2)

                continue

            # So we have conflicting DLLs, in which case we do not proceed.
            sys.exit("""Error, conflicting DLLs for '%s'.
%s used by:
   %s
different from
%s used by
   %s""" % (dll_name, dll_filename1, "\n   ".join(sources1), dll_filename2,
            "\n   ".join(sources2)))

    dll_map = []

    for dll_filename, sources in iterItems(used_dlls):
        dll_name = os.path.basename(dll_filename)

        target_path = os.path.join(dist_dir, dll_name)

        shutil.copyfile(dll_filename, target_path)

        dll_map.append((dll_filename, dll_name))

        if Options.isShowInclusion():
            info("Included used shared library '%s' (used by %s)." %
                 (dll_filename, ", ".join(sources)))

    if Utils.getOS() == "Darwin":
        # For MacOS, the binary and the DLLs needs to be changed to reflect
        # the relative DLL location in the ".dist" folder.
        for standalone_entry_point in standalone_entry_points:
            fixupBinaryDLLPaths(
                binary_filename=standalone_entry_point[1],
                is_exe=standalone_entry_point is standalone_entry_points[0],
                dll_map=dll_map)

        for _original_path, dll_filename in dll_map:
            fixupBinaryDLLPaths(binary_filename=os.path.join(
                dist_dir, dll_filename),
                                is_exe=False,
                                dll_map=dll_map)

    if Utils.getOS() == "Linux":
        # For Linux, the "rpath" of libraries may be an issue and must be
        # removed.
        for standalone_entry_point in standalone_entry_points[1:]:
            removeSharedLibraryRPATH(standalone_entry_point[1])

        for _original_path, dll_filename in dll_map:
            removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))
예제 #52
0
def main():
    # PyLint for Python3 thinks we import from ourselves if we really
    # import from package, pylint:disable=I0021,no-name-in-module

    # Also high complexity.
    # pylint: disable=too-many-branches,too-many-locals,too-many-statements

    if "NUITKA_BINARY_NAME" in os.environ:
        sys.argv[0] = os.environ["NUITKA_BINARY_NAME"]

    if "NUITKA_PYTHONPATH" in os.environ:
        # Restore the PYTHONPATH gained from the site module, that we chose not
        # to have imported. pylint: disable=eval-used
        sys.path = eval(os.environ["NUITKA_PYTHONPATH"])
        del os.environ["NUITKA_PYTHONPATH"]
    else:
        # Remove path element added for being called via "__main__.py", this can
        # only lead to trouble, having e.g. a "distutils" in sys.path that comes
        # from "nuitka.distutils".
        sys.path = [
            path_element
            for path_element in sys.path
            if os.path.dirname(os.path.abspath(__file__)) != path_element
        ]

    # For re-execution, we might not have done this.
    from nuitka import Options                  # isort:skip
    Options.parseArgs()

    from nuitka.utils import Utils, Execution   # isort:skip

    import logging # isort:skip
    logging.basicConfig(format = "Nuitka:%(levelname)s:%(message)s")

    # We don't care, and these are triggered by run time calculations of "range" and
    # others, while on python2.7 they are disabled by default.

    warnings.simplefilter("ignore", DeprecationWarning)

    # We will run with the Python configuration as specified by the user, if it does
    # not match, we restart ourselves with matching configuration.
    needs_reexec = False

    current_version = "%d.%d" % (sys.version_info[0], sys.version_info[1])

    # We support to execute with a specified version.
    intended_version = Options.getIntendedPythonVersion()
    if intended_version is None:
        intended_version = current_version

    # If it's a different version, we find it by guessing it, otherwise we use the
    # one previously used.
    if current_version != intended_version:
        if Utils.getOS() == "Windows":
            python_binary = Execution.getPythonExePathWindows(
                intended_version,
                Options.getIntendedPythonArch()
            )
        else:
            python_binary = Execution.getExecutablePath("python" + intended_version)

        if python_binary is None:
            sys.exit(
                "Error, cannot find Python %s binary in PATH (%s)." % (
                    intended_version,
                    os.environ.get("PATH", "")
                )
            )

        needs_reexec = True
    else:
        python_binary = sys.executable

    if sys.flags.no_site == 0:
        needs_reexec = True

    # The hash randomization totally changes the created source code created,
    # changing it every single time Nuitka is run. This kills any attempt at
    # caching it, and comparing generated source code. While the created binary
    # actually may still use it, during compilation we don't want to. So lets
    # disable it.
    if os.environ.get("PYTHONHASHSEED", "-1") != '0':
        needs_reexec = True

    # In case we need to re-execute.
    if needs_reexec:
        if not Options.isAllowedToReexecute():
            sys.exit("Error, not allowed to re-execute, but that would be needed.")

        our_filename = sys.modules[__name__].__file__

        # Workaround for --python-version which will choke on existing, but
        # not matching .pyc files.
        if current_version != intended_version:
            pyc_filename = our_filename[:-2] + ".pyc"

            if os.path.exists(pyc_filename):
                try:
                    os.unlink(pyc_filename)
                except OSError:
                    pass

        # Execute with full path as the process name, so it can find itself and its
        # libraries.
        args = [
            python_binary,
            python_binary,
            "-S",
            our_filename,
        ]

        os.environ["NUITKA_BINARY_NAME"] = sys.modules["__main__"].__file__

        if Options.is_nuitka_run:
            args.append("--run")

        # Same arguments as before.
        args += sys.argv[1:] + list(Options.getMainArgs())

        if current_version == intended_version:
            os.environ["NUITKA_PYTHONPATH"] = repr(
                sys.path
            )

            from nuitka.importing.PreloadedPackages import detectPreLoadedPackagePaths, detectPthImportedPackages
            os.environ["NUITKA_NAMESPACES"] = repr(
                detectPreLoadedPackagePaths()
            )

            if "site" in sys.modules:
                os.environ["NUITKA_SITE_FILENAME"] = sys.modules["site"].__file__

                os.environ["NUITKA_PTH_IMPORTED"] = repr(detectPthImportedPackages())


        os.environ["NUITKA_SITE_FLAG"] = str(sys.flags.no_site) \
                                           if "no_site" not in Options.getPythonFlags() \
                                         else '1'

        os.environ["PYTHONHASHSEED"] = '0'

        Execution.callExec(args)

    if Options.isShowMemory():
        from nuitka.utils import MemoryUsage
        MemoryUsage.startMemoryTracing()

    # Inform the user about potential issues.
    if current_version not in Options.getSupportedPythonVersions():

        # Do not disturb run of automatic tests, detected from the presence of
        # that environment variable.
        if "PYTHON" not in os.environ:
            logging.warning(
                "The version '%s' is not currently supported. Expect problems.",
                current_version
            )

    if "NUITKA_NAMESPACES" in os.environ:
        # Restore the detected name space packages, that were force loaded in
        # site.py, and will need a free pass later on. pylint: disable=eval-used

        from nuitka.importing.PreloadedPackages import setPreloadedPackagePaths

        setPreloadedPackagePaths(eval(os.environ["NUITKA_NAMESPACES"]))
        del os.environ["NUITKA_NAMESPACES"]

    if "NUITKA_PTH_IMPORTED" in os.environ:
        # Restore the packages that the ".pth" files asked to import.
        # pylint: disable=eval-used

        from nuitka.importing.PreloadedPackages import setPthImportedPackages

        setPthImportedPackages(eval(os.environ["NUITKA_PTH_IMPORTED"]))
        del os.environ["NUITKA_PTH_IMPORTED"]

    # Now the real main program of Nuitka can take over.
    from nuitka import MainControl  # isort:skip
    MainControl.main()

    if Options.isShowMemory():
        MemoryUsage.showMemoryTrace()
예제 #53
0
def getSourceDirectoryPath(main_module):
    assert main_module.isCompiledPythonModule()

    return Options.getOutputPath(
        path=Utils.basename(getTreeFilenameWithSuffix(main_module, ".build")))
예제 #54
0
def getScanDirectories(package_name, original_dir):
    # Many cases, pylint: disable=too-many-branches

    cache_key = package_name, original_dir

    if cache_key in _scan_dir_cache:
        return _scan_dir_cache[cache_key]

    scan_dirs = [sys.prefix]

    if package_name is not None:
        from nuitka.importing.Importing import findModule

        package_dir = findModule(None, package_name, None, 0, False)[1]

        if os.path.isdir(package_dir):
            scan_dirs.append(package_dir)
            scan_dirs.extend(getSubDirectories(package_dir))

    if original_dir is not None:
        scan_dirs.append(original_dir)
        scan_dirs.extend(getSubDirectories(original_dir))

    if (Utils.isWin32Windows() and package_name is not None
            and package_name.isBelowNamespace("win32com")):
        pywin32_dir = getPyWin32Dir()

        if pywin32_dir is not None:
            scan_dirs.append(pywin32_dir)

    for path_dir in os.environ["PATH"].split(";"):
        if not os.path.isdir(path_dir):
            continue

        if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
            continue
        if areSamePaths(path_dir,
                        os.path.join(os.environ["SYSTEMROOT"], "System32")):
            continue
        if areSamePaths(path_dir,
                        os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
            continue

        scan_dirs.append(path_dir)

    result = []

    # Remove directories that hold no DLLs.
    for scan_dir in scan_dirs:
        sys.stdout.flush()

        # These are useless, but plenty.
        if os.path.basename(scan_dir) == "__pycache__":
            continue

        scan_dir = getDirectoryRealPath(scan_dir)

        # No DLLs, no use.
        if not any(entry[1].lower().endswith(".dll")
                   for entry in listDir(scan_dir)):
            continue

        result.append(os.path.realpath(scan_dir))

    _scan_dir_cache[cache_key] = result
    return result
예제 #55
0
 def getOutputFilename(self):
     return Utils.dirname(self.getFilename())
예제 #56
0
def _detectBinaryPathDLLsPosix(dll_filename):
    # This is complex, as it also includes the caching mechanism
    # pylint: disable=too-many-branches

    if ldd_result_cache.get(dll_filename):
        return ldd_result_cache[dll_filename]

    # Ask "ldd" about the libraries being used by the created binary, these
    # are the ones that interest us.
    result = set()

    # This is the rpath of the Python binary, which will be effective when
    # loading the other DLLs too. This happens at least for Python installs
    # on Travis. pylint: disable=global-statement
    global _detected_python_rpath
    if _detected_python_rpath is None and not Utils.isPosixWindows():
        _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False

        if _detected_python_rpath:
            _detected_python_rpath = _detected_python_rpath.replace(
                "$ORIGIN", os.path.dirname(sys.executable))

    with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath):
        process = subprocess.Popen(
            args=["ldd", dll_filename],
            stdin=getNullInput(),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )

        stdout, stderr = process.communicate()

        stderr = b"\n".join(
            line for line in stderr.splitlines() if not line.startswith(
                b"ldd: warning: you do not have execution permission for"))

        inclusion_logger.debug("ldd output for %s is:\n%s" %
                               (dll_filename, stdout))

        if stderr:
            inclusion_logger.debug("ldd error for %s is:\n%s" %
                                   (dll_filename, stderr))

        for line in stdout.split(b"\n"):
            if not line:
                continue

            if b"=>" not in line:
                continue

            part = line.split(b" => ", 2)[1]

            if b"(" in part:
                filename = part[:part.rfind(b"(") - 1]
            else:
                filename = part

            if not filename:
                continue

            if python_version >= 0x300:
                filename = filename.decode("utf-8")

            # Sometimes might use stuff not found or supplied by ldd itself.
            if filename in ("not found", "ldd"):
                continue

            # Do not include kernel / glibc specific libraries. This list has been
            # assembled by looking what are the most common .so files provided by
            # glibc packages from ArchLinux, Debian Stretch and CentOS.
            #
            # Online sources:
            #  - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html
            #  - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html
            #  - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html
            #  - https://packages.debian.org/stretch/amd64/libc6/filelist
            #
            # Note: This list may still be incomplete. Some additional libraries
            # might be provided by glibc - it may vary between the package versions
            # and between Linux distros. It might or might not be a problem in the
            # future, but it should be enough for now.
            if os.path.basename(filename).startswith((
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
            )):
                continue

            result.add(filename)

    ldd_result_cache[dll_filename] = result

    sub_result = set(result)

    for sub_dll_filename in result:
        sub_result = sub_result.union(
            _detectBinaryPathDLLsPosix(sub_dll_filename))

    return sub_result
예제 #57
0
파일: Standalone.py 프로젝트: luzpaz/Nuitka
def scanStandardLibraryPath(stdlib_dir):
    # There is a lot of black-listing here, done in branches, so there
    # is many of them, but that's acceptable, pylint: disable=too-many-branches

    for root, dirs, filenames in os.walk(stdlib_dir):
        import_path = root[len(stdlib_dir):].strip("/\\")
        import_path = import_path.replace('\\', '.').replace('/', '.')

        if import_path == "":
            if "site-packages" in dirs:
                dirs.remove("site-packages")
            if "dist-packages" in dirs:
                dirs.remove("dist-packages")
            if "test" in dirs:
                dirs.remove("test")
            if "idlelib" in dirs:
                dirs.remove("idlelib")
            if "turtledemo" in dirs:
                dirs.remove("turtledemo")

            if "ensurepip" in filenames:
                filenames.remove("ensurepip")
            if "ensurepip" in dirs:
                dirs.remove("ensurepip")

            # Ignore "lib-dynload" and "lib-tk" and alikes.
            dirs[:] = [
                dirname for dirname in dirs if not dirname.startswith("lib-")
                if dirname != "Tools"
            ]

        if import_path in ("tkinter", "importlib", "ctypes", "unittest",
                           "sqlite3", "distutils", "email", "bsddb"):
            if "test" in dirs:
                dirs.remove("test")

        if import_path in ("lib2to3", "json", "distutils"):
            if "tests" in dirs:
                dirs.remove("tests")

        if python_version >= 340 and Utils.getOS() == "Windows":
            if import_path == "multiprocessing":
                filenames.remove("popen_fork.py")
                filenames.remove("popen_forkserver.py")
                filenames.remove("popen_spawn_posix.py")

        if Utils.getOS() == "NetBSD":
            if import_path == "xml.sax":
                filenames.remove("expatreader.py")

        for filename in filenames:
            if filename.endswith(".py") and filename not in ignore_modules:
                module_name = filename[:-3]
                if import_path == "":
                    yield module_name
                else:
                    yield import_path + '.' + module_name

        if python_version >= 300:
            if "__pycache__" in dirs:
                dirs.remove("__pycache__")

        for dirname in dirs:
            if import_path == "":
                yield dirname
            else:
                yield import_path + '.' + dirname
예제 #58
0
def makeSourceDirectory(main_module):
    """ Get the full list of modules imported, create code for all of them.

    """
    # We deal with a lot of details here, but rather one by one, and split makes
    # no sense, pylint: disable=R0912,R0914

    assert main_module.isCompiledPythonModule()

    # The global context used to generate code.
    global_context = CodeGeneration.makeGlobalContext()

    assert main_module in ModuleRegistry.getDoneModules()

    # We might have chosen to include it as bytecode, and only compiled it for
    # fun, and to find its imports. In this case, now we just can drop it. Or
    # a module may shadow a frozen module, but be a different one, then we can
    # drop the frozen one.
    # TODO: This really should be done when the compiled module comes into
    # existence.
    for module in ModuleRegistry.getDoneUserModules():
        if module.isCompiledPythonModule():
            uncompiled_module = ModuleRegistry.getUncompiledModule(
                module_name     = module.getFullName(),
                module_filename = module.getCompileTimeFilename()
            )

            if uncompiled_module is not None:
                # We now need to decide which one to keep, compiled or uncompiled
                # module. Some uncompiled modules may have been asked by the user
                # or technically required. By default, frozen code if it exists
                # is preferred, as it will be from standalone mode adding it.
                if uncompiled_module.isUserProvided():
                    ModuleRegistry.removeDoneModule(module)
                else:
                    ModuleRegistry.removeUncompiledModule(uncompiled_module)

    # Lets check if the recurse-to modules are actually present, and warn the
    # user if one of those was not found.
    for any_case_module in Options.getShallFollowModules():
        for module in ModuleRegistry.getDoneUserModules():
            if module.getFullName() == any_case_module:
                break
        else:
            warning(
                "Didn't recurse to '%s', apparently not used." % \
                any_case_module
            )

    # Prepare code generation, i.e. execute finalization for it.
    for module in ModuleRegistry.getDoneModules():
        if module.isCompiledPythonModule():
            Finalization.prepareCodeGeneration(module)

    # Pick filenames.
    source_dir = getSourceDirectoryPath(main_module)

    module_filenames = pickSourceFilenames(
        source_dir = source_dir,
        modules    = ModuleRegistry.getDoneModules()
    )

    # First pass, generate code and use constants doing so, but prepare the
    # final code generation only, because constants code will be added at the
    # end only.
    prepared_modules = {}

    for module in ModuleRegistry.getDoneModules():
        if module.isCompiledPythonModule():
            cpp_filename = module_filenames[module]

            prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode(
                global_context = global_context,
                module         = module,
                module_name    = module.getFullName(),
            )

            # Main code constants need to be allocated already too.
            if module is main_module and not Options.shallMakeModule():
                prepared_modules[cpp_filename][1].getConstantCode(0)

    # Second pass, generate the actual module code into the files.
    for module in ModuleRegistry.getDoneModules():
        if module.isCompiledPythonModule():
            cpp_filename = module_filenames[module]

            template_values, module_context = prepared_modules[cpp_filename]

            source_code = CodeGeneration.generateModuleCode(
                module_context  = module_context,
                template_values = template_values
            )

            writeSourceCode(
                filename    = cpp_filename,
                source_code = source_code
            )

            if Options.isShowInclusion():
                info("Included compiled module '%s'." % module.getFullName())
        elif module.isPythonShlibModule():
            target_filename = Utils.joinpath(
                getStandaloneDirectoryPath(main_module),
                *module.getFullName().split('.')
            )

            if Utils.getOS() == "Windows":
                target_filename += ".pyd"
            else:
                target_filename += ".so"

            target_dir = Utils.dirname(target_filename)

            if not Utils.isDir(target_dir):
                Utils.makePath(target_dir)

            shutil.copy(
                module.getFilename(),
                target_filename
            )

            standalone_entry_points.append(
                (target_filename, module.getPackage())
            )
        elif module.isUncompiledPythonModule():
            pass
        else:
            assert False, module

    writeSourceCode(
        filename    = Utils.joinpath(
            source_dir,
            "__constants.c"
        ),
        source_code = ConstantCodes.getConstantsDefinitionCode(
            context = global_context
        )
    )

    helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
        ModuleRegistry.getDoneUserModules()
    )

    writeSourceCode(
        filename    = Utils.joinpath(source_dir, "__helpers.h"),
        source_code = helper_decl_code
    )

    writeSourceCode(
        filename    = Utils.joinpath(source_dir, "__helpers.c"),
        source_code = helper_impl_code
    )
예제 #59
0
def runScons(main_module, quiet):
    # Scons gets transported many details, that we express as variables, and
    # have checks for them, leading to many branches, pylint: disable=R0912

    python_version_str = "%d.%d" % (sys.version_info[0], sys.version_info[1])

    if hasattr(sys, "abiflags"):
        if Options.isPythonDebug() or \
           hasattr(sys, "getobjects"):
            if sys.abiflags.startswith('d'):
                python_version_str += sys.abiflags
            else:
                python_version_str += 'd' + sys.abiflags
        else:
            python_version_str += sys.abiflags

    def asBoolStr(value):
        return "true" if value else "false"

    options = {
        "name"            : Utils.basename(
            getTreeFilenameWithSuffix(main_module, "")
        ),
        "result_name"     : getResultBasepath(main_module),
        "source_dir"      : getSourceDirectoryPath(main_module),
        "debug_mode"      : asBoolStr(Options.isDebug()),
        "python_debug"    : asBoolStr(Options.isPythonDebug()),
        "unstripped_mode" : asBoolStr(Options.isUnstripped()),
        "module_mode"     : asBoolStr(Options.shallMakeModule()),
        "optimize_mode"   : asBoolStr(Options.isOptimize()),
        "full_compat"     : asBoolStr(Options.isFullCompat()),
        "experimental"    : asBoolStr(Options.isExperimental()),
        "trace_mode"      : asBoolStr(Options.shallTraceExecution()),
        "python_version"  : python_version_str,
        "target_arch"     : Utils.getArchitecture(),
        "python_prefix"   : sys.prefix,
        "nuitka_src"      : SconsInterface.getSconsDataPath(),
        "module_count"    : "%d" % (
            1 + \
            len(ModuleRegistry.getDoneUserModules()) + \
            len(ModuleRegistry.getUncompiledNonTechnicalModules())
        )
    }

    # Ask Scons to cache on Windows, except where the directory is thrown
    # away. On non-Windows you can should use ccache instead.
    if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows":
        options["cache_mode"] = "true"

    if Options.isLto():
        options["lto_mode"] = "true"

    if Options.shallDisableConsoleWindow():
        options["win_disable_console"] = "true"

    if Options.isStandaloneMode():
        options["standalone_mode"] = "true"

    if not Options.isStandaloneMode() and \
       not Options.shallMakeModule() and \
       isUninstalledPython():
        options["uninstalled_python"] = "true"

    if ModuleRegistry.getUncompiledTechnicalModules():
        options["frozen_modules"] = str(
            len(ModuleRegistry.getUncompiledTechnicalModules())
        )

    if Options.isShowScons():
        options["show_scons"] = "true"

    if Options.isMingw():
        options["mingw_mode"] = "true"

    if Options.getMsvcVersion():
        msvc_version = Options.getMsvcVersion()

        msvc_version = msvc_version.replace("exp", "Exp")
        if '.' not in msvc_version:
            msvc_version += ".0"

        options["msvc_version"] = msvc_version

    if Options.isClang():
        options["clang_mode"] = "true"

    if Options.getIconPath():
        options["icon_path"] = Options.getIconPath()

    if Options.isProfile():
        options["profile_mode"] = "true"

    if "no_warnings" in getPythonFlags():
        options["no_python_warnings"] = "true"

    if python_version < 300 and sys.flags.py3k_warning:
        options["python_sysflag_py3k_warning"] = "true"

    if python_version < 300 and (sys.flags.division_warning or sys.flags.py3k_warning):
        options["python_sysflag_division_warning"] = "true"

    if sys.flags.bytes_warning:
        options["python_sysflag_bytes_warning"] = "true"

    if int(os.environ.get("NUITKA_SITE_FLAG", "no_site" in Options.getPythonFlags())):
        options["python_sysflag_no_site"] = "true"

    if "trace_imports" in Options.getPythonFlags():
        options["python_sysflag_verbose"] = "true"

    if python_version < 300 and sys.flags.unicode:
        options["python_sysflag_unicode"] = "true"


    return SconsInterface.runScons(options, quiet), options
예제 #60
0
def getOutputPath(path):
    if options.output_dir:
        return Utils.normpath(Utils.joinpath(options.output_dir, path))
    else:
        return path