Beispiel #1
0
def main():
    python_version = setup(suite="onefile", needs_io_encoding=True)

    search_mode = createSearchMode()

    if getOS() == "Linux":
        addExtendedExtraOptions(
            "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
        )

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "remove_output",
            # Keep the binary, normally "remove_output" includes that.
            "--keep-binary",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
            # The onefile can warn about zstandard not being installed.
            "ignore_warnings",
        ]

        if filename == "KeyboardInterruptTest.py":
            if getOS() == "Darwin":
                reportSkip(
                    "Exit code from KeyboardInterrupt on macOS is not yet good.",
                    ".",
                    filename,
                )
                continue

            if python_version < (3,):
                reportSkip(
                    "Python2 reports KeyboardInterrupt, but too late",
                    ".",
                    filename,
                )
                continue

            if os.name == "nt":
                reportSkip(
                    "Testing cannot send KeyboardInterrupt on Windows yet",
                    ".",
                    filename,
                )
                continue

            extra_flags.append("--send-ctrl-c")

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        test_logger.info(
            "Consider output of onefile mode compiled program: %s" % filename
        )

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")

        if filename == "KeyboardInterruptTest.py":
            continue

        # Then use "strace" on the result.
        with TimerReport(
            "Determining run time loaded files took %.2f", logger=test_logger
        ):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, command=[binary_filename]
            )

        illegal_accesses = checkLoadedFileAccesses(
            loaded_filenames=loaded_filenames, current_dir=os.getcwd()
        )

        if illegal_accesses:
            displayError(None, filename)
            displayRuntimeTraces(test_logger, binary_filename)

            test_logger.warning(
                "Should not access these file(s): '%s'." % ",".join(illegal_accesses)
            )

            search_mode.onErrorDetected(1)

        os.unlink(binary_filename)

    search_mode.finish()
Beispiel #2
0
def main():
    # Complex stuff, even more should become common code or project options though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(suite="standalone", needs_io_encoding=True)

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "--standalone",
            "remove_output",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
        ]

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        if filename == "Urllib3Using.py" and os.name == "nt":
            reportSkip(
                "Socket module early import not working on Windows currently",
                ".",
                filename,
            )
            continue

        if "Idna" in filename:
            # For the warnings of Python2.
            if python_version < (3, ):
                extra_flags.append("ignore_stderr")

        if filename == "CtypesUsing.py":
            extra_flags.append("plugin_disable:pylint-warnings")

        if filename == "GtkUsing.py":
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version < (2, 7):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith("Win"):
            if os.name != "nt":
                reportSkip("Windows only test", ".", filename)
                continue

        if filename == "TkInterUsing.py":
            if getOS() == "Darwin":
                reportSkip("Not working macOS yet", ".", filename)
                continue

            if getOS() == "Windows":
                reportSkip("Can hang on Windows CI.", ".", filename)
                continue

            # For the plug-in information.
            extra_flags.append("plugin_enable:tk-inter")

        if filename == "FlaskUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        # TODO: Once we have a no-Qt Plugin, we should use that.
        if filename == "MatplotlibUsing.py":
            # For the plugin warnings.
            extra_flags.append("ignore_warnings")

        if filename == "NumpyUsing.py":
            # TODO: Disabled for now.
            reportSkip("numpy.test not fully working yet", ".", filename)
            continue

        if filename == "PandasUsing.py":
            extra_flags.append("plugin_enable:numpy")
            extra_flags.append("plugin_disable:pylint-warnings")
            extra_flags.append("plugin_disable:pyqt5")
            extra_flags.append("plugin_disable:pyside2")
            extra_flags.append("plugin_disable:pyside6")

        if filename == "PmwUsing.py":
            extra_flags.append("plugin_enable:pmw-freezer")

        if filename == "OpenGLUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "GlfwUsing.py":
            # For the warnings.
            extra_flags.append("plugin_enable:numpy")

        if filename == "PasslibUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "Win32ComUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")):
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the plug-in information
            if filename.startswith("PySide2"):
                extra_flags.append("plugin_enable:pyside6")
            elif filename.startswith("PySide6"):
                extra_flags.append("plugin_enable:pyside6")
            elif filename.startswith("PyQt5"):
                extra_flags.append("plugin_enable:pyqt5")
            elif filename.startswith("PyQt6"):
                extra_flags.append("plugin_enable:pyqt6")

        test_logger.info(
            "Consider output of standalone mode compiled program: %s" %
            filename)

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        # Second check if glibc libraries haven't been accidentally
        # shipped with the standalone executable
        found_glibc_libs = []
        for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
            if os.path.basename(dist_filename).startswith((
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
            )):
                found_glibc_libs.append(dist_filename)

        if found_glibc_libs:
            test_logger.warning(
                "Should not ship glibc libraries with the standalone executable (found %s)"
                % found_glibc_libs)
            sys.exit(1)

        binary_filename = os.path.join(
            filename[:-3] + ".dist",
            filename[:-3] + (".exe" if os.name == "nt" else ""))

        # Then use "strace" on the result.
        with TimerReport("Determining run time loaded files took %.2f",
                         logger=test_logger):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, command=[binary_filename])

        illegal_accesses = checkLoadedFileAccesses(
            loaded_filenames=loaded_filenames, current_dir=os.getcwd())

        if illegal_accesses:
            displayError(None, filename)
            displayRuntimeTraces(test_logger, binary_filename)

            test_logger.warning("Should not access these file(s): '%r'." %
                                illegal_accesses)

            search_mode.onErrorDetected(1)

        removeDirectory(filename[:-3] + ".dist", ignore_errors=True)

    search_mode.finish()
Beispiel #3
0
def main():
    # Complex stuff, even more should become common code though.
    # pylint: disable=too-many-branches,too-many-locals,too-many-statements

    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "--standalone",
            "remove_output",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
        ]

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        # catch error
        if filename == "Boto3Using.py":
            reportSkip("boto3 test not fully working yet", ".", filename)
            continue

        if "Idna" in filename:
            # For the warnings of Python2.
            if python_version < (3,):
                extra_flags.append("ignore_stderr")

        if filename == "CtypesUsing.py":
            extra_flags.append("plugin_disable:pylint-warnings")

        if filename == "GtkUsing.py":
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version < (2, 7):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith("Win"):
            if os.name != "nt":
                reportSkip("Windows only test", ".", filename)
                continue

        if filename == "TkInterUsing.py":
            if getOS() == "Darwin":
                reportSkip("Not working macOS yet", ".", filename)
                continue

            # For the plug-in information.
            extra_flags.append("plugin_enable:tk-inter")

        if filename == "FlaskUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "NumpyUsing.py":
            extra_flags.append("plugin_enable:numpy")

            # TODO: Disabled for now.
            reportSkip("numpy.test not fully working yet", ".", filename)
            continue

        if filename == "PandasUsing.py":
            extra_flags.append("plugin_enable:numpy")
            extra_flags.append("plugin_disable:pylint-warnings")
            extra_flags.append("plugin_disable:qt-plugins")

        if filename == "PmwUsing.py":
            extra_flags.append("plugin_enable:pmw-freezer")

        if filename == "OpenGLUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "PasslibUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "PySideUsing.py":
            # TODO: Disabled due to lack of upstream support.
            reportSkip("PySide not supported yet", ".", filename)
            continue

        if filename == "Win32ComUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith(("PySide", "PyQt")):
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version < (2, 7):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the plug-in information.
            if getPythonVendor() != "Anaconda":
                extra_flags.append("plugin_enable:qt-plugins")
            else:
                # For the plug-in not used information.
                extra_flags.append("ignore_warnings")

        test_logger.info(
            "Consider output of standalone mode compiled program: %s" % filename
        )

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        # Second check if glibc libraries haven't been accidentally
        # shipped with the standalone executable
        found_glibc_libs = []
        for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
            if os.path.basename(dist_filename).startswith(
                (
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
                )
            ):
                found_glibc_libs.append(dist_filename)

        if found_glibc_libs:
            test_logger.warning(
                "Should not ship glibc libraries with the standalone executable (found %s)"
                % found_glibc_libs
            )
            sys.exit(1)

        binary_filename = os.path.join(
            filename[:-3] + ".dist", filename[:-3] + (".exe" if os.name == "nt" else "")
        )

        # Then use "strace" on the result.
        with TimerReport(
            "Determining run time loaded files took %.2f", logger=test_logger
        ):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, path=binary_filename
            )

        current_dir = os.path.normpath(os.getcwd())
        current_dir = os.path.normcase(current_dir)

        illegal_access = False

        for loaded_filename in loaded_filenames:
            loaded_filename = os.path.normpath(loaded_filename)
            loaded_filename = os.path.normcase(loaded_filename)
            loaded_basename = os.path.basename(loaded_filename)

            if os.name == "nt":
                if areSamePaths(
                    os.path.dirname(loaded_filename),
                    os.path.normpath(
                        os.path.join(os.environ["SYSTEMROOT"], "System32")
                    ),
                ):
                    continue
                if areSamePaths(
                    os.path.dirname(loaded_filename),
                    os.path.normpath(
                        os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
                    ),
                ):
                    continue

                if r"windows\winsxs" in loaded_filename:
                    continue

                # Github actions have these in PATH overriding SYSTEMROOT
                if r"windows performance toolkit" in loaded_filename:
                    continue
                if r"powershell" in loaded_filename:
                    continue
                if r"azure dev spaces cli" in loaded_filename:
                    continue

            if loaded_filename.startswith(current_dir):
                continue

            if loaded_filename.startswith(os.path.abspath(current_dir)):
                continue

            if loaded_filename.startswith("/etc/"):
                continue

            if loaded_filename.startswith("/usr/etc/"):
                continue

            if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
                continue

            if loaded_filename.startswith("/dev/"):
                continue

            if loaded_filename.startswith("/tmp/"):
                continue

            if loaded_filename.startswith("/run/"):
                continue

            if loaded_filename.startswith("/usr/lib/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/X11/locale/"):
                continue

            # Themes may of course be loaded.
            if loaded_filename.startswith("/usr/share/themes"):
                continue
            if "gtk" in loaded_filename and "/engines/" in loaded_filename:
                continue

            if loaded_filename in (
                "/usr",
                "/usr/local",
                "/usr/local/lib",
                "/usr/share",
                "/usr/local/share",
                "/usr/lib64",
            ):
                continue

            # TCL/tk for tkinter for non-Windows is OK.
            if loaded_filename.startswith(
                (
                    "/usr/lib/tcltk/",
                    "/usr/share/tcltk/",
                    "/usr/lib/tcl/",
                    "/usr/lib64/tcl/",
                )
            ):
                continue
            if loaded_filename in (
                "/usr/lib/tcltk",
                "/usr/share/tcltk",
                "/usr/lib/tcl",
                "/usr/lib64/tcl",
            ):
                continue

            if loaded_filename in (
                "/lib",
                "/lib64",
                "/lib/sse2",
                "/lib/tls",
                "/lib64/tls",
                "/usr/lib/sse2",
                "/usr/lib/tls",
                "/usr/lib64/tls",
            ):
                continue

            if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
                continue
            if loaded_filename in (
                "/usr/share/tcl8.6/init.tcl",
                "/usr/share/tcl8.5/init.tcl",
            ):
                continue
            if loaded_filename in (
                "/usr/share/tcl8.6/encoding",
                "/usr/share/tcl8.5/encoding",
            ):
                continue

            # System SSL config on Linux. TODO: Should this not be included and
            # read from dist folder.
            if loaded_basename == "openssl.cnf":
                continue

            # Taking these from system is harmless and desirable
            if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
                continue

            # System C libraries are to be expected.
            if loaded_basename.startswith(
                (
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
                )
            ):
                continue

            # Loaded by C library potentially for DNS lookups.
            if loaded_basename.startswith(
                (
                    "libnss_",
                    "libnsl",
                    # Some systems load a lot more, this is CentOS 7 on OBS
                    "libattr.so.",
                    "libbz2.so.",
                    "libcap.so.",
                    "libdw.so.",
                    "libelf.so.",
                    "liblzma.so.",
                    # Some systems load a lot more, this is Fedora 26 on OBS
                    "libselinux.so.",
                    "libpcre.so.",
                    # And this is Fedora 29 on OBS
                    "libblkid.so.",
                    "libmount.so.",
                    "libpcre2-8.so.",
                    # CentOS 8 on OBS
                    "libuuid.so.",
                )
            ):
                continue

            # Loaded by dtruss on macOS X.
            if loaded_filename.startswith("/usr/lib/dtrace/"):
                continue

            # Loaded by cowbuilder and pbuilder on Debian
            if loaded_basename == ".ilist":
                continue
            if "cowdancer" in loaded_filename:
                continue
            if "eatmydata" in loaded_filename:
                continue

            # Loading from home directories is OK too.
            if (
                loaded_filename.startswith("/home/")
                or loaded_filename.startswith("/data/")
                or loaded_filename.startswith("/root/")
                or loaded_filename in ("/home", "/data", "/root")
            ):
                continue

            # For Debian builders, /build is OK too.
            if loaded_filename.startswith("/build/") or loaded_filename == "/build":
                continue

            # TODO: Unclear, loading gconv from filesystem of installed system
            # may be OK or not. I think it should be.
            if loaded_basename == "gconv-modules.cache":
                continue
            if "/gconv/" in loaded_filename:
                continue
            if loaded_basename.startswith("libicu"):
                continue
            if loaded_filename.startswith("/usr/share/icu/"):
                continue

            # Loading from caches is OK.
            if loaded_filename.startswith("/var/cache/"):
                continue

            lib_prefix_dir = "/usr/lib/python%d.%s" % (
                python_version[0],
                python_version[1],
            )

            # PySide accesses its directory.
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
                continue

            # GTK accesses package directories only.
            if loaded_filename == os.path.join(
                lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
            ):
                continue
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
                continue
            if loaded_filename == os.path.join(
                lib_prefix_dir, "dist-packages/gtk-2.0/gio"
            ):
                continue
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
                continue

            # PyQt5 seems to do this, but won't use contents then.
            if loaded_filename in (
                "/usr/lib/qt5/plugins",
                "/usr/lib/qt5",
                "/usr/lib64/qt5/plugins",
                "/usr/lib64/qt5",
                "/usr/lib/x86_64-linux-gnu/qt5/plugins",
                "/usr/lib/x86_64-linux-gnu/qt5",
                "/usr/lib/x86_64-linux-gnu",
                "/usr/lib",
            ):
                continue

            # Can look at the interpreters of the system.
            if loaded_basename in "python3":
                continue
            if loaded_basename in (
                "python%s" + supported_version
                for supported_version in (
                    getSupportedPythonVersions() + getPartiallySupportedPythonVersions()
                )
            ):
                continue

            # Current Python executable can actually be a symlink and
            # the real executable which it points to will be on the
            # loaded_filenames list. This is all fine, let's ignore it.
            # Also, because the loaded_filename can be yet another symlink
            # (this is weird, but it's true), let's better resolve its real
            # path too.
            if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
                continue

            # Accessing SE-Linux is OK.
            if loaded_filename in ("/sys/fs/selinux", "/selinux"):
                continue

            # Looking at device is OK.
            if loaded_filename.startswith("/sys/devices/"):
                continue

            # Allow reading time zone info of local system.
            if loaded_filename.startswith("/usr/share/zoneinfo/"):
                continue

            # The access to .pth files has no effect.
            if loaded_filename.endswith(".pth"):
                continue

            # Looking at site-package dir alone is alone.
            if loaded_filename.endswith(("site-packages", "dist-packages")):
                continue

            # QtNetwork insist on doing this it seems.
            if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
                continue

            # macOS uses these:
            if loaded_basename in (
                "libcrypto.1.0.0.dylib",
                "libssl.1.0.0.dylib",
                "libcrypto.1.1.dylib",
            ):
                continue

            # MSVC run time DLLs, seem to sometimes come from system.
            if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
                continue

            test_logger.warning("Should not access '%s'." % loaded_filename)
            illegal_access = True

        if illegal_access:
            if os.name != "nt":
                displayError(None, filename)
                displayRuntimeTraces(test_logger, binary_filename)

            search_mode.onErrorDetected(1)

        removeDirectory(filename[:-3] + ".dist", ignore_errors=True)

        if search_mode.abortIfExecuted():
            break

    search_mode.finish()
Beispiel #4
0
def main():
    # Complex stuff, even more should become common code though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    if getOS() == "Linux":
        addExtendedExtraOptions(
            "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
        )

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "--onefile",
            "remove_output",
            # Keep the binary, normally "remove_output" includes that.
            "--keep-binary",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
        ]

        if filename == "KeyboardInteruptTest.py":
            if python_version < (3,):
                reportSkip(
                    "Python2 reports KeyboardInterrupt, but too late",
                    ".",
                    filename,
                )
                continue

            if os.name == "nt":
                reportSkip(
                    ".",
                    filename,
                    "Testing cannot send KeyboardInterrupt on Windows yet",
                )
                continue

            extra_flags.append("--send-ctrl-c")

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        test_logger.info(
            "Consider output of onefile mode compiled program: %s" % filename
        )

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")

        if filename == "KeyboardInteruptTest.py":
            continue

        # Then use "strace" on the result.
        with TimerReport(
            "Determining run time loaded files took %.2f", logger=test_logger
        ):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, path=binary_filename
            )

        current_dir = os.path.normpath(os.getcwd())
        current_dir = os.path.normcase(current_dir)

        illegal_access = False

        for loaded_filename in loaded_filenames:
            loaded_filename = os.path.normpath(loaded_filename)
            loaded_filename = os.path.normcase(loaded_filename)
            loaded_basename = os.path.basename(loaded_filename)

            if os.name == "nt":
                if areSamePaths(
                    os.path.dirname(loaded_filename),
                    os.path.normpath(
                        os.path.join(os.environ["SYSTEMROOT"], "System32")
                    ),
                ):
                    continue
                if areSamePaths(
                    os.path.dirname(loaded_filename),
                    os.path.normpath(
                        os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
                    ),
                ):
                    continue

                if r"windows\winsxs" in loaded_filename:
                    continue

                # Github actions have these in PATH overriding SYSTEMROOT
                if r"windows performance toolkit" in loaded_filename:
                    continue
                if r"powershell" in loaded_filename:
                    continue
                if r"azure dev spaces cli" in loaded_filename:
                    continue
                if r"tortoisesvn" in loaded_filename:
                    continue

            if loaded_filename.startswith(current_dir):
                continue

            if loaded_filename.startswith(os.path.abspath(current_dir)):
                continue

            if loaded_filename.startswith("/etc/"):
                continue

            if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
                continue

            if loaded_filename.startswith("/dev/"):
                continue

            if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp":
                continue

            if loaded_filename.startswith("/run/"):
                continue

            if loaded_filename.startswith("/usr/lib/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/X11/locale/"):
                continue

            # Themes may of course be loaded.
            if loaded_filename.startswith("/usr/share/themes"):
                continue
            if "gtk" in loaded_filename and "/engines/" in loaded_filename:
                continue

            if loaded_filename in (
                "/usr",
                "/usr/local",
                "/usr/local/lib",
                "/usr/share",
                "/usr/local/share",
                "/usr/lib64",
            ):
                continue

            # TCL/tk for tkinter for non-Windows is OK.
            if loaded_filename.startswith(
                (
                    "/usr/lib/tcltk/",
                    "/usr/share/tcltk/",
                    "/usr/lib/tcl/",
                    "/usr/lib64/tcl/",
                )
            ):
                continue
            if loaded_filename in (
                "/usr/lib/tcltk",
                "/usr/share/tcltk",
                "/usr/lib/tcl",
                "/usr/lib64/tcl",
            ):
                continue

            if loaded_filename in (
                "/lib",
                "/lib64",
                "/lib/sse2",
                "/lib/tls",
                "/lib64/tls",
                "/usr/lib/sse2",
                "/usr/lib/tls",
                "/usr/lib64/tls",
            ):
                continue

            if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
                continue
            if loaded_filename in (
                "/usr/share/tcl8.6/init.tcl",
                "/usr/share/tcl8.5/init.tcl",
            ):
                continue
            if loaded_filename in (
                "/usr/share/tcl8.6/encoding",
                "/usr/share/tcl8.5/encoding",
            ):
                continue

            # System SSL config on Linux. TODO: Should this not be included and
            # read from dist folder.
            if loaded_basename == "openssl.cnf":
                continue

            # Taking these from system is harmless and desirable
            if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
                continue

            # System C libraries are to be expected.
            if loaded_basename.startswith(
                (
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
                )
            ):
                continue

            # Loaded by C library potentially for DNS lookups.
            if loaded_basename.startswith(
                (
                    "libnss_",
                    "libnsl",
                    # Some systems load a lot more, this is CentOS 7 on OBS
                    "libattr.so.",
                    "libbz2.so.",
                    "libcap.so.",
                    "libdw.so.",
                    "libelf.so.",
                    "liblzma.so.",
                    # Some systems load a lot more, this is Fedora 26 on OBS
                    "libselinux.so.",
                    "libpcre.so.",
                    # And this is Fedora 29 on OBS
                    "libblkid.so.",
                    "libmount.so.",
                    "libpcre2-8.so.",
                    # CentOS 8 on OBS
                    "libuuid.so.",
                )
            ):
                continue

            # Loaded by dtruss on macOS X.
            if loaded_filename.startswith("/usr/lib/dtrace/"):
                continue

            # Loaded by cowbuilder and pbuilder on Debian
            if loaded_basename == ".ilist":
                continue
            if "cowdancer" in loaded_filename:
                continue
            if "eatmydata" in loaded_filename:
                continue

            # Loading from home directories is OK too.
            if (
                loaded_filename.startswith("/home/")
                or loaded_filename.startswith("/data/")
                or loaded_filename.startswith("/root/")
                or loaded_filename in ("/home", "/data", "/root")
            ):
                continue

            # For Debian builders, /build is OK too.
            if loaded_filename.startswith("/build/") or loaded_filename == "/build":
                continue

            # TODO: Unclear, loading gconv from filesystem of installed system
            # may be OK or not. I think it should be.
            if loaded_basename == "gconv-modules.cache":
                continue
            if "/gconv/" in loaded_filename:
                continue
            if loaded_basename.startswith("libicu"):
                continue
            if loaded_filename.startswith("/usr/share/icu/"):
                continue

            # Loading from caches is OK.
            if loaded_filename.startswith("/var/cache/"):
                continue

            lib_prefix_dir = "/usr/lib/python%d.%s" % (
                python_version[0],
                python_version[1],
            )

            # PySide accesses its directory.
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
                continue

            # GTK accesses package directories only.
            if loaded_filename == os.path.join(
                lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
            ):
                continue
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
                continue
            if loaded_filename == os.path.join(
                lib_prefix_dir, "dist-packages/gtk-2.0/gio"
            ):
                continue
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
                continue

            # PyQt5 seems to do this, but won't use contents then.
            if loaded_filename in (
                "/usr/lib/qt5/plugins",
                "/usr/lib/qt5",
                "/usr/lib64/qt5/plugins",
                "/usr/lib64/qt5",
                "/usr/lib/x86_64-linux-gnu/qt5/plugins",
                "/usr/lib/x86_64-linux-gnu/qt5",
                "/usr/lib/x86_64-linux-gnu",
                "/usr/lib",
            ):
                continue

            # Can look at the interpreter of the system.
            if loaded_basename == "python3":
                continue

            # Current Python executable can actually be a symlink and
            # the real executable which it points to will be on the
            # loaded_filenames list. This is all fine, let's ignore it.
            # Also, because the loaded_filename can be yet another symlink
            # (this is weird, but it's true), let's better resolve its real
            # path too.
            if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
                continue

            # Accessing SE-Linux is OK.
            if loaded_filename in ("/sys/fs/selinux", "/selinux"):
                continue

            # Allow reading time zone info of local system.
            if loaded_filename.startswith("/usr/share/zoneinfo/"):
                continue

            # The access to .pth files has no effect.
            if loaded_filename.endswith(".pth"):
                continue

            # Looking at site-package dir alone is alone.
            if loaded_filename.endswith(("site-packages", "dist-packages")):
                continue

            # QtNetwork insist on doing this it seems.
            if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
                continue

            # macOS uses these:
            if loaded_basename in (
                "libcrypto.1.0.0.dylib",
                "libssl.1.0.0.dylib",
                "libcrypto.1.1.dylib",
            ):
                continue

            # Linux onefile uses this
            if loaded_basename.startswith("libfuse.so."):
                continue

            # MSVC run time DLLs, due to SxS come from system.
            if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
                continue

            test_logger.warning("Should not access '%s'." % loaded_filename)
            illegal_access = True

        if illegal_access:
            displayError(None, filename)
            displayRuntimeTraces(test_logger, binary_filename)

            search_mode.onErrorDetected(1)

        os.unlink(binary_filename)

    search_mode.finish()
Beispiel #5
0
    def action(stage_dir, root, path):
        command = [
            sys.executable,
            os.path.join("..", "..", "bin", "nuitka"),
            "--stand",
            "--run",
            "--output-dir=%s" % stage_dir,
            "--remove-output",
        ]

        filename = os.path.join(stage_dir, "importer.py")

        assert path.startswith(root)

        module_name = path[len(root) + 1:]
        module_name = module_name.split(".")[0]
        module_name = module_name.replace(os.path.sep, ".")

        module_name = ModuleName(module_name)

        with openTextFile(filename, "w") as output:
            plugin_names = set(["pylint-warnings", "anti-bloat"])
            if module_name.hasNamespace("PySide2"):
                plugin_names.add("pyside2")
            elif module_name.hasNamespace("PySide6"):
                plugin_names.add("pyside2")
            elif module_name.hasNamespace("PyQt5"):
                plugin_names.add("pyqt5")
            else:
                # TODO: We do not have a noqt plugin yet.
                plugin_names.add("pyqt5")

            for plugin_name in plugin_names:
                output.write("# nuitka-project: --enable-plugin=%s\n" %
                             plugin_name)

            # Make it an error to find unwanted bloat compiled in.
            output.write("# nuitka-project: --noinclude-default-mode=error\n")

            output.write("import " + module_name.asString() + "\n")
            output.write("print('OK.')")

        command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()

        command.append(filename)

        if checkSucceedsWithCPython(filename):
            try:
                output = check_output(command).splitlines()
            except Exception:  # only trying to check for no exception, pylint: disable=try-except-raise
                raise
            else:
                assert os.path.exists(filename[:-3] + ".dist")

                binary_filename = os.path.join(
                    filename[:-3] + ".dist",
                    "importer.exe" if os.name == "nt" else "importer",
                )
                loaded_filenames = getRuntimeTraceOfLoadedFiles(
                    logger=test_logger,
                    command=[binary_filename],
                )

                outside_accesses = checkLoadedFileAccesses(
                    loaded_filenames=loaded_filenames, current_dir=os.getcwd())

                if outside_accesses:
                    displayError(None, filename)
                    displayRuntimeTraces(test_logger, binary_filename)

                    test_logger.warning(
                        "Should not access these file(s): '%r'." %
                        outside_accesses)

                    search_mode.onErrorDetected(1)

                if output[-1] != b"OK.":
                    my_print(" ".join(command))
                    my_print(filename)
                    my_print(output)
                    test_logger.sysexit("FAIL.")

                my_print("OK.")

                assert not outside_accesses, outside_accesses

                shutil.rmtree(filename[:-3] + ".dist")
        else:
            my_print("SKIP (does not work with CPython)")
Beispiel #6
0
def diffRecursive(dir1, dir2):
    # Complex in nature, pylint: disable=too-many-branches

    done = set()

    result = False

    for path1, filename in listDir(dir1):
        if "cache-" in path1:
            continue

        path2 = os.path.join(dir2, filename)

        done.add(path1)

        # Skip these binary files and scons build database of course.
        # TODO: Temporary ignore ".bin", until we have something better than marshal which behaves
        # differently in compiled Nuitka:
        if filename.endswith((
                ".o",
                ".os",
                ".obj",
                ".dblite",
                ".tmp",
                ".sconsign",
                ".txt",
                ".bin",
                ".const",
                ".exp",
        )):
            continue

        if not os.path.exists(path2):
            test_logger.warning("Only in %s: %s" % (dir1, filename))
            result = False
            continue

        if os.path.isdir(path1):
            r = diffRecursive(path1, path2)
            if r:
                result = True
        elif os.path.isfile(path1):
            fromdate = time.ctime(os.stat(path1).st_mtime)
            todate = time.ctime(os.stat(path2).st_mtime)

            diff = difflib.unified_diff(
                a=readSource(path1).splitlines(),
                b=readSource(path2).splitlines(),
                fromfile=path1,
                tofile=path2,
                fromfiledate=fromdate,
                tofiledate=todate,
                n=3,
            )

            diff_list = list(diff)

            if diff_list:
                for line in diff_list:
                    try:
                        my_print(line)
                    except UnicodeEncodeError:
                        my_print(repr(line))

                result = True
        else:
            assert False, path1

    for path1, filename in listDir(dir2):
        if "cache-" in path1:
            continue

        path2 = os.path.join(dir2, filename)

        if path1 in done:
            continue

        if not os.path.exists(path1):
            test_logger.warning("Only in %s: %s" % (dir2, filename))
            result = False
            continue

    return result