def executePASS4(): test_logger.info("PASS 4: Compiling the compiler running from single exe.") exe_path = os.path.join(tmp_dir, "nuitka" + exe_suffix) compileAndCompareWith(exe_path, pass_number=4) test_logger.info("OK.")
def displayError(dirname, filename): assert dirname is None filename = filename[:-3] + ".dist" test_logger.info("Listing of dist folder '%s':" % filename) if os.name == "nt": command = "dir /b /s /a:-D %s" % filename else: command = "ls -Rla %s" % filename os.system(command)
def executePASS2(): test_logger.info( "PASS 2: Compiling from compiler running from entry '.exe' and many extension files." ) # Windows will load the compiled modules (pyd) only from PYTHONPATH, so we # have to add it. if os.name == "nt": os.environ["PYTHONPATH"] = ":".join(PACKAGE_LIST) compileAndCompareWith(nuitka=os.path.join(".", "nuitka" + exe_suffix), pass_number=2) # Undo the damage from above. if os.name == "nt": del os.environ["PYTHONPATH"] test_logger.info("OK.")
def executePASS3(): test_logger.info( "PASS 3: Compiling from compiler running from .py files to single .exe." ) exe_path = os.path.join(tmp_dir, "nuitka" + exe_suffix) if os.path.exists(exe_path): os.unlink(exe_path) build_path = os.path.join(tmp_dir, "nuitka.build") if os.path.exists(build_path): shutil.rmtree(build_path) path = os.path.join("..", "..", "bin", "nuitka") _traceCompilation(path=path, pass_number=3) command = [ os.environ["PYTHON"], nuitka_main_path, path, "--output-dir=%s" % tmp_dir, "--python-flag=-S", "--follow-imports", # "--include-package=nuitka.plugins", ] my_print("Command: ", " ".join(command)) result = subprocess.call(command) if result != 0: sys.exit(result) shutil.rmtree(build_path) test_logger.info("OK.")
def main(): python_version = setup(suite="onefile", needs_io_encoding=True) search_mode = createSearchMode() if getOS() == "Linux": addExtendedExtraOptions( "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg" ) for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "remove_output", # Keep the binary, normally "remove_output" includes that. "--keep-binary", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", # The onefile can warn about zstandard not being installed. "ignore_warnings", ] if filename == "KeyboardInterruptTest.py": if getOS() == "Darwin": reportSkip( "Exit code from KeyboardInterrupt on macOS is not yet good.", ".", filename, ) continue if python_version < (3,): reportSkip( "Python2 reports KeyboardInterrupt, but too late", ".", filename, ) continue if os.name == "nt": reportSkip( "Testing cannot send KeyboardInterrupt on Windows yet", ".", filename, ) continue extra_flags.append("--send-ctrl-c") # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue test_logger.info( "Consider output of onefile mode compiled program: %s" % filename ) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin") if filename == "KeyboardInterruptTest.py": continue # Then use "strace" on the result. with TimerReport( "Determining run time loaded files took %.2f", logger=test_logger ): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, command=[binary_filename] ) illegal_accesses = checkLoadedFileAccesses( loaded_filenames=loaded_filenames, current_dir=os.getcwd() ) if illegal_accesses: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) test_logger.warning( "Should not access these file(s): '%s'." % ",".join(illegal_accesses) ) search_mode.onErrorDetected(1) os.unlink(binary_filename) search_mode.finish()
def main(): # Complex stuff, even more should become common code or project options though. # pylint: disable=too-many-branches,too-many-statements python_version = setup(suite="standalone", needs_io_encoding=True) search_mode = createSearchMode() for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "--standalone", "remove_output", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", ] # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue if filename == "Urllib3Using.py" and os.name == "nt": reportSkip( "Socket module early import not working on Windows currently", ".", filename, ) continue if "Idna" in filename: # For the warnings of Python2. if python_version < (3, ): extra_flags.append("ignore_stderr") if filename == "CtypesUsing.py": extra_flags.append("plugin_disable:pylint-warnings") if filename == "GtkUsing.py": # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7): reportSkip("irrelevant Python version", ".", filename) continue # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith("Win"): if os.name != "nt": reportSkip("Windows only test", ".", filename) continue if filename == "TkInterUsing.py": if getOS() == "Darwin": reportSkip("Not working macOS yet", ".", filename) continue if getOS() == "Windows": reportSkip("Can hang on Windows CI.", ".", filename) continue # For the plug-in information. extra_flags.append("plugin_enable:tk-inter") if filename == "FlaskUsing.py": # For the warnings. extra_flags.append("ignore_warnings") # TODO: Once we have a no-Qt Plugin, we should use that. if filename == "MatplotlibUsing.py": # For the plugin warnings. extra_flags.append("ignore_warnings") if filename == "NumpyUsing.py": # TODO: Disabled for now. reportSkip("numpy.test not fully working yet", ".", filename) continue if filename == "PandasUsing.py": extra_flags.append("plugin_enable:numpy") extra_flags.append("plugin_disable:pylint-warnings") extra_flags.append("plugin_disable:pyqt5") extra_flags.append("plugin_disable:pyside2") extra_flags.append("plugin_disable:pyside6") if filename == "PmwUsing.py": extra_flags.append("plugin_enable:pmw-freezer") if filename == "OpenGLUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "GlfwUsing.py": # For the warnings. extra_flags.append("plugin_enable:numpy") if filename == "PasslibUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "Win32ComUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")): # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)): reportSkip("irrelevant Python version", ".", filename) continue # For the plug-in information if filename.startswith("PySide2"): extra_flags.append("plugin_enable:pyside6") elif filename.startswith("PySide6"): extra_flags.append("plugin_enable:pyside6") elif filename.startswith("PyQt5"): extra_flags.append("plugin_enable:pyqt5") elif filename.startswith("PyQt6"): extra_flags.append("plugin_enable:pyqt6") test_logger.info( "Consider output of standalone mode compiled program: %s" % filename) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) # Second check if glibc libraries haven't been accidentally # shipped with the standalone executable found_glibc_libs = [] for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")): if os.path.basename(dist_filename).startswith(( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", )): found_glibc_libs.append(dist_filename) if found_glibc_libs: test_logger.warning( "Should not ship glibc libraries with the standalone executable (found %s)" % found_glibc_libs) sys.exit(1) binary_filename = os.path.join( filename[:-3] + ".dist", filename[:-3] + (".exe" if os.name == "nt" else "")) # Then use "strace" on the result. with TimerReport("Determining run time loaded files took %.2f", logger=test_logger): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, command=[binary_filename]) illegal_accesses = checkLoadedFileAccesses( loaded_filenames=loaded_filenames, current_dir=os.getcwd()) if illegal_accesses: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) test_logger.warning("Should not access these file(s): '%r'." % illegal_accesses) search_mode.onErrorDetected(1) removeDirectory(filename[:-3] + ".dist", ignore_errors=True) search_mode.finish()
def main(): # Complex stuff, even more should become common code though. # pylint: disable=too-many-branches,too-many-locals,too-many-statements python_version = setup(needs_io_encoding=True) search_mode = createSearchMode() for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "--standalone", "remove_output", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", ] # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue # catch error if filename == "Boto3Using.py": reportSkip("boto3 test not fully working yet", ".", filename) continue if "Idna" in filename: # For the warnings of Python2. if python_version < (3,): extra_flags.append("ignore_stderr") if filename == "CtypesUsing.py": extra_flags.append("plugin_disable:pylint-warnings") if filename == "GtkUsing.py": # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7): reportSkip("irrelevant Python version", ".", filename) continue # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith("Win"): if os.name != "nt": reportSkip("Windows only test", ".", filename) continue if filename == "TkInterUsing.py": if getOS() == "Darwin": reportSkip("Not working macOS yet", ".", filename) continue # For the plug-in information. extra_flags.append("plugin_enable:tk-inter") if filename == "FlaskUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "NumpyUsing.py": extra_flags.append("plugin_enable:numpy") # TODO: Disabled for now. reportSkip("numpy.test not fully working yet", ".", filename) continue if filename == "PandasUsing.py": extra_flags.append("plugin_enable:numpy") extra_flags.append("plugin_disable:pylint-warnings") extra_flags.append("plugin_disable:qt-plugins") if filename == "PmwUsing.py": extra_flags.append("plugin_enable:pmw-freezer") if filename == "OpenGLUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "PasslibUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "PySideUsing.py": # TODO: Disabled due to lack of upstream support. reportSkip("PySide not supported yet", ".", filename) continue if filename == "Win32ComUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith(("PySide", "PyQt")): # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7): reportSkip("irrelevant Python version", ".", filename) continue # For the plug-in information. if getPythonVendor() != "Anaconda": extra_flags.append("plugin_enable:qt-plugins") else: # For the plug-in not used information. extra_flags.append("ignore_warnings") test_logger.info( "Consider output of standalone mode compiled program: %s" % filename ) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) # Second check if glibc libraries haven't been accidentally # shipped with the standalone executable found_glibc_libs = [] for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")): if os.path.basename(dist_filename).startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): found_glibc_libs.append(dist_filename) if found_glibc_libs: test_logger.warning( "Should not ship glibc libraries with the standalone executable (found %s)" % found_glibc_libs ) sys.exit(1) binary_filename = os.path.join( filename[:-3] + ".dist", filename[:-3] + (".exe" if os.name == "nt" else "") ) # Then use "strace" on the result. with TimerReport( "Determining run time loaded files took %.2f", logger=test_logger ): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, path=binary_filename ) current_dir = os.path.normpath(os.getcwd()) current_dir = os.path.normcase(current_dir) illegal_access = False for loaded_filename in loaded_filenames: loaded_filename = os.path.normpath(loaded_filename) loaded_filename = os.path.normcase(loaded_filename) loaded_basename = os.path.basename(loaded_filename) if os.name == "nt": if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "System32") ), ): continue if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "SysWOW64") ), ): continue if r"windows\winsxs" in loaded_filename: continue # Github actions have these in PATH overriding SYSTEMROOT if r"windows performance toolkit" in loaded_filename: continue if r"powershell" in loaded_filename: continue if r"azure dev spaces cli" in loaded_filename: continue if loaded_filename.startswith(current_dir): continue if loaded_filename.startswith(os.path.abspath(current_dir)): continue if loaded_filename.startswith("/etc/"): continue if loaded_filename.startswith("/usr/etc/"): continue if loaded_filename.startswith("/proc/") or loaded_filename == "/proc": continue if loaded_filename.startswith("/dev/"): continue if loaded_filename.startswith("/tmp/"): continue if loaded_filename.startswith("/run/"): continue if loaded_filename.startswith("/usr/lib/locale/"): continue if loaded_filename.startswith("/usr/share/locale/"): continue if loaded_filename.startswith("/usr/share/X11/locale/"): continue # Themes may of course be loaded. if loaded_filename.startswith("/usr/share/themes"): continue if "gtk" in loaded_filename and "/engines/" in loaded_filename: continue if loaded_filename in ( "/usr", "/usr/local", "/usr/local/lib", "/usr/share", "/usr/local/share", "/usr/lib64", ): continue # TCL/tk for tkinter for non-Windows is OK. if loaded_filename.startswith( ( "/usr/lib/tcltk/", "/usr/share/tcltk/", "/usr/lib/tcl/", "/usr/lib64/tcl/", ) ): continue if loaded_filename in ( "/usr/lib/tcltk", "/usr/share/tcltk", "/usr/lib/tcl", "/usr/lib64/tcl", ): continue if loaded_filename in ( "/lib", "/lib64", "/lib/sse2", "/lib/tls", "/lib64/tls", "/usr/lib/sse2", "/usr/lib/tls", "/usr/lib64/tls", ): continue if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"): continue if loaded_filename in ( "/usr/share/tcl8.6/init.tcl", "/usr/share/tcl8.5/init.tcl", ): continue if loaded_filename in ( "/usr/share/tcl8.6/encoding", "/usr/share/tcl8.5/encoding", ): continue # System SSL config on Linux. TODO: Should this not be included and # read from dist folder. if loaded_basename == "openssl.cnf": continue # Taking these from system is harmless and desirable if loaded_basename.startswith(("libz.so", "libgcc_s.so")): continue # System C libraries are to be expected. if loaded_basename.startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): continue # Loaded by C library potentially for DNS lookups. if loaded_basename.startswith( ( "libnss_", "libnsl", # Some systems load a lot more, this is CentOS 7 on OBS "libattr.so.", "libbz2.so.", "libcap.so.", "libdw.so.", "libelf.so.", "liblzma.so.", # Some systems load a lot more, this is Fedora 26 on OBS "libselinux.so.", "libpcre.so.", # And this is Fedora 29 on OBS "libblkid.so.", "libmount.so.", "libpcre2-8.so.", # CentOS 8 on OBS "libuuid.so.", ) ): continue # Loaded by dtruss on macOS X. if loaded_filename.startswith("/usr/lib/dtrace/"): continue # Loaded by cowbuilder and pbuilder on Debian if loaded_basename == ".ilist": continue if "cowdancer" in loaded_filename: continue if "eatmydata" in loaded_filename: continue # Loading from home directories is OK too. if ( loaded_filename.startswith("/home/") or loaded_filename.startswith("/data/") or loaded_filename.startswith("/root/") or loaded_filename in ("/home", "/data", "/root") ): continue # For Debian builders, /build is OK too. if loaded_filename.startswith("/build/") or loaded_filename == "/build": continue # TODO: Unclear, loading gconv from filesystem of installed system # may be OK or not. I think it should be. if loaded_basename == "gconv-modules.cache": continue if "/gconv/" in loaded_filename: continue if loaded_basename.startswith("libicu"): continue if loaded_filename.startswith("/usr/share/icu/"): continue # Loading from caches is OK. if loaded_filename.startswith("/var/cache/"): continue lib_prefix_dir = "/usr/lib/python%d.%s" % ( python_version[0], python_version[1], ) # PySide accesses its directory. if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"): continue # GTK accesses package directories only. if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gtk" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"): continue if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gio" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"): continue # PyQt5 seems to do this, but won't use contents then. if loaded_filename in ( "/usr/lib/qt5/plugins", "/usr/lib/qt5", "/usr/lib64/qt5/plugins", "/usr/lib64/qt5", "/usr/lib/x86_64-linux-gnu/qt5/plugins", "/usr/lib/x86_64-linux-gnu/qt5", "/usr/lib/x86_64-linux-gnu", "/usr/lib", ): continue # Can look at the interpreters of the system. if loaded_basename in "python3": continue if loaded_basename in ( "python%s" + supported_version for supported_version in ( getSupportedPythonVersions() + getPartiallySupportedPythonVersions() ) ): continue # Current Python executable can actually be a symlink and # the real executable which it points to will be on the # loaded_filenames list. This is all fine, let's ignore it. # Also, because the loaded_filename can be yet another symlink # (this is weird, but it's true), let's better resolve its real # path too. if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable): continue # Accessing SE-Linux is OK. if loaded_filename in ("/sys/fs/selinux", "/selinux"): continue # Looking at device is OK. if loaded_filename.startswith("/sys/devices/"): continue # Allow reading time zone info of local system. if loaded_filename.startswith("/usr/share/zoneinfo/"): continue # The access to .pth files has no effect. if loaded_filename.endswith(".pth"): continue # Looking at site-package dir alone is alone. if loaded_filename.endswith(("site-packages", "dist-packages")): continue # QtNetwork insist on doing this it seems. if loaded_basename.startswith(("libcrypto.so", "libssl.so")): continue # macOS uses these: if loaded_basename in ( "libcrypto.1.0.0.dylib", "libssl.1.0.0.dylib", "libcrypto.1.1.dylib", ): continue # MSVC run time DLLs, seem to sometimes come from system. if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"): continue test_logger.warning("Should not access '%s'." % loaded_filename) illegal_access = True if illegal_access: if os.name != "nt": displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) search_mode.onErrorDetected(1) removeDirectory(filename[:-3] + ".dist", ignore_errors=True) if search_mode.abortIfExecuted(): break search_mode.finish()
def main(): # Complex stuff, even more should become common code though. # pylint: disable=too-many-branches,too-many-statements python_version = setup(needs_io_encoding=True) search_mode = createSearchMode() if getOS() == "Linux": addExtendedExtraOptions( "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg" ) for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "--onefile", "remove_output", # Keep the binary, normally "remove_output" includes that. "--keep-binary", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", ] if filename == "KeyboardInteruptTest.py": if python_version < (3,): reportSkip( "Python2 reports KeyboardInterrupt, but too late", ".", filename, ) continue if os.name == "nt": reportSkip( ".", filename, "Testing cannot send KeyboardInterrupt on Windows yet", ) continue extra_flags.append("--send-ctrl-c") # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue test_logger.info( "Consider output of onefile mode compiled program: %s" % filename ) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin") if filename == "KeyboardInteruptTest.py": continue # Then use "strace" on the result. with TimerReport( "Determining run time loaded files took %.2f", logger=test_logger ): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, path=binary_filename ) current_dir = os.path.normpath(os.getcwd()) current_dir = os.path.normcase(current_dir) illegal_access = False for loaded_filename in loaded_filenames: loaded_filename = os.path.normpath(loaded_filename) loaded_filename = os.path.normcase(loaded_filename) loaded_basename = os.path.basename(loaded_filename) if os.name == "nt": if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "System32") ), ): continue if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "SysWOW64") ), ): continue if r"windows\winsxs" in loaded_filename: continue # Github actions have these in PATH overriding SYSTEMROOT if r"windows performance toolkit" in loaded_filename: continue if r"powershell" in loaded_filename: continue if r"azure dev spaces cli" in loaded_filename: continue if r"tortoisesvn" in loaded_filename: continue if loaded_filename.startswith(current_dir): continue if loaded_filename.startswith(os.path.abspath(current_dir)): continue if loaded_filename.startswith("/etc/"): continue if loaded_filename.startswith("/proc/") or loaded_filename == "/proc": continue if loaded_filename.startswith("/dev/"): continue if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp": continue if loaded_filename.startswith("/run/"): continue if loaded_filename.startswith("/usr/lib/locale/"): continue if loaded_filename.startswith("/usr/share/locale/"): continue if loaded_filename.startswith("/usr/share/X11/locale/"): continue # Themes may of course be loaded. if loaded_filename.startswith("/usr/share/themes"): continue if "gtk" in loaded_filename and "/engines/" in loaded_filename: continue if loaded_filename in ( "/usr", "/usr/local", "/usr/local/lib", "/usr/share", "/usr/local/share", "/usr/lib64", ): continue # TCL/tk for tkinter for non-Windows is OK. if loaded_filename.startswith( ( "/usr/lib/tcltk/", "/usr/share/tcltk/", "/usr/lib/tcl/", "/usr/lib64/tcl/", ) ): continue if loaded_filename in ( "/usr/lib/tcltk", "/usr/share/tcltk", "/usr/lib/tcl", "/usr/lib64/tcl", ): continue if loaded_filename in ( "/lib", "/lib64", "/lib/sse2", "/lib/tls", "/lib64/tls", "/usr/lib/sse2", "/usr/lib/tls", "/usr/lib64/tls", ): continue if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"): continue if loaded_filename in ( "/usr/share/tcl8.6/init.tcl", "/usr/share/tcl8.5/init.tcl", ): continue if loaded_filename in ( "/usr/share/tcl8.6/encoding", "/usr/share/tcl8.5/encoding", ): continue # System SSL config on Linux. TODO: Should this not be included and # read from dist folder. if loaded_basename == "openssl.cnf": continue # Taking these from system is harmless and desirable if loaded_basename.startswith(("libz.so", "libgcc_s.so")): continue # System C libraries are to be expected. if loaded_basename.startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): continue # Loaded by C library potentially for DNS lookups. if loaded_basename.startswith( ( "libnss_", "libnsl", # Some systems load a lot more, this is CentOS 7 on OBS "libattr.so.", "libbz2.so.", "libcap.so.", "libdw.so.", "libelf.so.", "liblzma.so.", # Some systems load a lot more, this is Fedora 26 on OBS "libselinux.so.", "libpcre.so.", # And this is Fedora 29 on OBS "libblkid.so.", "libmount.so.", "libpcre2-8.so.", # CentOS 8 on OBS "libuuid.so.", ) ): continue # Loaded by dtruss on macOS X. if loaded_filename.startswith("/usr/lib/dtrace/"): continue # Loaded by cowbuilder and pbuilder on Debian if loaded_basename == ".ilist": continue if "cowdancer" in loaded_filename: continue if "eatmydata" in loaded_filename: continue # Loading from home directories is OK too. if ( loaded_filename.startswith("/home/") or loaded_filename.startswith("/data/") or loaded_filename.startswith("/root/") or loaded_filename in ("/home", "/data", "/root") ): continue # For Debian builders, /build is OK too. if loaded_filename.startswith("/build/") or loaded_filename == "/build": continue # TODO: Unclear, loading gconv from filesystem of installed system # may be OK or not. I think it should be. if loaded_basename == "gconv-modules.cache": continue if "/gconv/" in loaded_filename: continue if loaded_basename.startswith("libicu"): continue if loaded_filename.startswith("/usr/share/icu/"): continue # Loading from caches is OK. if loaded_filename.startswith("/var/cache/"): continue lib_prefix_dir = "/usr/lib/python%d.%s" % ( python_version[0], python_version[1], ) # PySide accesses its directory. if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"): continue # GTK accesses package directories only. if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gtk" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"): continue if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gio" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"): continue # PyQt5 seems to do this, but won't use contents then. if loaded_filename in ( "/usr/lib/qt5/plugins", "/usr/lib/qt5", "/usr/lib64/qt5/plugins", "/usr/lib64/qt5", "/usr/lib/x86_64-linux-gnu/qt5/plugins", "/usr/lib/x86_64-linux-gnu/qt5", "/usr/lib/x86_64-linux-gnu", "/usr/lib", ): continue # Can look at the interpreter of the system. if loaded_basename == "python3": continue # Current Python executable can actually be a symlink and # the real executable which it points to will be on the # loaded_filenames list. This is all fine, let's ignore it. # Also, because the loaded_filename can be yet another symlink # (this is weird, but it's true), let's better resolve its real # path too. if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable): continue # Accessing SE-Linux is OK. if loaded_filename in ("/sys/fs/selinux", "/selinux"): continue # Allow reading time zone info of local system. if loaded_filename.startswith("/usr/share/zoneinfo/"): continue # The access to .pth files has no effect. if loaded_filename.endswith(".pth"): continue # Looking at site-package dir alone is alone. if loaded_filename.endswith(("site-packages", "dist-packages")): continue # QtNetwork insist on doing this it seems. if loaded_basename.startswith(("libcrypto.so", "libssl.so")): continue # macOS uses these: if loaded_basename in ( "libcrypto.1.0.0.dylib", "libssl.1.0.0.dylib", "libcrypto.1.1.dylib", ): continue # Linux onefile uses this if loaded_basename.startswith("libfuse.so."): continue # MSVC run time DLLs, due to SxS come from system. if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"): continue test_logger.warning("Should not access '%s'." % loaded_filename) illegal_access = True if illegal_access: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) search_mode.onErrorDetected(1) os.unlink(binary_filename) search_mode.finish()
def executePASS1(): test_logger.info( "PASS 1: Compiling to many compiled modules from compiler running from .py files." ) base_dir = os.path.join("..", "..") for package in PACKAGE_LIST: package = package.replace("/", os.path.sep) source_dir = os.path.join(base_dir, package) target_dir = package removeDirectory(path=target_dir, ignore_errors=False) os.mkdir(target_dir) for path, filename in listDir(target_dir): if filename.endswith((".so", ".dylib")): os.unlink(path) for path, filename in listDir(source_dir): if not filename.endswith(".py"): continue if filename.startswith(".#"): continue if filename != "__init__.py": _traceCompilation(path=path, pass_number=1) command = [ os.environ["PYTHON"], nuitka_main_path, "--module", "--nofollow-imports", "--output-dir=%s" % target_dir, "--no-pyi-file", path, ] command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() my_print("Command: ", " ".join(command)) result = subprocess.call(command) if result != 0: sys.exit(result) else: shutil.copyfile(path, os.path.join(target_dir, filename)) _traceCompilation(path=nuitka_main_path, pass_number=1) shutil.copyfile(nuitka_main_path, "nuitka-runner.py") command = [ os.environ["PYTHON"], nuitka_main_path, "--nofollow-imports", "--plugin-enable=pylint-warnings", "--output-dir=.", "--python-flag=-S", "nuitka-runner.py", ] command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() my_print("Command: ", " ".join(command)) result = subprocess.call(command) if result != 0: sys.exit(result) shutil.move("nuitka-runner" + exe_suffix, "nuitka" + exe_suffix) scons_inline_copy_path = os.path.join(base_dir, "nuitka", "build", "inline_copy") if os.path.exists(scons_inline_copy_path): copyTree(scons_inline_copy_path, os.path.join("nuitka", "build", "inline_copy")) # Copy required data files. for filename in ( "nuitka/build/Backend.scons", "nuitka/plugins/standard/anti-bloat.yml", "nuitka/plugins/standard/implicit-imports.yml", "nuitka/plugins/standard/data-files.yml", ): shutil.copyfile( os.path.join(base_dir, filename), filename, ) copyTree( os.path.join(base_dir, "nuitka", "codegen", "templates_c"), os.path.join("nuitka", "codegen", "templates_c"), ) copyTree( os.path.join(base_dir, "nuitka", "build", "static_src"), os.path.join("nuitka", "build", "static_src"), ) copyTree( os.path.join(base_dir, "nuitka", "build", "include"), os.path.join("nuitka", "build", "include"), ) # The data composer tool, use it by source. copyTree( os.path.join(base_dir, "nuitka", "tools"), os.path.join("nuitka", "tools"), ) test_logger.info("OK.")
def _traceCompilation(path, pass_number): test_logger.info("Compiling '%s' (PASS %d)." % (path, pass_number))