def detectBinaryDLLs( is_main_executable, source_dir, original_filename, binary_filename, package_name, use_cache, update_cache, ): """ Detect the DLLs used by a binary. Using "ldd" (Linux), "pefile" or "depends.exe" (Windows), or "otool" (macOS) the list of used DLLs is retrieved. """ if ( Utils.getOS() in ("Linux", "NetBSD", "FreeBSD", "OpenBSD") or Utils.isPosixWindows() ): return _detectBinaryPathDLLsPosix(dll_filename=original_filename) elif Utils.isWin32Windows() and Options.getWindowsDependencyTool() == "pefile": with TimerReport( "Finding dependencies for %s took %%.2f seconds" % binary_filename ): return detectBinaryPathDLLsWindowsPE( is_main_executable=is_main_executable, source_dir=source_dir, original_dir=os.path.dirname(original_filename), binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) elif Utils.isWin32Windows(): with TimerReport( "Running depends.exe for %s took %%.2f seconds" % binary_filename ): return detectBinaryPathDLLsWindowsDependencyWalker( is_main_executable=is_main_executable, source_dir=source_dir, original_dir=os.path.dirname(original_filename), binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) elif Utils.getOS() == "Darwin": return _detectBinaryPathDLLsMacOS( original_dir=os.path.dirname(original_filename), binary_filename=original_filename, keep_unresolved=False, ) else: # Support your platform above. assert False, Utils.getOS()
def detectBinaryDLLs(is_main_executable, source_dir, original_filename, binary_filename, package_name): """ Detect the DLLs used by a binary. Using "ldd" (Linux), "depends.exe" (Windows), or "otool" (MacOS) the list of used DLLs is retrieved. """ if Utils.getOS() in ("Linux", "NetBSD", "FreeBSD"): return _detectBinaryPathDLLsLinuxBSD(dll_filename=original_filename) elif Utils.getOS() == "Windows": with TimerReport("Running depends.exe for %s took %%.2f seconds" % binary_filename): return _detectBinaryPathDLLsWindows( is_main_executable=is_main_executable, source_dir=source_dir, original_dir=os.path.dirname(original_filename), binary_filename=binary_filename, package_name=package_name) elif Utils.getOS() == "Darwin": return _detectBinaryPathDLLsMacOS( original_dir=os.path.dirname(original_filename), binary_filename=original_filename) else: # Support your platform above. assert False, Utils.getOS()
def __init__(self, spawn, *args): threading.Thread.__init__(self) self.spawn = spawn self.args = args self.timer_report = TimerReport( message="Running %s took %%.2f seconds" % (repr(self.args).replace("%", "%%"), ), min_report_time=60, logger=scons_logger, ) self.result = None
def __init__(self, *args): threading.Thread.__init__(self) self.args = args self.timer_report = TimerReport( message="Running %s took %%.2f seconds" % (" ".join(_unescape(arg) for arg in self.args[2]).replace("%", "%%"), ), min_report_time=60, logger=scons_logger, ) self.result = None self.exception = None
def _detectBinaryDLLs( is_main_executable, source_dir, original_filename, binary_filename, package_name, use_cache, update_cache, ): """Detect the DLLs used by a binary. Using "ldd" (Linux), "depends.exe" (Windows), or "otool" (macOS) the list of used DLLs is retrieved. """ if (Utils.getOS() in ("Linux", "NetBSD", "FreeBSD", "OpenBSD") or Utils.isPosixWindows()): return _detectBinaryPathDLLsPosix( dll_filename=original_filename, package_name=package_name, original_dir=os.path.dirname(original_filename), ) elif Utils.isWin32Windows(): with TimerReport( message="Running 'depends.exe' for %s took %%.2f seconds" % binary_filename, decider=Options.isShowProgress, ): return detectBinaryPathDLLsWindowsDependencyWalker( is_main_executable=is_main_executable, source_dir=source_dir, original_dir=os.path.dirname(original_filename), binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) elif Utils.isMacOS(): return _detectBinaryPathDLLsMacOS( original_dir=os.path.dirname(original_filename), binary_filename=original_filename, package_name=package_name, keep_unresolved=False, recursive=True, ) else: # Support your platform above. assert False, Utils.getOS()
def __init__(self, cmdline, env): threading.Thread.__init__(self) self.cmdline = cmdline self.env = env self.data = None self.err = None self.exit_code = None self.timer_report = TimerReport( message="Running %s took %%.2f seconds" % repr(self.cmdline).replace("%", "%%"), min_report_time=60, logger=scons_logger, )
def mergeMultipleBranches(self, collections): assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work. if len(collections) == 1: self.replaceBranch(collections[0]) return None # print("Enter mergeMultipleBranches", len(collections)) with TimerReport( message="Running merge for %s took %%.2f seconds" % collections, decider=lambda: 0, ): variable_versions = defaultdict(OrderedSet) for collection in collections: for variable, version in iterItems( collection.variable_actives): variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: (version, ) = versions else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=tuple( self.getVariableTrace(variable, version) for version in versions), ) self.markCurrentVariableTrace(variable, version)
def main(): parser = OptionParser() _options, positional_args = parser.parse_args() if not positional_args: sys.exit("No DLLs given.") for filename in positional_args: my_print("Filename: %s" % filename) my_print("Version Information: %s" % getWindowsDLLVersion(filename)) my_print("SXS information (manifests):") sxs = getSxsFromDLL(filename=filename, with_data=True) if sxs: my_print(sxs) my_print("DLLs recursively dependended (depends.exe):") with TimerReport( message="Finding dependencies for %s took %%.2f seconds" % filename): r = detectBinaryPathDLLsWindowsDependencyWalker( is_main_executable=False, source_dir="notexist", original_dir=os.path.dirname(filename), binary_filename=filename, package_name=None, use_cache=False, update_cache=False, ) for dll_filename in sorted(r): my_print(" %s" % dll_filename) my_print("Total: %d" % len(r))
def executeSubTest(command, hide_output=False): with TimerReport(message="Overall execution of %r took %%.2f seconds" % command): _executeSubTest(command, hide_output)
def main(): python_version = setup(suite="onefile", needs_io_encoding=True) search_mode = createSearchMode() if getOS() == "Linux": addExtendedExtraOptions( "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg" ) for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "remove_output", # Keep the binary, normally "remove_output" includes that. "--keep-binary", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", # The onefile can warn about zstandard not being installed. "ignore_warnings", ] if filename == "KeyboardInterruptTest.py": if getOS() == "Darwin": reportSkip( "Exit code from KeyboardInterrupt on macOS is not yet good.", ".", filename, ) continue if python_version < (3,): reportSkip( "Python2 reports KeyboardInterrupt, but too late", ".", filename, ) continue if os.name == "nt": reportSkip( "Testing cannot send KeyboardInterrupt on Windows yet", ".", filename, ) continue extra_flags.append("--send-ctrl-c") # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue test_logger.info( "Consider output of onefile mode compiled program: %s" % filename ) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin") if filename == "KeyboardInterruptTest.py": continue # Then use "strace" on the result. with TimerReport( "Determining run time loaded files took %.2f", logger=test_logger ): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, command=[binary_filename] ) illegal_accesses = checkLoadedFileAccesses( loaded_filenames=loaded_filenames, current_dir=os.getcwd() ) if illegal_accesses: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) test_logger.warning( "Should not access these file(s): '%s'." % ",".join(illegal_accesses) ) search_mode.onErrorDetected(1) os.unlink(binary_filename) search_mode.finish()
def main(): # Complex stuff, even more should become common code or project options though. # pylint: disable=too-many-branches,too-many-statements python_version = setup(suite="standalone", needs_io_encoding=True) search_mode = createSearchMode() for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "--standalone", "remove_output", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", ] # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue if filename == "Urllib3Using.py" and os.name == "nt": reportSkip( "Socket module early import not working on Windows currently", ".", filename, ) continue if "Idna" in filename: # For the warnings of Python2. if python_version < (3, ): extra_flags.append("ignore_stderr") if filename == "CtypesUsing.py": extra_flags.append("plugin_disable:pylint-warnings") if filename == "GtkUsing.py": # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7): reportSkip("irrelevant Python version", ".", filename) continue # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith("Win"): if os.name != "nt": reportSkip("Windows only test", ".", filename) continue if filename == "TkInterUsing.py": if getOS() == "Darwin": reportSkip("Not working macOS yet", ".", filename) continue if getOS() == "Windows": reportSkip("Can hang on Windows CI.", ".", filename) continue # For the plug-in information. extra_flags.append("plugin_enable:tk-inter") if filename == "FlaskUsing.py": # For the warnings. extra_flags.append("ignore_warnings") # TODO: Once we have a no-Qt Plugin, we should use that. if filename == "MatplotlibUsing.py": # For the plugin warnings. extra_flags.append("ignore_warnings") if filename == "NumpyUsing.py": # TODO: Disabled for now. reportSkip("numpy.test not fully working yet", ".", filename) continue if filename == "PandasUsing.py": extra_flags.append("plugin_enable:numpy") extra_flags.append("plugin_disable:pylint-warnings") extra_flags.append("plugin_disable:pyqt5") extra_flags.append("plugin_disable:pyside2") extra_flags.append("plugin_disable:pyside6") if filename == "PmwUsing.py": extra_flags.append("plugin_enable:pmw-freezer") if filename == "OpenGLUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "GlfwUsing.py": # For the warnings. extra_flags.append("plugin_enable:numpy") if filename == "PasslibUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "Win32ComUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")): # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)): reportSkip("irrelevant Python version", ".", filename) continue # For the plug-in information if filename.startswith("PySide2"): extra_flags.append("plugin_enable:pyside6") elif filename.startswith("PySide6"): extra_flags.append("plugin_enable:pyside6") elif filename.startswith("PyQt5"): extra_flags.append("plugin_enable:pyqt5") elif filename.startswith("PyQt6"): extra_flags.append("plugin_enable:pyqt6") test_logger.info( "Consider output of standalone mode compiled program: %s" % filename) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) # Second check if glibc libraries haven't been accidentally # shipped with the standalone executable found_glibc_libs = [] for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")): if os.path.basename(dist_filename).startswith(( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", )): found_glibc_libs.append(dist_filename) if found_glibc_libs: test_logger.warning( "Should not ship glibc libraries with the standalone executable (found %s)" % found_glibc_libs) sys.exit(1) binary_filename = os.path.join( filename[:-3] + ".dist", filename[:-3] + (".exe" if os.name == "nt" else "")) # Then use "strace" on the result. with TimerReport("Determining run time loaded files took %.2f", logger=test_logger): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, command=[binary_filename]) illegal_accesses = checkLoadedFileAccesses( loaded_filenames=loaded_filenames, current_dir=os.getcwd()) if illegal_accesses: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) test_logger.warning("Should not access these file(s): '%r'." % illegal_accesses) search_mode.onErrorDetected(1) removeDirectory(filename[:-3] + ".dist", ignore_errors=True) search_mode.finish()
def main(): # Complex stuff, even more should become common code though. # pylint: disable=too-many-branches,too-many-locals,too-many-statements python_version = setup(needs_io_encoding=True) search_mode = createSearchMode() for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "--standalone", "remove_output", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", ] # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue # catch error if filename == "Boto3Using.py": reportSkip("boto3 test not fully working yet", ".", filename) continue if "Idna" in filename: # For the warnings of Python2. if python_version < (3,): extra_flags.append("ignore_stderr") if filename == "CtypesUsing.py": extra_flags.append("plugin_disable:pylint-warnings") if filename == "GtkUsing.py": # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7): reportSkip("irrelevant Python version", ".", filename) continue # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith("Win"): if os.name != "nt": reportSkip("Windows only test", ".", filename) continue if filename == "TkInterUsing.py": if getOS() == "Darwin": reportSkip("Not working macOS yet", ".", filename) continue # For the plug-in information. extra_flags.append("plugin_enable:tk-inter") if filename == "FlaskUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "NumpyUsing.py": extra_flags.append("plugin_enable:numpy") # TODO: Disabled for now. reportSkip("numpy.test not fully working yet", ".", filename) continue if filename == "PandasUsing.py": extra_flags.append("plugin_enable:numpy") extra_flags.append("plugin_disable:pylint-warnings") extra_flags.append("plugin_disable:qt-plugins") if filename == "PmwUsing.py": extra_flags.append("plugin_enable:pmw-freezer") if filename == "OpenGLUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "PasslibUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename == "PySideUsing.py": # TODO: Disabled due to lack of upstream support. reportSkip("PySide not supported yet", ".", filename) continue if filename == "Win32ComUsing.py": # For the warnings. extra_flags.append("ignore_warnings") if filename.startswith(("PySide", "PyQt")): # Don't test on platforms not supported by current Debian testing, and # which should be considered irrelevant by now. if python_version < (2, 7): reportSkip("irrelevant Python version", ".", filename) continue # For the plug-in information. if getPythonVendor() != "Anaconda": extra_flags.append("plugin_enable:qt-plugins") else: # For the plug-in not used information. extra_flags.append("ignore_warnings") test_logger.info( "Consider output of standalone mode compiled program: %s" % filename ) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) # Second check if glibc libraries haven't been accidentally # shipped with the standalone executable found_glibc_libs = [] for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")): if os.path.basename(dist_filename).startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): found_glibc_libs.append(dist_filename) if found_glibc_libs: test_logger.warning( "Should not ship glibc libraries with the standalone executable (found %s)" % found_glibc_libs ) sys.exit(1) binary_filename = os.path.join( filename[:-3] + ".dist", filename[:-3] + (".exe" if os.name == "nt" else "") ) # Then use "strace" on the result. with TimerReport( "Determining run time loaded files took %.2f", logger=test_logger ): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, path=binary_filename ) current_dir = os.path.normpath(os.getcwd()) current_dir = os.path.normcase(current_dir) illegal_access = False for loaded_filename in loaded_filenames: loaded_filename = os.path.normpath(loaded_filename) loaded_filename = os.path.normcase(loaded_filename) loaded_basename = os.path.basename(loaded_filename) if os.name == "nt": if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "System32") ), ): continue if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "SysWOW64") ), ): continue if r"windows\winsxs" in loaded_filename: continue # Github actions have these in PATH overriding SYSTEMROOT if r"windows performance toolkit" in loaded_filename: continue if r"powershell" in loaded_filename: continue if r"azure dev spaces cli" in loaded_filename: continue if loaded_filename.startswith(current_dir): continue if loaded_filename.startswith(os.path.abspath(current_dir)): continue if loaded_filename.startswith("/etc/"): continue if loaded_filename.startswith("/usr/etc/"): continue if loaded_filename.startswith("/proc/") or loaded_filename == "/proc": continue if loaded_filename.startswith("/dev/"): continue if loaded_filename.startswith("/tmp/"): continue if loaded_filename.startswith("/run/"): continue if loaded_filename.startswith("/usr/lib/locale/"): continue if loaded_filename.startswith("/usr/share/locale/"): continue if loaded_filename.startswith("/usr/share/X11/locale/"): continue # Themes may of course be loaded. if loaded_filename.startswith("/usr/share/themes"): continue if "gtk" in loaded_filename and "/engines/" in loaded_filename: continue if loaded_filename in ( "/usr", "/usr/local", "/usr/local/lib", "/usr/share", "/usr/local/share", "/usr/lib64", ): continue # TCL/tk for tkinter for non-Windows is OK. if loaded_filename.startswith( ( "/usr/lib/tcltk/", "/usr/share/tcltk/", "/usr/lib/tcl/", "/usr/lib64/tcl/", ) ): continue if loaded_filename in ( "/usr/lib/tcltk", "/usr/share/tcltk", "/usr/lib/tcl", "/usr/lib64/tcl", ): continue if loaded_filename in ( "/lib", "/lib64", "/lib/sse2", "/lib/tls", "/lib64/tls", "/usr/lib/sse2", "/usr/lib/tls", "/usr/lib64/tls", ): continue if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"): continue if loaded_filename in ( "/usr/share/tcl8.6/init.tcl", "/usr/share/tcl8.5/init.tcl", ): continue if loaded_filename in ( "/usr/share/tcl8.6/encoding", "/usr/share/tcl8.5/encoding", ): continue # System SSL config on Linux. TODO: Should this not be included and # read from dist folder. if loaded_basename == "openssl.cnf": continue # Taking these from system is harmless and desirable if loaded_basename.startswith(("libz.so", "libgcc_s.so")): continue # System C libraries are to be expected. if loaded_basename.startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): continue # Loaded by C library potentially for DNS lookups. if loaded_basename.startswith( ( "libnss_", "libnsl", # Some systems load a lot more, this is CentOS 7 on OBS "libattr.so.", "libbz2.so.", "libcap.so.", "libdw.so.", "libelf.so.", "liblzma.so.", # Some systems load a lot more, this is Fedora 26 on OBS "libselinux.so.", "libpcre.so.", # And this is Fedora 29 on OBS "libblkid.so.", "libmount.so.", "libpcre2-8.so.", # CentOS 8 on OBS "libuuid.so.", ) ): continue # Loaded by dtruss on macOS X. if loaded_filename.startswith("/usr/lib/dtrace/"): continue # Loaded by cowbuilder and pbuilder on Debian if loaded_basename == ".ilist": continue if "cowdancer" in loaded_filename: continue if "eatmydata" in loaded_filename: continue # Loading from home directories is OK too. if ( loaded_filename.startswith("/home/") or loaded_filename.startswith("/data/") or loaded_filename.startswith("/root/") or loaded_filename in ("/home", "/data", "/root") ): continue # For Debian builders, /build is OK too. if loaded_filename.startswith("/build/") or loaded_filename == "/build": continue # TODO: Unclear, loading gconv from filesystem of installed system # may be OK or not. I think it should be. if loaded_basename == "gconv-modules.cache": continue if "/gconv/" in loaded_filename: continue if loaded_basename.startswith("libicu"): continue if loaded_filename.startswith("/usr/share/icu/"): continue # Loading from caches is OK. if loaded_filename.startswith("/var/cache/"): continue lib_prefix_dir = "/usr/lib/python%d.%s" % ( python_version[0], python_version[1], ) # PySide accesses its directory. if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"): continue # GTK accesses package directories only. if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gtk" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"): continue if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gio" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"): continue # PyQt5 seems to do this, but won't use contents then. if loaded_filename in ( "/usr/lib/qt5/plugins", "/usr/lib/qt5", "/usr/lib64/qt5/plugins", "/usr/lib64/qt5", "/usr/lib/x86_64-linux-gnu/qt5/plugins", "/usr/lib/x86_64-linux-gnu/qt5", "/usr/lib/x86_64-linux-gnu", "/usr/lib", ): continue # Can look at the interpreters of the system. if loaded_basename in "python3": continue if loaded_basename in ( "python%s" + supported_version for supported_version in ( getSupportedPythonVersions() + getPartiallySupportedPythonVersions() ) ): continue # Current Python executable can actually be a symlink and # the real executable which it points to will be on the # loaded_filenames list. This is all fine, let's ignore it. # Also, because the loaded_filename can be yet another symlink # (this is weird, but it's true), let's better resolve its real # path too. if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable): continue # Accessing SE-Linux is OK. if loaded_filename in ("/sys/fs/selinux", "/selinux"): continue # Looking at device is OK. if loaded_filename.startswith("/sys/devices/"): continue # Allow reading time zone info of local system. if loaded_filename.startswith("/usr/share/zoneinfo/"): continue # The access to .pth files has no effect. if loaded_filename.endswith(".pth"): continue # Looking at site-package dir alone is alone. if loaded_filename.endswith(("site-packages", "dist-packages")): continue # QtNetwork insist on doing this it seems. if loaded_basename.startswith(("libcrypto.so", "libssl.so")): continue # macOS uses these: if loaded_basename in ( "libcrypto.1.0.0.dylib", "libssl.1.0.0.dylib", "libcrypto.1.1.dylib", ): continue # MSVC run time DLLs, seem to sometimes come from system. if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"): continue test_logger.warning("Should not access '%s'." % loaded_filename) illegal_access = True if illegal_access: if os.name != "nt": displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) search_mode.onErrorDetected(1) removeDirectory(filename[:-3] + ".dist", ignore_errors=True) if search_mode.abortIfExecuted(): break search_mode.finish()
def main(): # Complex stuff, even more should become common code though. # pylint: disable=too-many-branches,too-many-statements python_version = setup(needs_io_encoding=True) search_mode = createSearchMode() if getOS() == "Linux": addExtendedExtraOptions( "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg" ) for filename in sorted(os.listdir(".")): if not filename.endswith(".py"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if not active: test_logger.info("Skipping %s" % filename) continue extra_flags = [ "expect_success", "--onefile", "remove_output", # Keep the binary, normally "remove_output" includes that. "--keep-binary", # Cache the CPython results for re-use, they will normally not change. "cpython_cache", # To understand what is slow. "timing", ] if filename == "KeyboardInteruptTest.py": if python_version < (3,): reportSkip( "Python2 reports KeyboardInterrupt, but too late", ".", filename, ) continue if os.name == "nt": reportSkip( ".", filename, "Testing cannot send KeyboardInterrupt on Windows yet", ) continue extra_flags.append("--send-ctrl-c") # skip each test if their respective requirements are not met requirements_met, error_message = checkRequirements(filename) if not requirements_met: reportSkip(error_message, ".", filename) continue test_logger.info( "Consider output of onefile mode compiled program: %s" % filename ) # First compare so we know the program behaves identical. compareWithCPython( dirname=None, filename=filename, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, on_error=displayError, ) binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin") if filename == "KeyboardInteruptTest.py": continue # Then use "strace" on the result. with TimerReport( "Determining run time loaded files took %.2f", logger=test_logger ): loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, path=binary_filename ) current_dir = os.path.normpath(os.getcwd()) current_dir = os.path.normcase(current_dir) illegal_access = False for loaded_filename in loaded_filenames: loaded_filename = os.path.normpath(loaded_filename) loaded_filename = os.path.normcase(loaded_filename) loaded_basename = os.path.basename(loaded_filename) if os.name == "nt": if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "System32") ), ): continue if areSamePaths( os.path.dirname(loaded_filename), os.path.normpath( os.path.join(os.environ["SYSTEMROOT"], "SysWOW64") ), ): continue if r"windows\winsxs" in loaded_filename: continue # Github actions have these in PATH overriding SYSTEMROOT if r"windows performance toolkit" in loaded_filename: continue if r"powershell" in loaded_filename: continue if r"azure dev spaces cli" in loaded_filename: continue if r"tortoisesvn" in loaded_filename: continue if loaded_filename.startswith(current_dir): continue if loaded_filename.startswith(os.path.abspath(current_dir)): continue if loaded_filename.startswith("/etc/"): continue if loaded_filename.startswith("/proc/") or loaded_filename == "/proc": continue if loaded_filename.startswith("/dev/"): continue if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp": continue if loaded_filename.startswith("/run/"): continue if loaded_filename.startswith("/usr/lib/locale/"): continue if loaded_filename.startswith("/usr/share/locale/"): continue if loaded_filename.startswith("/usr/share/X11/locale/"): continue # Themes may of course be loaded. if loaded_filename.startswith("/usr/share/themes"): continue if "gtk" in loaded_filename and "/engines/" in loaded_filename: continue if loaded_filename in ( "/usr", "/usr/local", "/usr/local/lib", "/usr/share", "/usr/local/share", "/usr/lib64", ): continue # TCL/tk for tkinter for non-Windows is OK. if loaded_filename.startswith( ( "/usr/lib/tcltk/", "/usr/share/tcltk/", "/usr/lib/tcl/", "/usr/lib64/tcl/", ) ): continue if loaded_filename in ( "/usr/lib/tcltk", "/usr/share/tcltk", "/usr/lib/tcl", "/usr/lib64/tcl", ): continue if loaded_filename in ( "/lib", "/lib64", "/lib/sse2", "/lib/tls", "/lib64/tls", "/usr/lib/sse2", "/usr/lib/tls", "/usr/lib64/tls", ): continue if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"): continue if loaded_filename in ( "/usr/share/tcl8.6/init.tcl", "/usr/share/tcl8.5/init.tcl", ): continue if loaded_filename in ( "/usr/share/tcl8.6/encoding", "/usr/share/tcl8.5/encoding", ): continue # System SSL config on Linux. TODO: Should this not be included and # read from dist folder. if loaded_basename == "openssl.cnf": continue # Taking these from system is harmless and desirable if loaded_basename.startswith(("libz.so", "libgcc_s.so")): continue # System C libraries are to be expected. if loaded_basename.startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): continue # Loaded by C library potentially for DNS lookups. if loaded_basename.startswith( ( "libnss_", "libnsl", # Some systems load a lot more, this is CentOS 7 on OBS "libattr.so.", "libbz2.so.", "libcap.so.", "libdw.so.", "libelf.so.", "liblzma.so.", # Some systems load a lot more, this is Fedora 26 on OBS "libselinux.so.", "libpcre.so.", # And this is Fedora 29 on OBS "libblkid.so.", "libmount.so.", "libpcre2-8.so.", # CentOS 8 on OBS "libuuid.so.", ) ): continue # Loaded by dtruss on macOS X. if loaded_filename.startswith("/usr/lib/dtrace/"): continue # Loaded by cowbuilder and pbuilder on Debian if loaded_basename == ".ilist": continue if "cowdancer" in loaded_filename: continue if "eatmydata" in loaded_filename: continue # Loading from home directories is OK too. if ( loaded_filename.startswith("/home/") or loaded_filename.startswith("/data/") or loaded_filename.startswith("/root/") or loaded_filename in ("/home", "/data", "/root") ): continue # For Debian builders, /build is OK too. if loaded_filename.startswith("/build/") or loaded_filename == "/build": continue # TODO: Unclear, loading gconv from filesystem of installed system # may be OK or not. I think it should be. if loaded_basename == "gconv-modules.cache": continue if "/gconv/" in loaded_filename: continue if loaded_basename.startswith("libicu"): continue if loaded_filename.startswith("/usr/share/icu/"): continue # Loading from caches is OK. if loaded_filename.startswith("/var/cache/"): continue lib_prefix_dir = "/usr/lib/python%d.%s" % ( python_version[0], python_version[1], ) # PySide accesses its directory. if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"): continue # GTK accesses package directories only. if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gtk" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"): continue if loaded_filename == os.path.join( lib_prefix_dir, "dist-packages/gtk-2.0/gio" ): continue if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"): continue # PyQt5 seems to do this, but won't use contents then. if loaded_filename in ( "/usr/lib/qt5/plugins", "/usr/lib/qt5", "/usr/lib64/qt5/plugins", "/usr/lib64/qt5", "/usr/lib/x86_64-linux-gnu/qt5/plugins", "/usr/lib/x86_64-linux-gnu/qt5", "/usr/lib/x86_64-linux-gnu", "/usr/lib", ): continue # Can look at the interpreter of the system. if loaded_basename == "python3": continue # Current Python executable can actually be a symlink and # the real executable which it points to will be on the # loaded_filenames list. This is all fine, let's ignore it. # Also, because the loaded_filename can be yet another symlink # (this is weird, but it's true), let's better resolve its real # path too. if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable): continue # Accessing SE-Linux is OK. if loaded_filename in ("/sys/fs/selinux", "/selinux"): continue # Allow reading time zone info of local system. if loaded_filename.startswith("/usr/share/zoneinfo/"): continue # The access to .pth files has no effect. if loaded_filename.endswith(".pth"): continue # Looking at site-package dir alone is alone. if loaded_filename.endswith(("site-packages", "dist-packages")): continue # QtNetwork insist on doing this it seems. if loaded_basename.startswith(("libcrypto.so", "libssl.so")): continue # macOS uses these: if loaded_basename in ( "libcrypto.1.0.0.dylib", "libssl.1.0.0.dylib", "libcrypto.1.1.dylib", ): continue # Linux onefile uses this if loaded_basename.startswith("libfuse.so."): continue # MSVC run time DLLs, due to SxS come from system. if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"): continue test_logger.warning("Should not access '%s'." % loaded_filename) illegal_access = True if illegal_access: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) search_mode.onErrorDetected(1) os.unlink(binary_filename) search_mode.finish()
def mergeMultipleBranches(self, collections): # This one is really complex, pylint: disable=too-many-branches assert collections # Optimize for length 1, which is trivial merge and needs not a # lot of work, and length 2 has dedicated code as it's so frequent. merge_size = len(collections) if merge_size == 1: self.replaceBranch(collections[0]) return elif merge_size == 2: return self.mergeBranches(*collections) # print("Enter mergeMultipleBranches", len(collections)) with TimerReport( message="Running merge for %s took %%.2f seconds" % collections, decider=lambda: 0, ): variable_versions = defaultdict(OrderedSet) for collection in collections: for variable, version in iterItems( collection.variable_actives): variable_versions[variable].add(version) for collection in collections: for variable, versions in iterItems(variable_versions): if variable not in collection.variable_actives: versions.add(0) self.variable_actives = {} for variable, versions in iterItems(variable_versions): if len(versions) == 1: (version, ) = versions else: traces = [] escaped = [] winner_version = None for version in versions: trace = self.getVariableTrace(variable, version) if trace.isEscapeTrace(): winner_version = version escaped_trace = trace.previous if escaped_trace in traces: traces.remove(trace.previous) escaped.append(escaped) traces.append(trace) else: if trace not in escaped: traces.append(trace) if len(traces) == 1: version = winner_version assert winner_version is not None else: version = self.addVariableMergeMultipleTrace( variable=variable, traces=tuple(traces)) self.markCurrentVariableTrace(variable, version)
def main(): parser = OptionParser() parser.add_option( "--verbose", action="store_true", dest="verbose", default=False, help="""\ Be verbose in output. Default is %default.""", ) parser.add_option( "--pefile", action="store_true", dest="pefile", default=False, help="""\ Use pefile dependencies. Default is %default.""", ) options, positional_args = parser.parse_args() if not positional_args: sys.exit("No DLLs given.") for filename in positional_args: print("Filename:", filename) print("Version Information:", getWindowsDLLVersion(filename)) print("DLLs directly dependended (pefile):", getPEFileInformation(filename)) print("SXS information (manifests):") sxs = getSxsFromDLL(filename=filename, with_data=True) if sxs: print(sxs) print("DLLs recursively dependended (pefile):") if options.pefile: with TimerReport( message="Finding dependencies for %s took %%.2f seconds" % filename): from nuitka import Options Options.options.dependency_tool = "pefile" r = detectBinaryPathDLLsWindowsPE( is_main_executable=False, source_dir="notexist", original_dir=os.path.dirname(filename), binary_filename=filename, package_name=None, use_cache=False, update_cache=True, ) Options.options.dependency_tool = "depends.exe" for dll_filename in sorted(r): print(" ", dll_filename) print("Total: %d" % len(r)) print("DLLs recursively dependended (depends.exe):") with TimerReport( message="Finding dependencies for %s took %%.2f seconds" % filename): r = detectBinaryPathDLLsWindowsDependencyWalker( is_main_executable=False, source_dir="notexist", original_dir=os.path.dirname(filename), binary_filename=filename, package_name=None, use_cache=False, update_cache=False, ) for dll_filename in sorted(r): print(" ", dll_filename) print("Total: %d" % len(r))