def addDLLInfo(count, source_dir, original_filename, binary_filename, package_name): used_dlls = _detectBinaryDLLs( is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) # Allow plugins to prevent inclusion, this may discard things from used_dlls. Plugins.removeDllDependencies(dll_filename=binary_filename, dll_filenames=used_dlls) for dll_filename in sorted(tuple(used_dlls)): if not os.path.isfile(dll_filename): if _not_found_dlls: general.warning("""\ Dependency '%s' could not be found, expect runtime issues. If this is working with Python, report a Nuitka bug.""" % dll_filename) _not_found_dlls.add(dll_filename) used_dlls.remove(dll_filename) reportProgressBar(binary_filename) return binary_filename, package_name, used_dlls
def addDLLInfo(count, source_dir, original_filename, binary_filename, package_name): used_dlls = detectBinaryDLLs( is_main_executable=count == 0, source_dir=source_dir, original_filename=original_filename, binary_filename=binary_filename, package_name=package_name, use_cache=use_cache, update_cache=update_cache, ) # Allow plugins to prevent inclusion, this may discard things from used_dlls. Plugins.removeDllDependencies(dll_filename=binary_filename, dll_filenames=used_dlls) for dll_filename in sorted(tuple(used_dlls)): if not os.path.isfile(dll_filename): if _unfound_dlls: general.warning( "Dependency '%s' could not be found, you might need to copy it manually." % dll_filename) _unfound_dlls.add(dll_filename) used_dlls.remove(dll_filename) return binary_filename, used_dlls
def _parsePEFileOutput(binary_filename, scan_dirs, result): pe = _getPEFile(binary_filename) # Some DLLs (eg numpy) don't have imports if not hasattr(pe, "DIRECTORY_ENTRY_IMPORT"): info( "Warning: no DIRECTORY_ENTRY_IMPORT PE section for library '%s'!" % binary_filename) return # Get DLL imports from PE file for imported_module in pe.DIRECTORY_ENTRY_IMPORT: dll_filename = imported_module.dll.decode() # Try to guess DLL path from scan dirs for scan_dir in scan_dirs: try: guessed_path = os.path.join(scan_dir, dll_filename) if os.path.isfile(guessed_path): dll_filename = guessed_path break except TypeError: pass dll_name = os.path.basename(dll_filename).upper() # Win API can be assumed. if dll_name.startswith("API-MS-WIN-") or dll_name.startswith( "EXT-MS-WIN-"): continue if dll_name in _win_dll_whitelist: continue # Allow plugins to prevent inclusion. blocked = Plugins.removeDllDependencies(dll_filename=dll_filename, dll_filenames=result) for to_remove in blocked: result.discard(to_remove) result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _detectBinaryPathDLLsLinuxBSD(dll_filename): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None: _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( b"$ORIGIN", os.path.dirname(sys.executable).encode("utf-8")) with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath): process = subprocess.Popen(args=["ldd", dll_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _stderr = process.communicate() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 300: filename = filename.decode("utf-8") # Sometimes might use stuff not found. if filename == "not found": continue # Do not include kernel specific libraries. if os.path.basename(filename).startswith( ("libc.so.", "libpthread.so.", "libm.so.", "libdl.so.")): continue result.add(filename) # Allow plugins to prevent inclusion. blocked = Plugins.removeDllDependencies(dll_filename=dll_filename, dll_filenames=result) for to_remove in blocked: result.discard(to_remove) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsLinuxBSD(sub_dll_filename)) return sub_result
def _parseDependsExeOutput2(lines, result): inside = False first = False for line in lines: if "| Module Dependency Tree |" in line: inside = True first = True continue if not inside: continue if "| Module List |" in line: break if "]" not in line: continue dll_filename = line[line.find("]") + 2 :].rstrip() dll_filename = os.path.normcase(dll_filename) # Skip DLLs that failed to load, apparently not needed anyway. if "E" in line[: line.find("]")]: continue # Skip missing DLLs, apparently not needed anyway. if "?" in line[: line.find("]")]: # One exception are PythonXY.DLL if dll_filename.startswith("python") and dll_filename.endswith(".dll"): dll_filename = os.path.join( os.environ["SYSTEMROOT"], "SysWOW64" if getArchitecture() == "x86_64" else "System32", dll_filename, ) dll_filename = os.path.normcase(dll_filename) else: continue dll_filename = os.path.abspath(dll_filename) dll_name = os.path.basename(dll_filename) # Ignore this runtime DLL of Python2. if dll_name in ("msvcr90.dll",): continue # The executable itself is of course exempted. We cannot check its path # because depends.exe mistreats unicode paths. if first: first = False continue assert os.path.isfile(dll_filename), (dll_filename, line) # Allow plugins to prevent inclusion. TODO: This should be called with # only the new ones. blocked = Plugins.removeDllDependencies( dll_filename=dll_filename, dll_filenames=result ) for to_remove in blocked: result.discard(to_remove) result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _detectBinaryPathDLLsPosix(dll_filename): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None and not Utils.isPosixWindows(): _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( b"$ORIGIN", os.path.dirname(sys.executable).encode("utf-8") ) with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath): process = subprocess.Popen( args=["ldd", dll_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, _stderr = process.communicate() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[: part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 300: filename = filename.decode("utf-8") # Sometimes might use stuff not found or supplied by ldd itself. if filename in ("not found", "ldd"): continue # Do not include kernel / glibc specific libraries. This list has been # assembled by looking what are the most common .so files provided by # glibc packages from ArchLinux, Debian Stretch and CentOS. # # Online sources: # - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html # - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html # - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html # - https://packages.debian.org/stretch/amd64/libc6/filelist # # Note: This list may still be incomplete. Some additional libraries # might be provided by glibc - it may vary between the package versions # and between Linux distros. It might or might not be a problem in the # future, but it should be enough for now. if os.path.basename(filename).startswith( ( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", ) ): continue result.add(filename) # Allow plugins to prevent inclusion. blocked = Plugins.removeDllDependencies( dll_filename=dll_filename, dll_filenames=result ) for to_remove in blocked: result.discard(to_remove) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union(_detectBinaryPathDLLsPosix(sub_dll_filename)) return sub_result
def _parsePEFileOutput( binary_filename, scan_dirs, is_main_executable, source_dir, original_dir, use_cache, update_cache, ): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches,too-many-locals result = OrderedSet() if use_cache or update_cache: cache_filename = _getCacheFilename( dependency_tool="pefile", is_main_executable=is_main_executable, source_dir=source_dir, original_dir=original_dir, binary_filename=binary_filename, ) if use_cache: with withFileLock(): if not os.path.exists(cache_filename): use_cache = False if use_cache: # TODO: We are lazy with the format, pylint: disable=eval-used extracted = eval(getFileContents(cache_filename)) else: if Options.isShowProgress(): info("Analysing dependencies of '%s'." % binary_filename) extracted = getPEFileInformation(binary_filename) if update_cache: with withFileLock(): with open(cache_filename, "w") as cache_file: print(repr(extracted), file=cache_file) # Add native system directory based on pe file architecture and os architecture # Python 32: system32 = syswow64 = 32 bits systemdirectory # Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory # Get DLL imports from PE file for dll_name in extracted["DLLs"]: dll_name = dll_name.upper() # Try determine DLL path from scan dirs for scan_dir in scan_dirs: dll_filename = os.path.normcase( os.path.abspath(os.path.join(scan_dir, dll_name)) ) if os.path.isfile(dll_filename): break else: if dll_name.startswith("API-MS-WIN-") or dll_name.startswith("EXT-MS-WIN-"): continue # Found via RC_MANIFEST as copied from Python. if dll_name == "MSVCR90.DLL": continue if dll_name.startswith("python") and dll_name.endswith(".dll"): dll_filename = os.path.join( os.environ["SYSTEMROOT"], "SysWOW64" if getArchitecture() == "x86_64" else "System32", dll_name, ) dll_filename = os.path.normcase(dll_filename) else: continue if dll_filename not in result: result.add(dll_filename) # TODO: Shouldn't be here. blocked = Plugins.removeDllDependencies( dll_filename=binary_filename, dll_filenames=result ) for to_remove in blocked: result.discard(to_remove) return result