def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = _detectBinaryDLLs(
is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name,
use_cache=use_cache,
update_cache=update_cache,
)
# Allow plugins to prevent inclusion, this may discard things from used_dlls.
Plugins.removeDllDependencies(dll_filename=binary_filename,
dll_filenames=used_dlls)
for dll_filename in sorted(tuple(used_dlls)):
if not os.path.isfile(dll_filename):
if _not_found_dlls:
general.warning("""\
Dependency '%s' could not be found, expect runtime issues. If this is
working with Python, report a Nuitka bug.""" % dll_filename)
_not_found_dlls.add(dll_filename)
used_dlls.remove(dll_filename)
reportProgressBar(binary_filename)
return binary_filename, package_name, used_dlls
def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = detectBinaryDLLs(
is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name,
use_cache=use_cache,
update_cache=update_cache,
)
# Allow plugins to prevent inclusion, this may discard things from used_dlls.
Plugins.removeDllDependencies(dll_filename=binary_filename,
dll_filenames=used_dlls)
for dll_filename in sorted(tuple(used_dlls)):
if not os.path.isfile(dll_filename):
if _unfound_dlls:
general.warning(
"Dependency '%s' could not be found, you might need to copy it manually."
% dll_filename)
_unfound_dlls.add(dll_filename)
used_dlls.remove(dll_filename)
return binary_filename, used_dlls
def _parsePEFileOutput(binary_filename, scan_dirs, result):
pe = _getPEFile(binary_filename)
# Some DLLs (eg numpy) don't have imports
if not hasattr(pe, "DIRECTORY_ENTRY_IMPORT"):
info(
"Warning: no DIRECTORY_ENTRY_IMPORT PE section for library '%s'!" %
binary_filename)
return
# Get DLL imports from PE file
for imported_module in pe.DIRECTORY_ENTRY_IMPORT:
dll_filename = imported_module.dll.decode()
# Try to guess DLL path from scan dirs
for scan_dir in scan_dirs:
try:
guessed_path = os.path.join(scan_dir, dll_filename)
if os.path.isfile(guessed_path):
dll_filename = guessed_path
break
except TypeError:
pass
dll_name = os.path.basename(dll_filename).upper()
# Win API can be assumed.
if dll_name.startswith("API-MS-WIN-") or dll_name.startswith(
"EXT-MS-WIN-"):
continue
if dll_name in _win_dll_whitelist:
continue
# Allow plugins to prevent inclusion.
blocked = Plugins.removeDllDependencies(dll_filename=dll_filename,
dll_filenames=result)
for to_remove in blocked:
result.discard(to_remove)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _detectBinaryPathDLLsLinuxBSD(dll_filename):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None:
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
b"$ORIGIN",
os.path.dirname(sys.executable).encode("utf-8"))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath):
process = subprocess.Popen(args=["ldd", dll_filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _stderr = process.communicate()
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found.
if filename == "not found":
continue
# Do not include kernel specific libraries.
if os.path.basename(filename).startswith(
("libc.so.", "libpthread.so.", "libm.so.", "libdl.so.")):
continue
result.add(filename)
# Allow plugins to prevent inclusion.
blocked = Plugins.removeDllDependencies(dll_filename=dll_filename,
dll_filenames=result)
for to_remove in blocked:
result.discard(to_remove)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsLinuxBSD(sub_dll_filename))
return sub_result
def _parseDependsExeOutput2(lines, result):
inside = False
first = False
for line in lines:
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
dll_filename = line[line.find("]") + 2 :].rstrip()
dll_filename = os.path.normcase(dll_filename)
# Skip DLLs that failed to load, apparently not needed anyway.
if "E" in line[: line.find("]")]:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[: line.find("]")]:
# One exception are PythonXY.DLL
if dll_filename.startswith("python") and dll_filename.endswith(".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64" if getArchitecture() == "x86_64" else "System32",
dll_filename,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
dll_filename = os.path.abspath(dll_filename)
dll_name = os.path.basename(dll_filename)
# Ignore this runtime DLL of Python2.
if dll_name in ("msvcr90.dll",):
continue
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), (dll_filename, line)
# Allow plugins to prevent inclusion. TODO: This should be called with
# only the new ones.
blocked = Plugins.removeDllDependencies(
dll_filename=dll_filename, dll_filenames=result
)
for to_remove in blocked:
result.discard(to_remove)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _detectBinaryPathDLLsPosix(dll_filename):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
b"$ORIGIN", os.path.dirname(sys.executable).encode("utf-8")
)
with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath):
process = subprocess.Popen(
args=["ldd", dll_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout, _stderr = process.communicate()
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[: part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Do not include kernel / glibc specific libraries. This list has been
# assembled by looking what are the most common .so files provided by
# glibc packages from ArchLinux, Debian Stretch and CentOS.
#
# Online sources:
# - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html
# - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html
# - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html
# - https://packages.debian.org/stretch/amd64/libc6/filelist
#
# Note: This list may still be incomplete. Some additional libraries
# might be provided by glibc - it may vary between the package versions
# and between Linux distros. It might or might not be a problem in the
# future, but it should be enough for now.
if os.path.basename(filename).startswith(
(
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)
):
continue
result.add(filename)
# Allow plugins to prevent inclusion.
blocked = Plugins.removeDllDependencies(
dll_filename=dll_filename, dll_filenames=result
)
for to_remove in blocked:
result.discard(to_remove)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(_detectBinaryPathDLLsPosix(sub_dll_filename))
return sub_result
def _parsePEFileOutput(
binary_filename,
scan_dirs,
is_main_executable,
source_dir,
original_dir,
use_cache,
update_cache,
):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches,too-many-locals
result = OrderedSet()
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="pefile",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
# TODO: We are lazy with the format, pylint: disable=eval-used
extracted = eval(getFileContents(cache_filename))
else:
if Options.isShowProgress():
info("Analysing dependencies of '%s'." % binary_filename)
extracted = getPEFileInformation(binary_filename)
if update_cache:
with withFileLock():
with open(cache_filename, "w") as cache_file:
print(repr(extracted), file=cache_file)
# Add native system directory based on pe file architecture and os architecture
# Python 32: system32 = syswow64 = 32 bits systemdirectory
# Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory
# Get DLL imports from PE file
for dll_name in extracted["DLLs"]:
dll_name = dll_name.upper()
# Try determine DLL path from scan dirs
for scan_dir in scan_dirs:
dll_filename = os.path.normcase(
os.path.abspath(os.path.join(scan_dir, dll_name))
)
if os.path.isfile(dll_filename):
break
else:
if dll_name.startswith("API-MS-WIN-") or dll_name.startswith("EXT-MS-WIN-"):
continue
# Found via RC_MANIFEST as copied from Python.
if dll_name == "MSVCR90.DLL":
continue
if dll_name.startswith("python") and dll_name.endswith(".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64" if getArchitecture() == "x86_64" else "System32",
dll_name,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
if dll_filename not in result:
result.add(dll_filename)
# TODO: Shouldn't be here.
blocked = Plugins.removeDllDependencies(
dll_filename=binary_filename, dll_filenames=result
)
for to_remove in blocked:
result.discard(to_remove)
return result
