def getScanDirectories(package_name, original_dir):
# Many cases, pylint: disable=too-many-branches
cache_key = package_name, original_dir
if cache_key in _scan_dir_cache:
return _scan_dir_cache[cache_key]
scan_dirs = [sys.prefix]
if package_name is not None:
scan_dirs.extend(_getPackageSpecificDLLDirectories(package_name))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
if (Utils.isWin32Windows() and package_name is not None
and package_name.isBelowNamespace("win32com")):
pywin32_dir = getPyWin32Dir()
if pywin32_dir is not None:
scan_dirs.append(pywin32_dir)
for path_dir in os.environ["PATH"].split(";"):
if not os.path.isdir(path_dir):
continue
if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "System32")):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
continue
scan_dirs.append(path_dir)
result = []
# Remove directories that hold no DLLs.
for scan_dir in scan_dirs:
sys.stdout.flush()
# These are useless, but plenty.
if os.path.basename(scan_dir) == "__pycache__":
continue
scan_dir = getDirectoryRealPath(scan_dir)
# No DLLs, no use.
if not any(entry[1].lower().endswith(".dll")
for entry in listDir(scan_dir)):
continue
result.append(os.path.realpath(scan_dir))
_scan_dir_cache[cache_key] = result
return result
def getScanDirectories(package_name, original_dir):
cache_key = package_name, original_dir
if cache_key in _scan_dir_cache:
return _scan_dir_cache[cache_key]
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
for path_dir in os.environ["PATH"].split(";"):
if not os.path.isdir(path_dir):
continue
if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "System32")):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
continue
scan_dirs.append(path_dir)
result = []
# Remove directories that hold no DLLs.
for scan_dir in scan_dirs:
sys.stdout.flush()
# These are useless, but plenty.
if os.path.basename(scan_dir) == "__pycache__":
continue
# No DLLs, no use.
if not any(entry[1].lower().endswith(".dll")
for entry in listDir(scan_dir)):
continue
result.append(scan_dir)
_scan_dir_cache[cache_key] = result
return result
def getUncompiledModule(module_name, module_filename):
for uncompiled_module in uncompiled_modules:
if module_name == uncompiled_module.getFullName():
if areSamePaths(
_normalizeModuleFilename(module_filename),
_normalizeModuleFilename(uncompiled_module.filename)):
return uncompiled_module
return None
def getUncompiledModule(module_name, module_filename):
for uncompiled_module in uncompiled_modules:
if module_name == uncompiled_module.getFullName():
if areSamePaths(
_normalizeModuleFilename(module_filename),
_normalizeModuleFilename(uncompiled_module.filename),
):
return uncompiled_module
return None
def runCodespell(filenames, verbose, write):
if verbose:
my_print("Consider", " ".join(filenames))
command = [
"codespell",
"-f",
"-I",
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"misc/codespell-ignore.txt",
),
]
if write:
command.append("-w")
command += filenames
if os.name == "nt":
extra_path = os.path.join(sys.prefix, "Scripts")
else:
extra_path = None
with withEnvironmentPathAdded("PATH", extra_path):
result = subprocess.call(command)
if result == 0:
for filename in filenames:
if areSamePaths(__file__, filename):
continue
contents = getFileContents(filename)
old_contents = contents
for word, replacement in replacements:
contents = contents.replace(word, replacement)
contents = contents.replace(word.title(), replacement.title())
if old_contents != contents:
putTextFileContents(filename, contents)
cleanupWindowsNewlines(filename)
if verbose:
if result != 0:
my_print("FAILED.")
else:
my_print("OK.")
return result == 0
def runOnefileCompressor(
compressor_python, dist_dir, onefile_output_filename, start_binary
):
if compressor_python is None:
from nuitka.tools.onefile_compressor.OnefileCompressor import (
attachOnefilePayload,
)
attachOnefilePayload(
dist_dir=dist_dir,
onefile_output_filename=onefile_output_filename,
start_binary=start_binary,
expect_compression=False,
)
elif areSamePaths(compressor_python.getPythonExe(), sys.executable):
from nuitka.tools.onefile_compressor.OnefileCompressor import (
attachOnefilePayload,
)
attachOnefilePayload(
dist_dir=dist_dir,
onefile_output_filename=onefile_output_filename,
start_binary=start_binary,
expect_compression=True,
)
else:
onefile_compressor_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), "..", "tools", "onefile_compressor")
)
mapping = {
"NUITKA_PACKAGE_HOME": os.path.dirname(
os.path.abspath(sys.modules["nuitka"].__path__[0])
)
}
mapping["NUITKA_PROGRESS_BAR"] = "1" if Options.shallUseProgressBar() else "0"
with withEnvironmentVarsOverridden(mapping):
subprocess.check_call(
[
compressor_python.getPythonExe(),
onefile_compressor_path,
dist_dir,
onefile_output_filename,
start_binary,
str(onefile_compressor_path is not None),
],
shell=False,
)
def consider(self, dirname, filename):
parts = [dirname, filename]
while None in parts:
parts.remove(None)
assert parts
path = os.path.join(*parts)
if self.active:
open(self.cache_filename, 'w').write(path)
return True
if areSamePaths(path, self.resume_from):
self.active = True
return self.active
def consider(self, dirname, filename):
parts = [dirname, filename]
while None in parts:
parts.remove(None)
assert parts
path = os.path.join(*parts)
if self.active:
putTextFileContents(self.cache_filename, contents=path)
return True
if areSamePaths(path, self.resume_from):
self.active = True
return self.active
def consider(self, dirname, filename):
parts = [dirname, filename]
while None in parts:
parts.remove(None)
assert parts
path = os.path.join(*parts)
if self.active:
with open(self.cache_filename, "w") as f:
f.write(path)
return True
if areSamePaths(path, self.resume_from):
self.active = True
return self.active
def _resolveBinaryPathDLLsMacOS(original_dir, binary_filename, paths,
package_specific_dirs):
had_self = False
result = OrderedDict()
rpaths = _detectBinaryRPathsMacOS(original_dir, binary_filename)
rpaths.update(package_specific_dirs)
for path in paths:
if path.startswith("@rpath/"):
# Resolve rpath to just the ones given, first match.
for rpath in rpaths:
if os.path.exists(os.path.join(rpath, path[7:])):
resolved_path = os.path.normpath(
os.path.join(rpath, path[7:]))
break
else:
# This is only a guess, might be missing package specific directories.
resolved_path = os.path.join(original_dir, path[7:])
elif path.startswith("@loader_path/"):
resolved_path = os.path.join(original_dir, path[13:])
elif os.path.basename(path) == os.path.basename(binary_filename):
# We ignore the references to itself coming from the library id.
continue
else:
resolved_path = path
if not os.path.exists(resolved_path):
inclusion_logger.sysexit(
"Error, failed to resolve DLL path %s (for %s), please report the bug."
% (path, binary_filename))
# Some libraries depend on themselves.
if areSamePaths(binary_filename, resolved_path):
had_self = True
continue
result[resolved_path] = path
return had_self, result
def _match(cls, dirname, filename, candidate):
# Circular dependency.
from .Common import getStartDir
parts = [dirname, filename]
while None in parts:
parts.remove(None)
assert parts
path = os.path.join(*parts)
candidates = (
dirname,
filename,
filename.rsplit(".", 1)[0],
filename.rsplit(".", 1)[0].replace("Test", ""),
path,
path.rsplit(".", 1)[0],
path.rsplit(".", 1)[0].replace("Test", ""),
)
return candidate.rstrip("/") in candidates or areSamePaths(
os.path.join(getStartDir(), candidate), filename)
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
my_print("Skipping", filename)
continue
extra_flags = [
"expect_success",
"standalone",
"remove_output",
# For enum plugin info
"ignore_infos",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
# catch error
if filename == "Boto3Using.py":
reportSkip("boto3 test not fully working yet", ".", filename)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version.startswith("2"):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
# extra_flags.append("plugin_enable:data-files")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PySideUsing.py":
# TODO: Disabled due to lack of upstream support.
reportSkip("PySide not supported yet", ".", filename)
continue
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide", "PyQt")):
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information.
if getPythonVendor() != "Anaconda":
extra_flags.append("plugin_enable:qt-plugins")
else:
# For the plug-in not used information.
extra_flags.append("ignore_warnings")
my_print("Consider output of recursively compiled program:", filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
my_print(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"System32")),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64")),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith(
"/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/"):
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith((
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith((
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith(
"/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
# PySide accesses its directory.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/PySide"):
continue
# GTK accesses package directories only.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gtk"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/glib"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gio"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(
sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# MSVC run time DLLs, seem to sometimes come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
my_print("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
my_print("Listing of dist folder:")
os.system("ls -Rla %s" % filename[:-3] + ".dist")
my_print("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
if search_mode.abortIfExecuted():
break
search_mode.finish()
loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "System32")),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
if getOS() == "Linux":
addExtendedExtraOptions(
"--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--onefile",
"remove_output",
# Keep the binary, normally "remove_output" includes that.
"--keep-binary",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
if filename == "KeyboardInteruptTest.py":
if python_version < (3,):
reportSkip(
"Python2 reports KeyboardInterrupt, but too late",
".",
filename,
)
continue
if os.name == "nt":
reportSkip(
".",
filename,
"Testing cannot send KeyboardInterrupt on Windows yet",
)
continue
extra_flags.append("--send-ctrl-c")
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
test_logger.info(
"Consider output of onefile mode compiled program: %s" % filename
)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")
if filename == "KeyboardInteruptTest.py":
continue
# Then use "strace" on the result.
with TimerReport(
"Determining run time loaded files took %.2f", logger=test_logger
):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, path=binary_filename
)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "System32")
),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if r"azure dev spaces cli" in loaded_filename:
continue
if r"tortoisesvn" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp":
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith(
(
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)
):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith(
(
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)
):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith(
(
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)
):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (
loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")
):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith("/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
lib_prefix_dir = "/usr/lib/python%d.%s" % (
python_version[0],
python_version[1],
)
# PySide accesses its directory.
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
continue
# GTK accesses package directories only.
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
continue
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gio"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# Linux onefile uses this
if loaded_basename.startswith("libfuse.so."):
continue
# MSVC run time DLLs, due to SxS come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
test_logger.warning("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
search_mode.onErrorDetected(1)
os.unlink(binary_filename)
search_mode.finish()
def _injectCcache(env, cc_path, python_prefix, target_arch,
assume_yes_for_downloads):
ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if ccache_binary is None:
ccache_binary = getExecutablePath("ccache", env=env)
if ccache_binary is None:
for candidate in _getCcacheGuessedPaths(python_prefix):
scons_details_logger.info(
"Checking if ccache is at '%s' guessed path." % candidate)
if os.path.exists(candidate):
ccache_binary = candidate
scons_details_logger.info(
"Using ccache '%s' from guessed path." % ccache_binary)
break
if ccache_binary is None:
if isWin32Windows():
url = "https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12-windows-32.zip"
ccache_binary = getCachedDownload(
url=url,
is_arch_specific=False,
specificity=url.rsplit("/", 2)[1],
flatten=True,
binary="ccache.exe",
message=
"Nuitka will make use of ccache to speed up repeated compilation.",
reject=None,
assume_yes_for_downloads=assume_yes_for_downloads,
)
elif isMacOS():
# TODO: Do not yet have M1 access to create one and 10.14 is minimum
# we managed to compile ccache for.
if target_arch != "arm64" and tuple(
int(d)
for d in platform.release().split(".")) >= (18, 2):
url = "https://nuitka.net/ccache/v4.2.1/ccache-4.2.1.zip"
ccache_binary = getCachedDownload(
url=url,
is_arch_specific=False,
specificity=url.rsplit("/", 2)[1],
flatten=True,
binary="ccache",
message=
"Nuitka will make use of ccache to speed up repeated compilation.",
reject=None,
assume_yes_for_downloads=assume_yes_for_downloads,
)
else:
scons_details_logger.info(
"Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable."
% ccache_binary)
if ccache_binary is not None and os.path.exists(ccache_binary):
# Make sure the
# In case we are on Windows, make sure the Anaconda form runs outside of Anaconda
# environment, by adding DLL folder to PATH.
assert areSamePaths(
getExecutablePath(os.path.basename(env.the_compiler), env=env),
cc_path)
# We use absolute paths for CC, pass it like this, as ccache does not like absolute.
env["CXX"] = env["CC"] = '"%s" "%s"' % (ccache_binary, cc_path)
# Spare ccache the detection of the compiler, seems it will also misbehave when it's
# prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage.
env["LINK"] = cc_path
scons_details_logger.info(
"Found ccache '%s' to cache C compilation result." % ccache_binary)
scons_details_logger.info(
"Providing real CC path '%s' via PATH extension." % cc_path)
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 0x300:
command = command.encode("utf8")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args=[sys.executable, "-s", "-S", "-v", tmp_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(os.environ, PYTHONIOENCODING="utf_8"),
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf-8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300:
if module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "shlib", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "shlib", filename))
for module_name, _prio, kind, filename in sorted(detections):
if kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
if str is not bytes:
command = command.encode("utf8")
_stdout, stderr, exit_code = executeProcess(
command=(
sys.executable,
"-s",
"-S",
"-v",
"-c",
"import sys;exec(sys.stdin.read())",
),
stdin=command,
env=dict(os.environ, PYTHONIOENCODING="utf-8"),
)
assert type(stderr) is bytes
# Don't let errors here go unnoticed.
if exit_code != 0:
# An error by the user pressing CTRL-C should not lead to the below output.
if b"KeyboardInterrupt" in stderr:
general.sysexit("Pressed CTRL-C while detecting early imports.")
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"from" and python_version < 0x300:
filename = parts[1][len(b"from "):]
if str is not bytes: # For consistency, and maybe later reuse
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
else:
assert False
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300 and module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "extension", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "extension", filename))
for module_name, _priority, kind, filename in sorted(detections):
if isStandardLibraryNoAutoInclusionModule(module_name):
continue
if kind == "extension":
_detectedExtensionModule(
filename=filename,
module_name=module_name,
result=result,
technical=technical,
)
elif kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
else:
assert False, kind
return result
