def getDependsExePath():
"""Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if getArchitecture() == "x86":
depends_url = "https://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "https://dependencywalker.com/depends22_x64.zip"
return getCachedDownload(
url=depends_url,
is_arch_specific=True,
binary="depends.exe",
flatten=True,
specifity="", # Note: If there ever was an update, put version here.
message="""\
Nuitka will make use of Dependency Walker (https://dependencywalker.com) tool
to analyze the dependencies of Python extension modules.""",
reject=
"Nuitka does not work in --standalone or --onefile on Windows without.",
assume_yes_for_downloads=assumeYesForDownloads(),
)
def runSconsBackend(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=False),
"source_dir": OutputDirectories.getSourceDirectoryPath(),
"nuitka_python": asBoolStr(isNuitkaPython()),
"debug_mode": asBoolStr(Options.is_debug),
"python_debug": asBoolStr(Options.isPythonDebug()),
"module_mode": asBoolStr(Options.shallMakeModule()),
"full_compat": asBoolStr(Options.is_fullcompat),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"python_version": python_version_str,
"target_arch": getArchitecture(),
"python_prefix": getDirectoryRealPath(getSystemPrefixPath()),
"nuitka_src": SconsInterface.getSconsDataPath(),
"module_count": "%d"
% (
1
+ len(ModuleRegistry.getDoneModules())
+ len(ModuleRegistry.getUncompiledNonTechnicalModules())
),
}
if Options.isLowMemory():
options["low_memory"] = asBoolStr(True)
if not Options.shallMakeModule():
options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False)
main_module = ModuleRegistry.getRootTopModule()
assert main_module.isMainModule()
main_module_name = main_module.getFullName()
if main_module_name != "__main__":
options["main_module_name"] = main_module_name
if Options.shallUseStaticLibPython():
options["static_libpython"] = getSystemStaticLibPythonPath()
if isDebianPackagePython():
options["debian_python"] = asBoolStr(True)
if isMSYS2MingwPython():
options["msys2_mingw_python"] = asBoolStr(True)
if isAnacondaPython():
options["anaconda_python"] = asBoolStr(True)
if isApplePython():
options["apple_python"] = asBoolStr(True)
if isPyenvPython():
options["pyenv_python"] = asBoolStr(True)
if Options.isStandaloneMode():
options["standalone_mode"] = asBoolStr(True)
if Options.isOnefileMode():
options["onefile_mode"] = asBoolStr(True)
if Options.isOnefileTempDirMode():
options["onefile_temp_mode"] = asBoolStr(True)
if Options.getForcedStdoutPath():
options["forced_stdout_path"] = Options.getForcedStdoutPath()
if Options.getForcedStderrPath():
options["forced_stderr_path"] = Options.getForcedStderrPath()
if Options.shallTreatUninstalledPython():
options["uninstalled_python"] = asBoolStr(True)
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules())
)
if Options.isProfile():
options["profile_mode"] = asBoolStr(True)
if hasPythonFlagNoWarnings():
options["no_python_warnings"] = asBoolStr(True)
if hasPythonFlagNoAsserts():
options["python_sysflag_optimize"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = asBoolStr(True)
if python_version < 0x300 and (
sys.flags.division_warning or sys.flags.py3k_warning
):
options["python_sysflag_division_warning"] = asBoolStr(True)
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = asBoolStr(True)
if int(os.environ.get("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())):
options["python_sysflag_no_site"] = asBoolStr(True)
if Options.hasPythonFlagTraceImports():
options["python_sysflag_verbose"] = asBoolStr(True)
if Options.hasPythonFlagNoRandomization():
options["python_sysflag_no_randomization"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.unicode:
options["python_sysflag_unicode"] = asBoolStr(True)
if python_version >= 0x370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = asBoolStr(True)
if hasPythonFlagUnbuffered():
options["python_sysflag_unbuffered"] = asBoolStr(True)
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
if Options.shallMakeModule():
options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
SconsInterface.setCommonOptions(options)
if Options.shallCreatePgoInput():
options["pgo_mode"] = "python"
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
)
if not result:
return result, options
# Need to make it usable before executing it.
executePostProcessing()
_runPythonPgoBinary()
return True, options
# Need to restart compilation from scratch here.
if Options.isPgoMode():
# For C level PGO, we have a 2 pass system. TODO: Make it more global for onefile
# and standalone mode proper support, which might need data files to be
# there, which currently are not yet there, so it won't run.
if Options.isPgoMode():
options["pgo_mode"] = "generate"
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
)
if not result:
return result, options
# Need to make it usable before executing it.
executePostProcessing()
_runCPgoBinary()
options["pgo_mode"] = "use"
result = (
SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
),
options,
)
if options.get("pgo_mode") == "use" and _wasMsvcMode():
_deleteMsvcPGOFiles(pgo_mode="use")
return result
def _runOnefileScons(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True)
SconsInterface.cleanSconsDirectory(source_dir)
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=True),
"result_exe": OutputDirectories.getResultFullpath(onefile=True),
"source_dir": source_dir,
"debug_mode": asBoolStr(Options.is_debug),
"unstripped_mode": asBoolStr(Options.isUnstripped()),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"target_arch": getArchitecture(),
"python_prefix": sys.prefix,
"nuitka_src": SconsInterface.getSconsDataPath(),
"compiled_exe": OutputDirectories.getResultFullpath(onefile=False),
}
# Ask Scons to cache on Windows, except where the directory is thrown
# away. On non-Windows you can should use ccache instead.
if not Options.isRemoveBuildDir() and getOS() == "Windows":
options["cache_mode"] = "true"
if Options.isLto():
options["lto_mode"] = "true"
if Options.shallDisableConsoleWindow():
options["win_disable_console"] = "true"
if Options.isShowScons():
options["show_scons"] = "true"
if Options.isMingw64():
options["mingw_mode"] = "true"
if Options.getMsvcVersion():
msvc_version = Options.getMsvcVersion()
msvc_version = msvc_version.replace("exp", "Exp")
if "." not in msvc_version:
msvc_version += ".0"
options["msvc_version"] = msvc_version
if getOS() == "Windows":
options["noelf_mode"] = "true"
if Options.isClang():
options["clang_mode"] = "true"
cpp_defines = Plugins.getPreprocessorSymbols()
if cpp_defines:
options["cpp_defines"] = ",".join(
"%s%s%s" % (key, "=" if value else "", value or "")
for key, value in cpp_defines.items())
link_libraries = Plugins.getExtraLinkLibraries()
if link_libraries:
options["link_libraries"] = ",".join(link_libraries)
if Options.shallRunInDebugger():
options["full_names"] = "true"
if Options.assumeYesForDownloads():
options["assume_yes_for_downloads"] = "true"
onefile_env_values = {}
if not Options.isWindowsOnefileTempDirMode():
# Merge version information if necessary, to avoid collisions, or deep nesting
# in file system.
product_version = version_resources["ProductVersion"]
file_version = version_resources["FileVersion"]
if product_version != file_version:
effective_version = "%s-%s" % (product_version, file_version)
else:
effective_version = file_version
onefile_env_values["ONEFILE_COMPANY"] = version_resources[
"CompanyName"]
onefile_env_values["ONEFILE_PRODUCT"] = version_resources[
"ProductName"]
onefile_env_values["ONEFILE_VERSION"] = effective_version
with withEnvironmentVarsOverriden(onefile_env_values):
result = SconsInterface.runScons(options=options,
quiet=quiet,
scons_filename="WindowsOnefile.scons")
# Exit if compilation failed.
if not result:
scons_logger.sysexit(
"Error, one file bootstrap build for Windows failed.")
if Options.isRemoveBuildDir():
general.info("Removing onefile build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping onefile build directory %r." % source_dir)
def _parseDependsExeOutput2(lines, result):
inside = False
first = False
for line in lines:
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
dll_filename = line[line.find("]") + 2 :].rstrip()
dll_filename = os.path.normcase(dll_filename)
# Skip DLLs that failed to load, apparently not needed anyway.
if "E" in line[: line.find("]")]:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[: line.find("]")]:
# One exception are PythonXY.DLL
if dll_filename.startswith("python") and dll_filename.endswith(".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64" if getArchitecture() == "x86_64" else "System32",
dll_filename,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
dll_filename = os.path.abspath(dll_filename)
dll_name = os.path.basename(dll_filename)
# Ignore this runtime DLL of Python2.
if dll_name in ("msvcr90.dll",):
continue
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), (dll_filename, line)
# Allow plugins to prevent inclusion. TODO: This should be called with
# only the new ones.
blocked = Plugins.removeDllDependencies(
dll_filename=dll_filename, dll_filenames=result
)
for to_remove in blocked:
result.discard(to_remove)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _parsePEFileOutput(
binary_filename,
scan_dirs,
is_main_executable,
source_dir,
original_dir,
use_cache,
update_cache,
):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches,too-many-locals
result = OrderedSet()
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="pefile",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
# TODO: We are lazy with the format, pylint: disable=eval-used
extracted = eval(getFileContents(cache_filename))
else:
if Options.isShowProgress():
info("Analysing dependencies of '%s'." % binary_filename)
extracted = getPEFileInformation(binary_filename)
if update_cache:
with withFileLock():
with open(cache_filename, "w") as cache_file:
print(repr(extracted), file=cache_file)
# Add native system directory based on pe file architecture and os architecture
# Python 32: system32 = syswow64 = 32 bits systemdirectory
# Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory
# Get DLL imports from PE file
for dll_name in extracted["DLLs"]:
dll_name = dll_name.upper()
# Try determine DLL path from scan dirs
for scan_dir in scan_dirs:
dll_filename = os.path.normcase(
os.path.abspath(os.path.join(scan_dir, dll_name))
)
if os.path.isfile(dll_filename):
break
else:
if dll_name.startswith("API-MS-WIN-") or dll_name.startswith("EXT-MS-WIN-"):
continue
# Found via RC_MANIFEST as copied from Python.
if dll_name == "MSVCR90.DLL":
continue
if dll_name.startswith("python") and dll_name.endswith(".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64" if getArchitecture() == "x86_64" else "System32",
dll_name,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
if dll_filename not in result:
result.add(dll_filename)
# TODO: Shouldn't be here.
blocked = Plugins.removeDllDependencies(
dll_filename=binary_filename, dll_filenames=result
)
for to_remove in blocked:
result.discard(to_remove)
return result
def _detectBinaryPathDLLsWindowsPE(is_main_executable, source_dir,
original_dir, binary_filename,
package_name):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches,too-many-locals
result = set()
cache_filename = _getCacheFilename(is_main_executable, source_dir,
original_dir, binary_filename)
if (os.path.exists(cache_filename)
and not Options.shallNotUseDependsExeCachedResults()):
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if os.path.isdir(original_dir):
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
if Options.isExperimental("use_pefile_fullrecurse"):
try:
scan_dirs.extend(getSubDirectories(get_python_lib()))
except OSError:
print(
"Cannot recurse into site-packages for dependencies. Path not found."
)
else:
# Fix for missing pywin32 inclusion when using pythonwin library, no way to detect that automagically
# Since there are more than one dependencies on pywintypes37.dll, let's include this anyway
# In recursive mode, using dirname(original_dir) won't always work, hence get_python_lib
try:
scan_dirs.append(os.path.join(get_python_lib(),
"pywin32_system32"))
except OSError:
pass
# Add native system directory based on pe file architecture and os architecture
# Python 32: system32 = syswow64 = 32 bits systemdirectory
# Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory
binary_file_is_64bit = _isPE64(binary_filename)
python_is_64bit = getArchitecture() == "x86_64"
if binary_file_is_64bit is not python_is_64bit:
print("Warning: Using Python x64=%s with x64=%s binary dependencies" %
(binary_file_is_64bit, python_is_64bit))
if binary_file_is_64bit:
# This is actually not useful as of today since we don't compile 32 bits on 64 bits
if python_is_64bit:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
"System32"))
else:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64"))
else:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "System32"))
if Options.isExperimental("use_pefile_recurse"):
# Recursive one level scanning of all .pyd and .dll in the original_dir too
# This shall fix a massive list of missing dependencies that may come with included libraries which themselves
# need to be scanned for inclusions
for root, _, filenames in os.walk(original_dir):
for optional_libary in filenames:
if optional_libary.endswith(
".dll") or optional_libary.endswith(".pyd"):
_parsePEFileOutput(os.path.join(root, optional_libary),
scan_dirs, result)
_parsePEFileOutput(binary_filename, scan_dirs, result)
if not Options.shallNotStoreDependsExeCachedResults():
with open(cache_filename, "w") as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def _parseDependsExeOutput2(lines):
result = OrderedSet()
inside = False
first = False
for line in lines:
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
dll_filename = line[line.find("]") + 2:].rstrip()
dll_filename = os.path.normcase(dll_filename)
# Skip DLLs that failed to load, apparently not needed anyway.
if "E" in line[:line.find("]")]:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[:line.find("]")]:
# One exception are PythonXY.DLL
if dll_filename.startswith("python") and dll_filename.endswith(
".dll"):
dll_filename = os.path.join(
os.environ["SYSTEMROOT"],
"SysWOW64"
if getArchitecture() == "x86_64" else "System32",
dll_filename,
)
dll_filename = os.path.normcase(dll_filename)
else:
continue
dll_filename = os.path.abspath(dll_filename)
dll_name = os.path.basename(dll_filename)
# Ignore this runtime DLL of Python2, will be coming via manifest.
if dll_name in ("msvcr90.dll", ):
continue
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), (dll_filename, line)
result.add(os.path.normcase(os.path.abspath(dll_filename)))
return result
def isClang():
""":returns: bool derived from ``--clang`` or enforced by platform, e.g. macOS or FreeBSD some targets."""
return (options.clang or isMacOS() or isOpenBSD()
or (isFreeBSD() and getArchitecture() != "powerpc"))
def _runOnefileScons(quiet, onefile_compression):
source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True)
SconsInterface.cleanSconsDirectory(source_dir)
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name":
OutputDirectories.getResultBasepath(onefile=True),
"result_exe":
OutputDirectories.getResultFullpath(onefile=True),
"source_dir":
source_dir,
"debug_mode":
asBoolStr(Options.is_debug),
"experimental":
",".join(Options.getExperimentalIndications()),
"trace_mode":
asBoolStr(Options.shallTraceExecution()),
"target_arch":
getArchitecture(),
"python_prefix":
sys.prefix,
"nuitka_src":
SconsInterface.getSconsDataPath(),
"compiled_exe":
OutputDirectories.getResultFullpath(onefile=False),
"onefile_compression":
asBoolStr(onefile_compression),
"onefile_splash_screen":
asBoolStr(Options.getWindowsSplashScreen() is not None),
}
if Options.isClang():
options["clang_mode"] = "true"
SconsInterface.setCommonOptions(options)
onefile_env_values = {}
if Options.isOnefileTempDirMode():
onefile_env_values[
"ONEFILE_TEMP_SPEC"] = Options.getOnefileTempDirSpec(
use_default=True)
else:
# Merge version information if possible, to avoid collisions, or deep nesting
# in file system.
product_version = version_resources["ProductVersion"]
file_version = version_resources["FileVersion"]
if product_version != file_version:
effective_version = "%s-%s" % (product_version, file_version)
else:
effective_version = file_version
onefile_env_values["ONEFILE_COMPANY"] = version_resources[
"CompanyName"]
onefile_env_values["ONEFILE_PRODUCT"] = version_resources[
"ProductName"]
onefile_env_values["ONEFILE_VERSION"] = effective_version
with withEnvironmentVarsOverridden(onefile_env_values):
result = SconsInterface.runScons(options=options,
quiet=quiet,
scons_filename="Onefile.scons")
# Exit if compilation failed.
if not result:
onefile_logger.sysexit("Error, onefile bootstrap binary build failed.")
if Options.isRemoveBuildDir():
onefile_logger.info("Removing onefile build directory %r." %
source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
onefile_logger.info("Keeping onefile build directory %r." % source_dir)
