def getDependsExePath(): """Return the path of depends.exe (for Windows). Will prompt the user to download if not already cached in AppData directory for Nuitka. """ if getArchitecture() == "x86": depends_url = "https://dependencywalker.com/depends22_x86.zip" else: depends_url = "https://dependencywalker.com/depends22_x64.zip" return getCachedDownload( url=depends_url, is_arch_specific=True, binary="depends.exe", flatten=True, specifity="", # Note: If there ever was an update, put version here. message="""\ Nuitka will make use of Dependency Walker (https://dependencywalker.com) tool to analyze the dependencies of Python extension modules.""", reject= "Nuitka does not work in --standalone or --onefile on Windows without.", assume_yes_for_downloads=assumeYesForDownloads(), )
def runSconsBackend(quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements asBoolStr = SconsInterface.asBoolStr options = { "result_name": OutputDirectories.getResultBasepath(onefile=False), "source_dir": OutputDirectories.getSourceDirectoryPath(), "nuitka_python": asBoolStr(isNuitkaPython()), "debug_mode": asBoolStr(Options.is_debug), "python_debug": asBoolStr(Options.isPythonDebug()), "module_mode": asBoolStr(Options.shallMakeModule()), "full_compat": asBoolStr(Options.is_fullcompat), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": asBoolStr(Options.shallTraceExecution()), "python_version": python_version_str, "target_arch": getArchitecture(), "python_prefix": getDirectoryRealPath(getSystemPrefixPath()), "nuitka_src": SconsInterface.getSconsDataPath(), "module_count": "%d" % ( 1 + len(ModuleRegistry.getDoneModules()) + len(ModuleRegistry.getUncompiledNonTechnicalModules()) ), } if Options.isLowMemory(): options["low_memory"] = asBoolStr(True) if not Options.shallMakeModule(): options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False) main_module = ModuleRegistry.getRootTopModule() assert main_module.isMainModule() main_module_name = main_module.getFullName() if main_module_name != "__main__": options["main_module_name"] = main_module_name if Options.shallUseStaticLibPython(): options["static_libpython"] = getSystemStaticLibPythonPath() if isDebianPackagePython(): options["debian_python"] = asBoolStr(True) if isMSYS2MingwPython(): options["msys2_mingw_python"] = asBoolStr(True) if isAnacondaPython(): options["anaconda_python"] = asBoolStr(True) if isApplePython(): options["apple_python"] = asBoolStr(True) if isPyenvPython(): options["pyenv_python"] = asBoolStr(True) if Options.isStandaloneMode(): options["standalone_mode"] = asBoolStr(True) if Options.isOnefileMode(): options["onefile_mode"] = asBoolStr(True) if Options.isOnefileTempDirMode(): options["onefile_temp_mode"] = asBoolStr(True) if Options.getForcedStdoutPath(): options["forced_stdout_path"] = Options.getForcedStdoutPath() if Options.getForcedStderrPath(): options["forced_stderr_path"] = Options.getForcedStderrPath() if Options.shallTreatUninstalledPython(): options["uninstalled_python"] = asBoolStr(True) if ModuleRegistry.getUncompiledTechnicalModules(): options["frozen_modules"] = str( len(ModuleRegistry.getUncompiledTechnicalModules()) ) if Options.isProfile(): options["profile_mode"] = asBoolStr(True) if hasPythonFlagNoWarnings(): options["no_python_warnings"] = asBoolStr(True) if hasPythonFlagNoAsserts(): options["python_sysflag_optimize"] = asBoolStr(True) if python_version < 0x300 and sys.flags.py3k_warning: options["python_sysflag_py3k_warning"] = asBoolStr(True) if python_version < 0x300 and ( sys.flags.division_warning or sys.flags.py3k_warning ): options["python_sysflag_division_warning"] = asBoolStr(True) if sys.flags.bytes_warning: options["python_sysflag_bytes_warning"] = asBoolStr(True) if int(os.environ.get("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())): options["python_sysflag_no_site"] = asBoolStr(True) if Options.hasPythonFlagTraceImports(): options["python_sysflag_verbose"] = asBoolStr(True) if Options.hasPythonFlagNoRandomization(): options["python_sysflag_no_randomization"] = asBoolStr(True) if python_version < 0x300 and sys.flags.unicode: options["python_sysflag_unicode"] = asBoolStr(True) if python_version >= 0x370 and sys.flags.utf8_mode: options["python_sysflag_utf8"] = asBoolStr(True) if hasPythonFlagUnbuffered(): options["python_sysflag_unbuffered"] = asBoolStr(True) abiflags = getPythonABI() if abiflags: options["abiflags"] = abiflags if Options.shallMakeModule(): options["module_suffix"] = getSharedLibrarySuffix(preferred=True) SconsInterface.setCommonOptions(options) if Options.shallCreatePgoInput(): options["pgo_mode"] = "python" result = SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ) if not result: return result, options # Need to make it usable before executing it. executePostProcessing() _runPythonPgoBinary() return True, options # Need to restart compilation from scratch here. if Options.isPgoMode(): # For C level PGO, we have a 2 pass system. TODO: Make it more global for onefile # and standalone mode proper support, which might need data files to be # there, which currently are not yet there, so it won't run. if Options.isPgoMode(): options["pgo_mode"] = "generate" result = SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ) if not result: return result, options # Need to make it usable before executing it. executePostProcessing() _runCPgoBinary() options["pgo_mode"] = "use" result = ( SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ), options, ) if options.get("pgo_mode") == "use" and _wasMsvcMode(): _deleteMsvcPGOFiles(pgo_mode="use") return result
def _runOnefileScons(quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True) SconsInterface.cleanSconsDirectory(source_dir) asBoolStr = SconsInterface.asBoolStr options = { "result_name": OutputDirectories.getResultBasepath(onefile=True), "result_exe": OutputDirectories.getResultFullpath(onefile=True), "source_dir": source_dir, "debug_mode": asBoolStr(Options.is_debug), "unstripped_mode": asBoolStr(Options.isUnstripped()), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": asBoolStr(Options.shallTraceExecution()), "target_arch": getArchitecture(), "python_prefix": sys.prefix, "nuitka_src": SconsInterface.getSconsDataPath(), "compiled_exe": OutputDirectories.getResultFullpath(onefile=False), } # Ask Scons to cache on Windows, except where the directory is thrown # away. On non-Windows you can should use ccache instead. if not Options.isRemoveBuildDir() and getOS() == "Windows": options["cache_mode"] = "true" if Options.isLto(): options["lto_mode"] = "true" if Options.shallDisableConsoleWindow(): options["win_disable_console"] = "true" if Options.isShowScons(): options["show_scons"] = "true" if Options.isMingw64(): options["mingw_mode"] = "true" if Options.getMsvcVersion(): msvc_version = Options.getMsvcVersion() msvc_version = msvc_version.replace("exp", "Exp") if "." not in msvc_version: msvc_version += ".0" options["msvc_version"] = msvc_version if getOS() == "Windows": options["noelf_mode"] = "true" if Options.isClang(): options["clang_mode"] = "true" cpp_defines = Plugins.getPreprocessorSymbols() if cpp_defines: options["cpp_defines"] = ",".join( "%s%s%s" % (key, "=" if value else "", value or "") for key, value in cpp_defines.items()) link_libraries = Plugins.getExtraLinkLibraries() if link_libraries: options["link_libraries"] = ",".join(link_libraries) if Options.shallRunInDebugger(): options["full_names"] = "true" if Options.assumeYesForDownloads(): options["assume_yes_for_downloads"] = "true" onefile_env_values = {} if not Options.isWindowsOnefileTempDirMode(): # Merge version information if necessary, to avoid collisions, or deep nesting # in file system. product_version = version_resources["ProductVersion"] file_version = version_resources["FileVersion"] if product_version != file_version: effective_version = "%s-%s" % (product_version, file_version) else: effective_version = file_version onefile_env_values["ONEFILE_COMPANY"] = version_resources[ "CompanyName"] onefile_env_values["ONEFILE_PRODUCT"] = version_resources[ "ProductName"] onefile_env_values["ONEFILE_VERSION"] = effective_version with withEnvironmentVarsOverriden(onefile_env_values): result = SconsInterface.runScons(options=options, quiet=quiet, scons_filename="WindowsOnefile.scons") # Exit if compilation failed. if not result: scons_logger.sysexit( "Error, one file bootstrap build for Windows failed.") if Options.isRemoveBuildDir(): general.info("Removing onefile build directory %r." % source_dir) removeDirectory(path=source_dir, ignore_errors=False) assert not os.path.exists(source_dir) else: general.info("Keeping onefile build directory %r." % source_dir)
def _parseDependsExeOutput2(lines, result): inside = False first = False for line in lines: if "| Module Dependency Tree |" in line: inside = True first = True continue if not inside: continue if "| Module List |" in line: break if "]" not in line: continue dll_filename = line[line.find("]") + 2 :].rstrip() dll_filename = os.path.normcase(dll_filename) # Skip DLLs that failed to load, apparently not needed anyway. if "E" in line[: line.find("]")]: continue # Skip missing DLLs, apparently not needed anyway. if "?" in line[: line.find("]")]: # One exception are PythonXY.DLL if dll_filename.startswith("python") and dll_filename.endswith(".dll"): dll_filename = os.path.join( os.environ["SYSTEMROOT"], "SysWOW64" if getArchitecture() == "x86_64" else "System32", dll_filename, ) dll_filename = os.path.normcase(dll_filename) else: continue dll_filename = os.path.abspath(dll_filename) dll_name = os.path.basename(dll_filename) # Ignore this runtime DLL of Python2. if dll_name in ("msvcr90.dll",): continue # The executable itself is of course exempted. We cannot check its path # because depends.exe mistreats unicode paths. if first: first = False continue assert os.path.isfile(dll_filename), (dll_filename, line) # Allow plugins to prevent inclusion. TODO: This should be called with # only the new ones. blocked = Plugins.removeDllDependencies( dll_filename=dll_filename, dll_filenames=result ) for to_remove in blocked: result.discard(to_remove) result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _parsePEFileOutput( binary_filename, scan_dirs, is_main_executable, source_dir, original_dir, use_cache, update_cache, ): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches,too-many-locals result = OrderedSet() if use_cache or update_cache: cache_filename = _getCacheFilename( dependency_tool="pefile", is_main_executable=is_main_executable, source_dir=source_dir, original_dir=original_dir, binary_filename=binary_filename, ) if use_cache: with withFileLock(): if not os.path.exists(cache_filename): use_cache = False if use_cache: # TODO: We are lazy with the format, pylint: disable=eval-used extracted = eval(getFileContents(cache_filename)) else: if Options.isShowProgress(): info("Analysing dependencies of '%s'." % binary_filename) extracted = getPEFileInformation(binary_filename) if update_cache: with withFileLock(): with open(cache_filename, "w") as cache_file: print(repr(extracted), file=cache_file) # Add native system directory based on pe file architecture and os architecture # Python 32: system32 = syswow64 = 32 bits systemdirectory # Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory # Get DLL imports from PE file for dll_name in extracted["DLLs"]: dll_name = dll_name.upper() # Try determine DLL path from scan dirs for scan_dir in scan_dirs: dll_filename = os.path.normcase( os.path.abspath(os.path.join(scan_dir, dll_name)) ) if os.path.isfile(dll_filename): break else: if dll_name.startswith("API-MS-WIN-") or dll_name.startswith("EXT-MS-WIN-"): continue # Found via RC_MANIFEST as copied from Python. if dll_name == "MSVCR90.DLL": continue if dll_name.startswith("python") and dll_name.endswith(".dll"): dll_filename = os.path.join( os.environ["SYSTEMROOT"], "SysWOW64" if getArchitecture() == "x86_64" else "System32", dll_name, ) dll_filename = os.path.normcase(dll_filename) else: continue if dll_filename not in result: result.add(dll_filename) # TODO: Shouldn't be here. blocked = Plugins.removeDllDependencies( dll_filename=binary_filename, dll_filenames=result ) for to_remove in blocked: result.discard(to_remove) return result
def _detectBinaryPathDLLsWindowsPE(is_main_executable, source_dir, original_dir, binary_filename, package_name): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches,too-many-locals result = set() cache_filename = _getCacheFilename(is_main_executable, source_dir, original_dir, binary_filename) if (os.path.exists(cache_filename) and not Options.shallNotUseDependsExeCachedResults()): for line in getFileContentByLine(cache_filename): line = line.strip() result.add(line) return result scan_dirs = [sys.prefix] if package_name is not None: from nuitka.importing.Importing import findModule package_dir = findModule(None, package_name, None, 0, False)[1] if os.path.isdir(package_dir): scan_dirs.append(package_dir) scan_dirs.extend(getSubDirectories(package_dir)) if os.path.isdir(original_dir): scan_dirs.append(original_dir) scan_dirs.extend(getSubDirectories(original_dir)) if Options.isExperimental("use_pefile_fullrecurse"): try: scan_dirs.extend(getSubDirectories(get_python_lib())) except OSError: print( "Cannot recurse into site-packages for dependencies. Path not found." ) else: # Fix for missing pywin32 inclusion when using pythonwin library, no way to detect that automagically # Since there are more than one dependencies on pywintypes37.dll, let's include this anyway # In recursive mode, using dirname(original_dir) won't always work, hence get_python_lib try: scan_dirs.append(os.path.join(get_python_lib(), "pywin32_system32")) except OSError: pass # Add native system directory based on pe file architecture and os architecture # Python 32: system32 = syswow64 = 32 bits systemdirectory # Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory binary_file_is_64bit = _isPE64(binary_filename) python_is_64bit = getArchitecture() == "x86_64" if binary_file_is_64bit is not python_is_64bit: print("Warning: Using Python x64=%s with x64=%s binary dependencies" % (binary_file_is_64bit, python_is_64bit)) if binary_file_is_64bit: # This is actually not useful as of today since we don't compile 32 bits on 64 bits if python_is_64bit: scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "System32")) else: scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")) else: scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "System32")) if Options.isExperimental("use_pefile_recurse"): # Recursive one level scanning of all .pyd and .dll in the original_dir too # This shall fix a massive list of missing dependencies that may come with included libraries which themselves # need to be scanned for inclusions for root, _, filenames in os.walk(original_dir): for optional_libary in filenames: if optional_libary.endswith( ".dll") or optional_libary.endswith(".pyd"): _parsePEFileOutput(os.path.join(root, optional_libary), scan_dirs, result) _parsePEFileOutput(binary_filename, scan_dirs, result) if not Options.shallNotStoreDependsExeCachedResults(): with open(cache_filename, "w") as cache_file: for dll_filename in result: print(dll_filename, file=cache_file) return result
def _parseDependsExeOutput2(lines): result = OrderedSet() inside = False first = False for line in lines: if "| Module Dependency Tree |" in line: inside = True first = True continue if not inside: continue if "| Module List |" in line: break if "]" not in line: continue dll_filename = line[line.find("]") + 2:].rstrip() dll_filename = os.path.normcase(dll_filename) # Skip DLLs that failed to load, apparently not needed anyway. if "E" in line[:line.find("]")]: continue # Skip missing DLLs, apparently not needed anyway. if "?" in line[:line.find("]")]: # One exception are PythonXY.DLL if dll_filename.startswith("python") and dll_filename.endswith( ".dll"): dll_filename = os.path.join( os.environ["SYSTEMROOT"], "SysWOW64" if getArchitecture() == "x86_64" else "System32", dll_filename, ) dll_filename = os.path.normcase(dll_filename) else: continue dll_filename = os.path.abspath(dll_filename) dll_name = os.path.basename(dll_filename) # Ignore this runtime DLL of Python2, will be coming via manifest. if dll_name in ("msvcr90.dll", ): continue # The executable itself is of course exempted. We cannot check its path # because depends.exe mistreats unicode paths. if first: first = False continue assert os.path.isfile(dll_filename), (dll_filename, line) result.add(os.path.normcase(os.path.abspath(dll_filename))) return result
def isClang(): """:returns: bool derived from ``--clang`` or enforced by platform, e.g. macOS or FreeBSD some targets.""" return (options.clang or isMacOS() or isOpenBSD() or (isFreeBSD() and getArchitecture() != "powerpc"))
def _runOnefileScons(quiet, onefile_compression): source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True) SconsInterface.cleanSconsDirectory(source_dir) asBoolStr = SconsInterface.asBoolStr options = { "result_name": OutputDirectories.getResultBasepath(onefile=True), "result_exe": OutputDirectories.getResultFullpath(onefile=True), "source_dir": source_dir, "debug_mode": asBoolStr(Options.is_debug), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": asBoolStr(Options.shallTraceExecution()), "target_arch": getArchitecture(), "python_prefix": sys.prefix, "nuitka_src": SconsInterface.getSconsDataPath(), "compiled_exe": OutputDirectories.getResultFullpath(onefile=False), "onefile_compression": asBoolStr(onefile_compression), "onefile_splash_screen": asBoolStr(Options.getWindowsSplashScreen() is not None), } if Options.isClang(): options["clang_mode"] = "true" SconsInterface.setCommonOptions(options) onefile_env_values = {} if Options.isOnefileTempDirMode(): onefile_env_values[ "ONEFILE_TEMP_SPEC"] = Options.getOnefileTempDirSpec( use_default=True) else: # Merge version information if possible, to avoid collisions, or deep nesting # in file system. product_version = version_resources["ProductVersion"] file_version = version_resources["FileVersion"] if product_version != file_version: effective_version = "%s-%s" % (product_version, file_version) else: effective_version = file_version onefile_env_values["ONEFILE_COMPANY"] = version_resources[ "CompanyName"] onefile_env_values["ONEFILE_PRODUCT"] = version_resources[ "ProductName"] onefile_env_values["ONEFILE_VERSION"] = effective_version with withEnvironmentVarsOverridden(onefile_env_values): result = SconsInterface.runScons(options=options, quiet=quiet, scons_filename="Onefile.scons") # Exit if compilation failed. if not result: onefile_logger.sysexit("Error, onefile bootstrap binary build failed.") if Options.isRemoveBuildDir(): onefile_logger.info("Removing onefile build directory %r." % source_dir) removeDirectory(path=source_dir, ignore_errors=False) assert not os.path.exists(source_dir) else: onefile_logger.info("Keeping onefile build directory %r." % source_dir)