def addClangClPathFromMSVC(env, target_arch, show_scons_mode): cl_exe = getExecutablePath("cl", env=env) if cl_exe is None: scons_logger.warning( "Visual Studio required for for Clang on Windows.") return clang_dir = cl_exe = os.path.join(cl_exe[:cl_exe.lower().rfind("msvc")], "Llvm") if target_arch == "x86_64": clang_dir = os.path.join(clang_dir, "x64", "bin") else: clang_dir = os.path.join(clang_dir, "bin") if os.path.exists(clang_dir): if show_scons_mode: scons_logger.info("Adding MSVC directory %r for Clang to PATH." % clang_dir) addToPATH(env, clang_dir, prefix=True) else: if show_scons_mode: scons_logger.info("No Clang component for MSVC found." % clang_dir)
def enableCcache( the_compiler, env, source_dir, python_prefix, show_scons_mode, assume_yes_for_downloads, ): # The ccache needs absolute path, otherwise it will not work. ccache_logfile = os.path.abspath( os.path.join(source_dir, "ccache-%d.txt" % os.getpid())) setEnvironmentVariable(env, "CCACHE_LOGFILE", ccache_logfile) env["CCACHE_LOGFILE"] = ccache_logfile # First check if it's not already supposed to be a ccache, then do nothing. cc_path = getExecutablePath(the_compiler, env=env) cc_is_link, cc_link_path = getLinkTarget(cc_path) if cc_is_link and os.path.basename(cc_link_path) == "ccache": if show_scons_mode: scons_logger.info( "Chosen compiler %s is pointing to ccache %s already." % (cc_path, cc_link_path)) return True return _injectCcache( the_compiler=the_compiler, cc_path=cc_path, env=env, python_prefix=python_prefix, show_scons_mode=show_scons_mode, assume_yes_for_downloads=assume_yes_for_downloads, )
def reportSlowCompilation(cmd, delta_time): # TODO: for linking, we ought to apply a different timer maybe and attempt to extra # the source file that is causing the issues: pylint: disable=unused-argument if _current != _total: scons_logger.info( "Slow C compilation detected, used %.0fs so far, scalability problem." % delta_time )
def reportCCompiler(env, context): cc_output = env.the_cc_name if env.the_cc_name == "cl": cc_output = "%s %s" % (env.the_cc_name, getMsvcVersionString(env)) else: cc_output = env.the_cc_name scons_logger.info("%s C compiler: %s (%s)." % (context, env.the_compiler, cc_output))
def checkClcache(install_dir): candidate = os.path.join(install_dir, "scripts", "clcache.exe") if show_scons_mode: scons_logger.info( "Checking if clcache is at '%s' python installation path." % candidate) if os.path.exists(candidate): return True
def updateSconsProgressBar(): # Check if link is next, pylint: disable=global-statement global _current _current += 1 reportProgressBar(item=None, update=True) if _current == _total: closeSconsProgressBar() scons_logger.info( "%s linking program (no progress information available)." % _stage )
def switchFromGccToGpp(env): if not env.gcc_mode or env.clang_mode: env.gcc_version = None return env.gcc_version = myDetectVersion(env, env.the_compiler) if env.gcc_version is None: scons_logger.sysexit("""\ Error, failed to detect gcc version of backend compiler %r. """ % env.the_compiler) if "++" in env.the_cc_name: scons_logger.sysexit("""\ Error, compiler %s is apparently a C++ compiler, specify a C compiler instead. """ % env.the_cc_name) # Enforce the minimum version, selecting a potentially existing g++-4.5 # binary if it's not high enough. This is esp. useful under Debian which # allows all compiler to exist next to each other and where g++ might not be # good enough, but g++-4.5 would be. if env.gcc_version < (4, 4): scons_logger.sysexit("""\ The gcc compiler %s (version %s) doesn't have the sufficient \ version (>= 4.4).""" % (env.the_compiler, env.gcc_version)) # CondaCC or newer. if env.mingw_mode and env.gcc_version < (5, 3): scons_logger.sysexit("""\ The MinGW64 compiler %s (version %s) doesn't have the sufficient \ version (>= 5.3).""" % (env.the_compiler, env.gcc_version)) if env.gcc_version < (5, ): scons_logger.info( "The provided gcc is too old, switching to its g++ instead.") # Switch to g++ from gcc then if possible, when C11 mode is false. the_gpp_compiler = os.path.join( os.path.dirname(env.the_compiler), os.path.basename(env.the_compiler).replace("gcc", "g++"), ) if getExecutablePath(the_gpp_compiler, env=env): env.the_compiler = the_gpp_compiler env.the_cc_name = env.the_cc_name.replace("gcc", "g++") else: scons_logger.sysexit( "Error, your gcc is too old for C11 support, and no related g++ to workaround that is found." )
def runSpawnMonitored(spawn, sh, escape, cmd, args, env): thread = SpawnThread(spawn, sh, escape, cmd, args, env) thread.start() # Allow a minute before warning for long compile time. thread.join(60) if thread.is_alive(): scons_logger.info( "Slow C compilation detected, used %.0fs so far, this might indicate scalability problems." % thread.timer_report.getTimer().getDelta()) thread.join() return thread.getSpawnResult()
def runProcessMonitored(cmdline, env): thread = SubprocessThread(cmdline, env) thread.start() # Allow a minute before warning for long compile time. thread.join(60) if thread.is_alive(): scons_logger.info( "Slow C compilation detected, used %.0fs so far, this might indicate scalability problems." % thread.timer_report.getTimer().getDelta()) thread.join() return thread.getProcessResult()
def switchFromGccToGpp(gcc_version, the_compiler, the_cc_name, env): if gcc_version is not None and gcc_version < (5,): scons_logger.info("The provided gcc is too old, switching to g++ instead.") # Switch to g++ from gcc then if possible, when C11 mode is false. the_gpp_compiler = os.path.join( os.path.dirname(the_compiler), os.path.basename(the_compiler).replace("gcc", "g++"), ) if getExecutablePath(the_gpp_compiler, env=env): the_compiler = the_gpp_compiler the_cc_name = the_cc_name.replace("gcc", "g++") else: scons_logger.sysexit( "Error, your gcc is too old for C11 support, and no related g++ to workaround that is found." ) return the_compiler, the_cc_name
def checkCachingSuccess(source_dir): ccache_logfile = getSconsReportValue(source_dir=source_dir, key="CCACHE_LOGFILE") if ccache_logfile is not None: stats = _getCcacheStatistics(ccache_logfile) if not stats: scons_logger.warning("You are not using ccache.") else: counts = defaultdict(int) for _command, result in stats.items(): # These are not important to our users, time based decisions differentiate these. if result in ("cache hit (direct)", "cache hit (preprocessed)"): result = "cache hit" # Newer ccache has these, but they duplicate: if result in ( "direct_cache_hit", "direct_cache_miss", "preprocessed_cache_hit", "preprocessed_cache_miss", "primary_storage_miss", ): continue if result == "primary_storage_hit": result = "cache hit" if result == "cache_miss": result = "cache miss" # Usage of incbin causes this for the constants blob integration. if result in ("unsupported code directive", "disabled"): continue counts[result] += 1 scons_logger.info("Compiled %d C files using ccache." % len(stats)) for result, count in counts.items(): scons_logger.info( "Cached C files (using ccache) with result '%s': %d" % (result, count)) if os.name == "nt": clcache_stats_filename = getSconsReportValue(source_dir=source_dir, key="CLCACHE_STATS") if clcache_stats_filename is not None and os.path.exists( clcache_stats_filename): stats = eval( # lazy, pylint: disable=eval-used getFileContents(clcache_stats_filename)) clcache_hit = stats["CacheHits"] clcache_miss = stats["CacheMisses"] scons_logger.info( "Compiled %d C files using clcache with %d cache hits and %d cache misses." % (clcache_hit + clcache_miss, clcache_hit, clcache_miss))
def checkCachingSuccess(source_dir): ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE") if ccache_logfile is not None: stats = _getCcacheStatistics(ccache_logfile) if not stats: scons_logger.warning("You are not using ccache.") else: counts = defaultdict(int) for _command, result in stats.items(): counts[result] += 1 scons_logger.info("Compiled %d C files using ccache." % len(stats)) for result, count in counts.items(): scons_logger.info( "Cached C files (using ccache) with result '%s': %d" % (result, count)) clcache_logfile = getSconsReportValue(source_dir, "CLCACHE_LOG") if clcache_logfile is not None: stats = _getClcacheStatistics(clcache_logfile) if not stats: scons_logger.warning("You are not using clcache.") else: counts = defaultdict(int) for _command, result in stats.items(): counts[result] += 1 scons_logger.info("Compiled %d C files using clcache." % len(stats)) for result, count in counts.items(): scons_logger.info( "Cached C files (using clcache) with result '%s': %d" % (result, count))
def checkCachingSuccess(source_dir): ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE") if ccache_logfile is not None: stats = _getCcacheStatistics(ccache_logfile) if not stats: scons_logger.warning("You are not using ccache.") else: counts = defaultdict(int) for _command, result in stats.items(): # These are not important to our users, time based decisions differentiate these. if result in ("cache hit (direct)", "cache hit (preprocessed)"): result = "cache hit" counts[result] += 1 scons_logger.info("Compiled %d C files using ccache." % len(stats)) for result, count in counts.items(): scons_logger.info( "Cached C files (using ccache) with result '%s': %d" % (result, count) ) if os.name == "nt": clcache_stats_filename = getSconsReportValue(source_dir, "CLCACHE_STATS") if clcache_stats_filename is not None and os.path.exists( clcache_stats_filename ): stats = eval( # lazy, pylint: disable=eval-used getFileContents(clcache_stats_filename) ) clcache_hit = stats["CacheHits"] clcache_miss = stats["CacheMisses"] scons_logger.info( "Compiled %d C files using clcache with %d cache hits and %d cache misses." % (clcache_hit + clcache_miss, clcache_hit, clcache_miss) )
def checkWindowsCompilerFound(env, target_arch, assume_yes_for_downloads): """Remove compiler of wrong arch or too old gcc and replace with downloaded winlibs gcc.""" if os.name == "nt": # On Windows, in case MSVC was not found and not previously forced, use the # winlibs MinGW64 as a download, and use it as a fallback. compiler_path = getExecutablePath(env["CC"], env=env) # Drop wrong arch compiler, most often found by scans. There might be wrong gcc or cl on the PATH. if compiler_path is not None: the_cc_name = os.path.basename(compiler_path) decision, linker_arch, compiler_arch = decideArchMismatch( target_arch=target_arch, mingw_mode=isGccName(the_cc_name), msvc_mode=not isGccName(the_cc_name), the_cc_name=the_cc_name, compiler_path=compiler_path, ) if decision: # This will trigger using it to use our own gcc in branch below. compiler_path = None scons_logger.info( "Mismatch between Python binary (%r -> %r) and C compiler (%r -> %r) arches, ignored!" % ( os.environ["NUITKA_PYTHON_EXE_PATH"], linker_arch, compiler_path, compiler_arch, )) if compiler_path is not None: the_cc_name = os.path.basename(compiler_path) if isGccName(the_cc_name): gcc_version = myDetectVersion(env, compiler_path) min_version = (8, ) if gcc_version is not None and gcc_version < min_version: # This also will trigger using it to use our own gcc in branch below. compiler_path = None scons_logger.info( "Too old gcc %r (%r < %r) ignored!" % (compiler_path, gcc_version, min_version)) if compiler_path is None: # This will succeed to find "gcc.exe" when conda install m2w64-gcc has # been done. compiler_path = getDownloadedGccPath( target_arch=target_arch, assume_yes_for_downloads=assume_yes_for_downloads, ) addToPATH(env, os.path.dirname(compiler_path), prefix=True) env = createEnvironment( tools=["mingw"], mingw_mode=True, msvc_version=None, target_arch=target_arch, ) env["CC"] = compiler_path return env
def _injectCcache(the_compiler, cc_path, env, python_prefix, show_scons_mode): # We deal with a lot of cases here, pylint: disable=too-many-branches ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY") # If not provided, search it in PATH and guessed directories. if ccache_binary is None: ccache_binary = getExecutablePath("ccache", env=env) if ccache_binary is None: for candidate in _getCcacheGuessedPaths(python_prefix): if show_scons_mode: scons_logger.info( "Checking if ccache is at '%s' guessed path." % candidate) if os.path.exists(candidate): ccache_binary = candidate if show_scons_mode: scons_logger.info( "Using ccache '%s' from guessed path." % ccache_binary) break else: if show_scons_mode: scons_logger.info( "Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable." % ccache_binary) if ccache_binary is not None and os.path.exists(ccache_binary): # In case we are on Windows, make sure the Anaconda form runs outside of Anaconda # environment, by adding DLL folder to PATH. if os.name == "nt": conda_dll_dir = os.path.normpath( os.path.join(ccache_binary, "..", "..", "Library", "mingw-w64", "bin")) if os.path.exists(conda_dll_dir): addToPATH(env, conda_dll_dir, prefix=False) assert getExecutablePath(os.path.basename(the_compiler), env=env) == cc_path # Since we use absolute paths for CC, pass it like this, as ccache does not like absolute. env["CXX"] = env["CC"] = "%s %s" % (ccache_binary, os.path.basename(cc_path)) # Spare ccache the detection of the compiler, seems it will also misbehave when it's # prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage. env["LINK"] = cc_path # Do not consider scons cache anymore. if show_scons_mode: scons_logger.info( "Found ccache '%s' to cache C compilation result." % ccache_binary) scons_logger.info( "Providing real CC path '%s' via PATH extension." % cc_path) result = True else: if isWin32Windows(): scons_logger.warning( "Didn't find ccache for C level caching, follow Nuitka user manual description." ) result = False return result
def enableClcache(the_compiler, env, source_dir, python_prefix, show_scons_mode): # Many branches to deal with, pylint: disable=too-many-branches # The ccache needs absolute path, otherwise it will not work. clcache_logfile = os.path.abspath( os.path.join(source_dir, "clcache-%d.txt" % os.getpid())) # Our spawn function will pick it up from the output. setEnvironmentVariable(env, "CLCACHE_LOG", clcache_logfile) env["CLCACHE_LOG"] = clcache_logfile clcache_binary = os.environ.get("NUITKA_CLCACHE_BINARY") # If not provided, search it in PATH and guessed directories. if clcache_binary is None: clcache_binary = getExecutablePath("clcache", env) if clcache_binary is None: for candidate in _getClcacheGuessedPaths(python_prefix): if show_scons_mode: scons_logger.info( "Checking if clcache is at '%s' guessed path." % candidate) if os.path.exists(candidate): clcache_binary = candidate if show_scons_mode: scons_logger.info( "Using clcache '%s' from guessed path." % clcache_binary) break if clcache_binary is None and os.name == "nt": def checkClcache(install_dir): candidate = os.path.join(install_dir, "scripts", "clcache.exe") if show_scons_mode: scons_logger.info( "Checking if clcache is at '%s' python installation path." % candidate) if os.path.exists(candidate): return True candidate = getPythonInstallPathWindows(supported=("3.6", "3.7", "3.8", "3.9"), decider=checkClcache) if candidate is not None: clcache_binary = os.path.join(candidate, "scripts", "clcache.exe") if show_scons_mode: scons_logger.info( "Using clcache '%s' from registry path." % clcache_binary) else: if show_scons_mode: scons_logger.info( "Using clcache '%s' from NUITKA_CLCACHE_BINARY environment variable." % clcache_binary) if clcache_binary is not None and os.path.exists(clcache_binary): cl_binary = getExecutablePath(the_compiler, env) # The compiler is passed via environment. setEnvironmentVariable(env, "CLCACHE_CL", cl_binary) env["CXX"] = env["CC"] = clcache_binary if show_scons_mode: scons_logger.info( "Found clcache '%s' to cache C compilation result." % clcache_binary) scons_logger.info( "Providing real cl.exe path '%s' via environment." % cl_binary) result = True else: scons_logger.warning( "Didn't find clcache for C level caching, follow Nuitka user manual description." ) result = False return result
def _injectCcache(the_compiler, cc_path, env, python_prefix, show_scons_mode, assume_yes_for_downloads): ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY") # If not provided, search it in PATH and guessed directories. if ccache_binary is None: ccache_binary = getExecutablePath("ccache", env=env) if ccache_binary is None: for candidate in _getCcacheGuessedPaths(python_prefix): if show_scons_mode: scons_logger.info( "Checking if ccache is at '%s' guessed path." % candidate) if os.path.exists(candidate): ccache_binary = candidate if show_scons_mode: scons_logger.info( "Using ccache '%s' from guessed path." % ccache_binary) break if ccache_binary is None and os.name == "nt": url = "https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12-windows-32.zip" ccache_binary = getCachedDownload( url=url, is_arch_specific=False, specifity=url.rsplit("/", 2)[1], flatten=True, binary="ccache.exe", message= "Nuitka will make use of ccache to speed up repeated compilation.", reject=None, assume_yes_for_downloads=assume_yes_for_downloads, ) else: if show_scons_mode: scons_logger.info( "Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable." % ccache_binary) if ccache_binary is not None and os.path.exists(ccache_binary): # Make sure the # In case we are on Windows, make sure the Anaconda form runs outside of Anaconda # environment, by adding DLL folder to PATH. assert getExecutablePath(os.path.basename(the_compiler), env=env) == cc_path # We use absolute paths for CC, pass it like this, as ccache does not like absolute. env["CXX"] = env["CC"] = "%s %s" % (ccache_binary, os.path.basename(cc_path)) # Spare ccache the detection of the compiler, seems it will also misbehave when it's # prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage. env["LINK"] = cc_path # Do not consider scons cache anymore. if show_scons_mode: scons_logger.info( "Found ccache '%s' to cache C compilation result." % ccache_binary) scons_logger.info( "Providing real CC path '%s' via PATH extension." % cc_path) result = True else: if isWin32Windows(): scons_logger.warning( "Didn't find ccache for C level caching, follow Nuitka user manual description." ) result = False return result
def checkWindowsCompilerFound(env, target_arch, clang_mode, msvc_version, assume_yes_for_downloads): """Remove compiler of wrong arch or too old gcc and replace with downloaded winlibs gcc.""" if os.name == "nt": # On Windows, in case MSVC was not found and not previously forced, use the # winlibs MinGW64 as a download, and use it as a fallback. compiler_path = getExecutablePath(env["CC"], env=env) scons_details_logger.info("Checking usability of %r from %r" % (compiler_path, env["CC"])) # Drop wrong arch compiler, most often found by scans. There might be wrong gcc or cl on the PATH. if compiler_path is not None: the_cc_name = os.path.basename(compiler_path) decision, linker_arch, compiler_arch = decideArchMismatch( target_arch=target_arch, mingw_mode=isGccName(the_cc_name), msvc_mode=not isGccName(the_cc_name), the_cc_name=the_cc_name, compiler_path=compiler_path, ) if decision: scons_logger.info( "Mismatch between Python binary (%r -> %r) and C compiler (%r -> %r) arches, that compiler is ignored!" % ( os.environ["NUITKA_PYTHON_EXE_PATH"], linker_arch, compiler_path, compiler_arch, )) # This will trigger using it to use our own gcc in branch below. compiler_path = None env["CC"] = None if compiler_path is not None and msvc_version is not None: if msvc_version == "latest": scons_logger.info("MSVC version resolved to %s." % getMsvcVersionString(env)) # Requested a specific MSVC version, check if that worked. elif msvc_version != getMsvcVersionString(env): scons_logger.info( "Failed to find requested MSVC version (%r != %r)." % (msvc_version, getMsvcVersionString(env))) # This will trigger error exit in branch below. compiler_path = None env["CC"] = None if compiler_path is not None: the_cc_name = os.path.basename(compiler_path) if isGccName(the_cc_name): gcc_version = myDetectVersion(env, compiler_path) min_version = (11, 2) if gcc_version is not None and (gcc_version < min_version or "force-winlibs-gcc" in env.experimental_flags): scons_logger.info( "Too old gcc %r (%r < %r) ignored!" % (compiler_path, gcc_version, min_version)) # This also will trigger using it to use our own gcc in branch below. compiler_path = None env["CC"] = None if compiler_path is None and msvc_version is None: scons_details_logger.info( "No usable C compiler, attempt fallback to winlibs gcc.") # This will download "gcc.exe" (and "clang.exe") when all others have been # rejected and MSVC is not enforced. compiler_path = getCachedDownloadedMinGW64( target_arch=target_arch, assume_yes_for_downloads=assume_yes_for_downloads, ) addToPATH(env, os.path.dirname(compiler_path), prefix=True) env = createEnvironment( mingw_mode=True, msvc_version=None, target_arch=target_arch, experimental=env.experimental_flags, ) if clang_mode: env["CC"] = os.path.join(os.path.dirname(compiler_path), "clang.exe") if env["CC"] is None: raiseNoCompilerFoundErrorExit() return env