def addClangClPathFromMSVC(env, target_arch, show_scons_mode):
cl_exe = getExecutablePath("cl", env=env)
if cl_exe is None:
scons_logger.warning(
"Visual Studio required for for Clang on Windows.")
return
clang_dir = cl_exe = os.path.join(cl_exe[:cl_exe.lower().rfind("msvc")],
"Llvm")
if target_arch == "x86_64":
clang_dir = os.path.join(clang_dir, "x64", "bin")
else:
clang_dir = os.path.join(clang_dir, "bin")
if os.path.exists(clang_dir):
if show_scons_mode:
scons_logger.info("Adding MSVC directory %r for Clang to PATH." %
clang_dir)
addToPATH(env, clang_dir, prefix=True)
else:
if show_scons_mode:
scons_logger.info("No Clang component for MSVC found." % clang_dir)
def enableCcache(
the_compiler,
env,
source_dir,
python_prefix,
show_scons_mode,
assume_yes_for_downloads,
):
# The ccache needs absolute path, otherwise it will not work.
ccache_logfile = os.path.abspath(
os.path.join(source_dir, "ccache-%d.txt" % os.getpid()))
setEnvironmentVariable(env, "CCACHE_LOGFILE", ccache_logfile)
env["CCACHE_LOGFILE"] = ccache_logfile
# First check if it's not already supposed to be a ccache, then do nothing.
cc_path = getExecutablePath(the_compiler, env=env)
cc_is_link, cc_link_path = getLinkTarget(cc_path)
if cc_is_link and os.path.basename(cc_link_path) == "ccache":
if show_scons_mode:
scons_logger.info(
"Chosen compiler %s is pointing to ccache %s already." %
(cc_path, cc_link_path))
return True
return _injectCcache(
the_compiler=the_compiler,
cc_path=cc_path,
env=env,
python_prefix=python_prefix,
show_scons_mode=show_scons_mode,
assume_yes_for_downloads=assume_yes_for_downloads,
)
def reportSlowCompilation(cmd, delta_time):
# TODO: for linking, we ought to apply a different timer maybe and attempt to extra
# the source file that is causing the issues: pylint: disable=unused-argument
if _current != _total:
scons_logger.info(
"Slow C compilation detected, used %.0fs so far, scalability problem."
% delta_time
)
def reportCCompiler(env, context):
cc_output = env.the_cc_name
if env.the_cc_name == "cl":
cc_output = "%s %s" % (env.the_cc_name, getMsvcVersionString(env))
else:
cc_output = env.the_cc_name
scons_logger.info("%s C compiler: %s (%s)." %
(context, env.the_compiler, cc_output))
def checkClcache(install_dir):
candidate = os.path.join(install_dir, "scripts", "clcache.exe")
if show_scons_mode:
scons_logger.info(
"Checking if clcache is at '%s' python installation path."
% candidate)
if os.path.exists(candidate):
return True
def updateSconsProgressBar():
# Check if link is next, pylint: disable=global-statement
global _current
_current += 1
reportProgressBar(item=None, update=True)
if _current == _total:
closeSconsProgressBar()
scons_logger.info(
"%s linking program (no progress information available)." % _stage
)
def switchFromGccToGpp(env):
if not env.gcc_mode or env.clang_mode:
env.gcc_version = None
return
env.gcc_version = myDetectVersion(env, env.the_compiler)
if env.gcc_version is None:
scons_logger.sysexit("""\
Error, failed to detect gcc version of backend compiler %r.
""" % env.the_compiler)
if "++" in env.the_cc_name:
scons_logger.sysexit("""\
Error, compiler %s is apparently a C++ compiler, specify a C compiler instead.
""" % env.the_cc_name)
# Enforce the minimum version, selecting a potentially existing g++-4.5
# binary if it's not high enough. This is esp. useful under Debian which
# allows all compiler to exist next to each other and where g++ might not be
# good enough, but g++-4.5 would be.
if env.gcc_version < (4, 4):
scons_logger.sysexit("""\
The gcc compiler %s (version %s) doesn't have the sufficient \
version (>= 4.4).""" % (env.the_compiler, env.gcc_version))
# CondaCC or newer.
if env.mingw_mode and env.gcc_version < (5, 3):
scons_logger.sysexit("""\
The MinGW64 compiler %s (version %s) doesn't have the sufficient \
version (>= 5.3).""" % (env.the_compiler, env.gcc_version))
if env.gcc_version < (5, ):
scons_logger.info(
"The provided gcc is too old, switching to its g++ instead.")
# Switch to g++ from gcc then if possible, when C11 mode is false.
the_gpp_compiler = os.path.join(
os.path.dirname(env.the_compiler),
os.path.basename(env.the_compiler).replace("gcc", "g++"),
)
if getExecutablePath(the_gpp_compiler, env=env):
env.the_compiler = the_gpp_compiler
env.the_cc_name = env.the_cc_name.replace("gcc", "g++")
else:
scons_logger.sysexit(
"Error, your gcc is too old for C11 support, and no related g++ to workaround that is found."
)
def runSpawnMonitored(spawn, sh, escape, cmd, args, env):
thread = SpawnThread(spawn, sh, escape, cmd, args, env)
thread.start()
# Allow a minute before warning for long compile time.
thread.join(60)
if thread.is_alive():
scons_logger.info(
"Slow C compilation detected, used %.0fs so far, this might indicate scalability problems."
% thread.timer_report.getTimer().getDelta())
thread.join()
return thread.getSpawnResult()
def runProcessMonitored(cmdline, env):
thread = SubprocessThread(cmdline, env)
thread.start()
# Allow a minute before warning for long compile time.
thread.join(60)
if thread.is_alive():
scons_logger.info(
"Slow C compilation detected, used %.0fs so far, this might indicate scalability problems."
% thread.timer_report.getTimer().getDelta())
thread.join()
return thread.getProcessResult()
def switchFromGccToGpp(gcc_version, the_compiler, the_cc_name, env):
if gcc_version is not None and gcc_version < (5,):
scons_logger.info("The provided gcc is too old, switching to g++ instead.")
# Switch to g++ from gcc then if possible, when C11 mode is false.
the_gpp_compiler = os.path.join(
os.path.dirname(the_compiler),
os.path.basename(the_compiler).replace("gcc", "g++"),
)
if getExecutablePath(the_gpp_compiler, env=env):
the_compiler = the_gpp_compiler
the_cc_name = the_cc_name.replace("gcc", "g++")
else:
scons_logger.sysexit(
"Error, your gcc is too old for C11 support, and no related g++ to workaround that is found."
)
return the_compiler, the_cc_name
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir=source_dir,
key="CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)",
"cache hit (preprocessed)"):
result = "cache hit"
# Newer ccache has these, but they duplicate:
if result in (
"direct_cache_hit",
"direct_cache_miss",
"preprocessed_cache_hit",
"preprocessed_cache_miss",
"primary_storage_miss",
):
continue
if result == "primary_storage_hit":
result = "cache hit"
if result == "cache_miss":
result = "cache miss"
# Usage of incbin causes this for the constants blob integration.
if result in ("unsupported code directive", "disabled"):
continue
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d" %
(result, count))
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(source_dir=source_dir,
key="CLCACHE_STATS")
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename):
stats = eval( # lazy, pylint: disable=eval-used
getFileContents(clcache_stats_filename))
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss))
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d" %
(result, count))
clcache_logfile = getSconsReportValue(source_dir, "CLCACHE_LOG")
if clcache_logfile is not None:
stats = _getClcacheStatistics(clcache_logfile)
if not stats:
scons_logger.warning("You are not using clcache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
counts[result] += 1
scons_logger.info("Compiled %d C files using clcache." %
len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using clcache) with result '%s': %d" %
(result, count))
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)", "cache hit (preprocessed)"):
result = "cache hit"
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d"
% (result, count)
)
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(source_dir, "CLCACHE_STATS")
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename
):
stats = eval( # lazy, pylint: disable=eval-used
getFileContents(clcache_stats_filename)
)
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss)
)
def checkWindowsCompilerFound(env, target_arch, assume_yes_for_downloads):
"""Remove compiler of wrong arch or too old gcc and replace with downloaded winlibs gcc."""
if os.name == "nt":
# On Windows, in case MSVC was not found and not previously forced, use the
# winlibs MinGW64 as a download, and use it as a fallback.
compiler_path = getExecutablePath(env["CC"], env=env)
# Drop wrong arch compiler, most often found by scans. There might be wrong gcc or cl on the PATH.
if compiler_path is not None:
the_cc_name = os.path.basename(compiler_path)
decision, linker_arch, compiler_arch = decideArchMismatch(
target_arch=target_arch,
mingw_mode=isGccName(the_cc_name),
msvc_mode=not isGccName(the_cc_name),
the_cc_name=the_cc_name,
compiler_path=compiler_path,
)
if decision:
# This will trigger using it to use our own gcc in branch below.
compiler_path = None
scons_logger.info(
"Mismatch between Python binary (%r -> %r) and C compiler (%r -> %r) arches, ignored!"
% (
os.environ["NUITKA_PYTHON_EXE_PATH"],
linker_arch,
compiler_path,
compiler_arch,
))
if compiler_path is not None:
the_cc_name = os.path.basename(compiler_path)
if isGccName(the_cc_name):
gcc_version = myDetectVersion(env, compiler_path)
min_version = (8, )
if gcc_version is not None and gcc_version < min_version:
# This also will trigger using it to use our own gcc in branch below.
compiler_path = None
scons_logger.info(
"Too old gcc %r (%r < %r) ignored!" %
(compiler_path, gcc_version, min_version))
if compiler_path is None:
# This will succeed to find "gcc.exe" when conda install m2w64-gcc has
# been done.
compiler_path = getDownloadedGccPath(
target_arch=target_arch,
assume_yes_for_downloads=assume_yes_for_downloads,
)
addToPATH(env, os.path.dirname(compiler_path), prefix=True)
env = createEnvironment(
tools=["mingw"],
mingw_mode=True,
msvc_version=None,
target_arch=target_arch,
)
env["CC"] = compiler_path
return env
def _injectCcache(the_compiler, cc_path, env, python_prefix, show_scons_mode):
# We deal with a lot of cases here, pylint: disable=too-many-branches
ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if ccache_binary is None:
ccache_binary = getExecutablePath("ccache", env=env)
if ccache_binary is None:
for candidate in _getCcacheGuessedPaths(python_prefix):
if show_scons_mode:
scons_logger.info(
"Checking if ccache is at '%s' guessed path." %
candidate)
if os.path.exists(candidate):
ccache_binary = candidate
if show_scons_mode:
scons_logger.info(
"Using ccache '%s' from guessed path." %
ccache_binary)
break
else:
if show_scons_mode:
scons_logger.info(
"Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable."
% ccache_binary)
if ccache_binary is not None and os.path.exists(ccache_binary):
# In case we are on Windows, make sure the Anaconda form runs outside of Anaconda
# environment, by adding DLL folder to PATH.
if os.name == "nt":
conda_dll_dir = os.path.normpath(
os.path.join(ccache_binary, "..", "..", "Library", "mingw-w64",
"bin"))
if os.path.exists(conda_dll_dir):
addToPATH(env, conda_dll_dir, prefix=False)
assert getExecutablePath(os.path.basename(the_compiler),
env=env) == cc_path
# Since we use absolute paths for CC, pass it like this, as ccache does not like absolute.
env["CXX"] = env["CC"] = "%s %s" % (ccache_binary,
os.path.basename(cc_path))
# Spare ccache the detection of the compiler, seems it will also misbehave when it's
# prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage.
env["LINK"] = cc_path
# Do not consider scons cache anymore.
if show_scons_mode:
scons_logger.info(
"Found ccache '%s' to cache C compilation result." %
ccache_binary)
scons_logger.info(
"Providing real CC path '%s' via PATH extension." % cc_path)
result = True
else:
if isWin32Windows():
scons_logger.warning(
"Didn't find ccache for C level caching, follow Nuitka user manual description."
)
result = False
return result
def enableClcache(the_compiler, env, source_dir, python_prefix,
show_scons_mode):
# Many branches to deal with, pylint: disable=too-many-branches
# The ccache needs absolute path, otherwise it will not work.
clcache_logfile = os.path.abspath(
os.path.join(source_dir, "clcache-%d.txt" % os.getpid()))
# Our spawn function will pick it up from the output.
setEnvironmentVariable(env, "CLCACHE_LOG", clcache_logfile)
env["CLCACHE_LOG"] = clcache_logfile
clcache_binary = os.environ.get("NUITKA_CLCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if clcache_binary is None:
clcache_binary = getExecutablePath("clcache", env)
if clcache_binary is None:
for candidate in _getClcacheGuessedPaths(python_prefix):
if show_scons_mode:
scons_logger.info(
"Checking if clcache is at '%s' guessed path." %
candidate)
if os.path.exists(candidate):
clcache_binary = candidate
if show_scons_mode:
scons_logger.info(
"Using clcache '%s' from guessed path." %
clcache_binary)
break
if clcache_binary is None and os.name == "nt":
def checkClcache(install_dir):
candidate = os.path.join(install_dir, "scripts", "clcache.exe")
if show_scons_mode:
scons_logger.info(
"Checking if clcache is at '%s' python installation path."
% candidate)
if os.path.exists(candidate):
return True
candidate = getPythonInstallPathWindows(supported=("3.6", "3.7",
"3.8", "3.9"),
decider=checkClcache)
if candidate is not None:
clcache_binary = os.path.join(candidate, "scripts",
"clcache.exe")
if show_scons_mode:
scons_logger.info(
"Using clcache '%s' from registry path." %
clcache_binary)
else:
if show_scons_mode:
scons_logger.info(
"Using clcache '%s' from NUITKA_CLCACHE_BINARY environment variable."
% clcache_binary)
if clcache_binary is not None and os.path.exists(clcache_binary):
cl_binary = getExecutablePath(the_compiler, env)
# The compiler is passed via environment.
setEnvironmentVariable(env, "CLCACHE_CL", cl_binary)
env["CXX"] = env["CC"] = clcache_binary
if show_scons_mode:
scons_logger.info(
"Found clcache '%s' to cache C compilation result." %
clcache_binary)
scons_logger.info(
"Providing real cl.exe path '%s' via environment." % cl_binary)
result = True
else:
scons_logger.warning(
"Didn't find clcache for C level caching, follow Nuitka user manual description."
)
result = False
return result
def _injectCcache(the_compiler, cc_path, env, python_prefix, show_scons_mode,
assume_yes_for_downloads):
ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if ccache_binary is None:
ccache_binary = getExecutablePath("ccache", env=env)
if ccache_binary is None:
for candidate in _getCcacheGuessedPaths(python_prefix):
if show_scons_mode:
scons_logger.info(
"Checking if ccache is at '%s' guessed path." %
candidate)
if os.path.exists(candidate):
ccache_binary = candidate
if show_scons_mode:
scons_logger.info(
"Using ccache '%s' from guessed path." %
ccache_binary)
break
if ccache_binary is None and os.name == "nt":
url = "https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12-windows-32.zip"
ccache_binary = getCachedDownload(
url=url,
is_arch_specific=False,
specifity=url.rsplit("/", 2)[1],
flatten=True,
binary="ccache.exe",
message=
"Nuitka will make use of ccache to speed up repeated compilation.",
reject=None,
assume_yes_for_downloads=assume_yes_for_downloads,
)
else:
if show_scons_mode:
scons_logger.info(
"Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable."
% ccache_binary)
if ccache_binary is not None and os.path.exists(ccache_binary):
# Make sure the
# In case we are on Windows, make sure the Anaconda form runs outside of Anaconda
# environment, by adding DLL folder to PATH.
assert getExecutablePath(os.path.basename(the_compiler),
env=env) == cc_path
# We use absolute paths for CC, pass it like this, as ccache does not like absolute.
env["CXX"] = env["CC"] = "%s %s" % (ccache_binary,
os.path.basename(cc_path))
# Spare ccache the detection of the compiler, seems it will also misbehave when it's
# prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage.
env["LINK"] = cc_path
# Do not consider scons cache anymore.
if show_scons_mode:
scons_logger.info(
"Found ccache '%s' to cache C compilation result." %
ccache_binary)
scons_logger.info(
"Providing real CC path '%s' via PATH extension." % cc_path)
result = True
else:
if isWin32Windows():
scons_logger.warning(
"Didn't find ccache for C level caching, follow Nuitka user manual description."
)
result = False
return result
def checkWindowsCompilerFound(env, target_arch, clang_mode, msvc_version,
assume_yes_for_downloads):
"""Remove compiler of wrong arch or too old gcc and replace with downloaded winlibs gcc."""
if os.name == "nt":
# On Windows, in case MSVC was not found and not previously forced, use the
# winlibs MinGW64 as a download, and use it as a fallback.
compiler_path = getExecutablePath(env["CC"], env=env)
scons_details_logger.info("Checking usability of %r from %r" %
(compiler_path, env["CC"]))
# Drop wrong arch compiler, most often found by scans. There might be wrong gcc or cl on the PATH.
if compiler_path is not None:
the_cc_name = os.path.basename(compiler_path)
decision, linker_arch, compiler_arch = decideArchMismatch(
target_arch=target_arch,
mingw_mode=isGccName(the_cc_name),
msvc_mode=not isGccName(the_cc_name),
the_cc_name=the_cc_name,
compiler_path=compiler_path,
)
if decision:
scons_logger.info(
"Mismatch between Python binary (%r -> %r) and C compiler (%r -> %r) arches, that compiler is ignored!"
% (
os.environ["NUITKA_PYTHON_EXE_PATH"],
linker_arch,
compiler_path,
compiler_arch,
))
# This will trigger using it to use our own gcc in branch below.
compiler_path = None
env["CC"] = None
if compiler_path is not None and msvc_version is not None:
if msvc_version == "latest":
scons_logger.info("MSVC version resolved to %s." %
getMsvcVersionString(env))
# Requested a specific MSVC version, check if that worked.
elif msvc_version != getMsvcVersionString(env):
scons_logger.info(
"Failed to find requested MSVC version (%r != %r)." %
(msvc_version, getMsvcVersionString(env)))
# This will trigger error exit in branch below.
compiler_path = None
env["CC"] = None
if compiler_path is not None:
the_cc_name = os.path.basename(compiler_path)
if isGccName(the_cc_name):
gcc_version = myDetectVersion(env, compiler_path)
min_version = (11, 2)
if gcc_version is not None and (gcc_version < min_version
or "force-winlibs-gcc"
in env.experimental_flags):
scons_logger.info(
"Too old gcc %r (%r < %r) ignored!" %
(compiler_path, gcc_version, min_version))
# This also will trigger using it to use our own gcc in branch below.
compiler_path = None
env["CC"] = None
if compiler_path is None and msvc_version is None:
scons_details_logger.info(
"No usable C compiler, attempt fallback to winlibs gcc.")
# This will download "gcc.exe" (and "clang.exe") when all others have been
# rejected and MSVC is not enforced.
compiler_path = getCachedDownloadedMinGW64(
target_arch=target_arch,
assume_yes_for_downloads=assume_yes_for_downloads,
)
addToPATH(env, os.path.dirname(compiler_path), prefix=True)
env = createEnvironment(
mingw_mode=True,
msvc_version=None,
target_arch=target_arch,
experimental=env.experimental_flags,
)
if clang_mode:
env["CC"] = os.path.join(os.path.dirname(compiler_path),
"clang.exe")
if env["CC"] is None:
raiseNoCompilerFoundErrorExit()
return env
