def _getCcacheStatistics(ccache_logfile):
data = {}
if os.path.exists(ccache_logfile):
re_command = re.compile(r"\[.*? (\d+) *\] Command line: (.*)$")
re_result = re.compile(r"\[.*? (\d+) *\] Result: (.*)$")
re_anything = re.compile(r"\[.*? (\d+) *\] (.*)$")
# Remember command from the pid, so later decision logged against pid
# can be matched against it.
commands = {}
for line in getFileContentByLine(ccache_logfile):
match = re_command.match(line)
if match:
pid, command = match.groups()
commands[pid] = command
match = re_result.match(line)
if match:
pid, result = match.groups()
result = result.strip()
try:
command = data[commands[pid]]
except KeyError:
# It seems writing to the file can be lossy, so we can have results for
# unknown commands, but we don't use the command yet anyway, so just
# be unique.
command = "unknown command leading to " + line
# Older ccache on e.g. RHEL6 wasn't explicit about linking.
if result == "unsupported compiler option":
if " -o " in command or "unknown command" in command:
result = "called for link"
# But still try to catch this with log output if it happens.
if result == "unsupported compiler option":
scons_logger.warning(
"Encountered unsupported compiler option for ccache in '%s'."
% command
)
all_text = []
for line2 in open(ccache_logfile):
match = re_anything.match(line2)
if match:
pid2, result = match.groups()
if pid == pid2:
all_text.append(result)
scons_logger.warning("Full scons output: %s" % all_text)
if result != "called for link":
data[command] = result
return data
def addClangClPathFromMSVC(env, target_arch, show_scons_mode):
cl_exe = getExecutablePath("cl", env=env)
if cl_exe is None:
scons_logger.warning(
"Visual Studio required for for Clang on Windows.")
return
clang_dir = cl_exe = os.path.join(cl_exe[:cl_exe.lower().rfind("msvc")],
"Llvm")
if target_arch == "x86_64":
clang_dir = os.path.join(clang_dir, "x64", "bin")
else:
clang_dir = os.path.join(clang_dir, "bin")
if os.path.exists(clang_dir):
if show_scons_mode:
scons_logger.info("Adding MSVC directory %r for Clang to PATH." %
clang_dir)
addToPATH(env, clang_dir, prefix=True)
else:
if show_scons_mode:
scons_logger.info("No Clang component for MSVC found." % clang_dir)
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir=source_dir,
key="CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)",
"cache hit (preprocessed)"):
result = "cache hit"
# Newer ccache has these, but they duplicate:
if result in (
"direct_cache_hit",
"direct_cache_miss",
"preprocessed_cache_hit",
"preprocessed_cache_miss",
"primary_storage_miss",
):
continue
if result == "primary_storage_hit":
result = "cache hit"
if result == "cache_miss":
result = "cache miss"
# Usage of incbin causes this for the constants blob integration.
if result in ("unsupported code directive", "disabled"):
continue
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d" %
(result, count))
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(source_dir=source_dir,
key="CLCACHE_STATS")
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename):
stats = eval( # lazy, pylint: disable=eval-used
getFileContents(clcache_stats_filename))
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss))
def extractClcacheLogFromOutput(data):
clcache_output = []
normal_output = []
for line in data.split(b"\n"):
# Remove the "\r", clcache and compiler may or may not output it.
line = line.strip()
if b"clcache.py" in line:
clcache_output.append(line)
else:
normal_output.append(line)
# Make sure we have Windows new lines for the compiler output though.
data = b"\r\n".join(normal_output)
if data:
data += b"\r\n"
for clcache_line in clcache_output:
match = re.search(b"Reusing cached object.*?for object file (.*)",
clcache_line)
if match:
_writeClcacheLog(match.group(1), "cache hit")
return data
match = re.search(b"Adding file (.*?) to cache", clcache_line)
if match:
_writeClcacheLog(match.group(1), "cache miss")
return data
match = re.search(b"Real compiler returned code (\\d+)", clcache_line)
if match and match.group(1) != b"0":
_writeClcacheLog(match.group(1), "compile error")
return data
match = re.search(b"Compiler source files: \\['(.*?)'\\]",
clcache_line)
if match:
_writeClcacheLog(match.group(1), "cache miss")
return data
if clcache_output:
# Sometimes no message at all might be recognized.
scons_logger.warning(
"Caching with clcache could not be decoded, got this:")
scons_logger.warning(b"\n".join(clcache_output))
return data
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)", "cache hit (preprocessed)"):
result = "cache hit"
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d"
% (result, count)
)
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(source_dir, "CLCACHE_STATS")
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename
):
stats = eval( # lazy, pylint: disable=eval-used
getFileContents(clcache_stats_filename)
)
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss)
)
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d" %
(result, count))
clcache_logfile = getSconsReportValue(source_dir, "CLCACHE_LOG")
if clcache_logfile is not None:
stats = _getClcacheStatistics(clcache_logfile)
if not stats:
scons_logger.warning("You are not using clcache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
counts[result] += 1
scons_logger.info("Compiled %d C files using clcache." %
len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using clcache) with result '%s': %d" %
(result, count))
def extractClcacheLogFromOutput(data):
clcache_output = []
normal_output = []
for line in data.split(b"\r\n"):
if b"clcache.py" in line:
clcache_output.append(line)
else:
normal_output.append(line)
data = b"\r\n".join(normal_output)
if data:
data += b"\r\n"
for clcache_line in clcache_output:
match = re.search(b"Reusing cached object.*?for object file (.*?)\\r",
clcache_line)
if match:
_writeClcacheLog(match.group(1), "cache hit")
return data
match = re.search(b"Adding file (.*?) to cache", clcache_line)
if match:
_writeClcacheLog(match.group(1), "cache miss")
return data
match = re.search(b"Compiler source files: \\['(.*?)'\\]",
clcache_line)
if match:
_writeClcacheLog(match.group(1), "cache miss")
return data
if clcache_output:
# Sometimes no message at all might be recognized.
scons_logger.warning("Caching with clcache could not be decoded.")
return data
def _injectCcache(the_compiler, cc_path, env, python_prefix, assume_yes_for_downloads):
ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if ccache_binary is None:
ccache_binary = getExecutablePath("ccache", env=env)
if ccache_binary is None:
for candidate in _getCcacheGuessedPaths(python_prefix):
scons_details_logger.info(
"Checking if ccache is at '%s' guessed path." % candidate
)
if os.path.exists(candidate):
ccache_binary = candidate
scons_details_logger.info(
"Using ccache '%s' from guessed path." % ccache_binary
)
break
if ccache_binary is None and os.name == "nt":
url = "https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12-windows-32.zip"
ccache_binary = getCachedDownload(
url=url,
is_arch_specific=False,
specifity=url.rsplit("/", 2)[1],
flatten=True,
binary="ccache.exe",
message="Nuitka will make use of ccache to speed up repeated compilation.",
reject=None,
assume_yes_for_downloads=assume_yes_for_downloads,
)
else:
scons_details_logger.info(
"Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable."
% ccache_binary
)
if ccache_binary is not None and os.path.exists(ccache_binary):
# Make sure the
# In case we are on Windows, make sure the Anaconda form runs outside of Anaconda
# environment, by adding DLL folder to PATH.
assert getExecutablePath(os.path.basename(the_compiler), env=env) == cc_path
# We use absolute paths for CC, pass it like this, as ccache does not like absolute.
env["CXX"] = env["CC"] = '"%s" "%s"' % (ccache_binary, cc_path)
# Spare ccache the detection of the compiler, seems it will also misbehave when it's
# prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage.
env["LINK"] = cc_path
scons_details_logger.info(
"Found ccache '%s' to cache C compilation result." % ccache_binary
)
scons_details_logger.info(
"Providing real CC path '%s' via PATH extension." % cc_path
)
else:
if isWin32Windows():
scons_logger.warning(
"Didn't find ccache for C level caching, follow Nuitka user manual description."
)
def _enableLtoSettings(
env,
lto_mode,
pgo_mode,
job_count,
):
# This is driven by branches on purpose and pylint: disable=too-many-branches,too-many-statements
orig_lto_mode = lto_mode
if lto_mode == "no":
lto_mode = False
reason = "disabled"
elif lto_mode == "yes":
lto_mode = True
reason = "enabled"
elif pgo_mode in ("use", "generate"):
lto_mode = True
reason = "PGO implies LTO"
elif env.msvc_mode and getMsvcVersion(env) >= 14:
lto_mode = True
reason = "known to be supported"
elif env.nuitka_python:
lto_mode = True
reason = "known to be supported (Nuitka-Python)"
elif (env.debian_python and env.gcc_mode and not env.clang_mode
and env.gcc_version >= (6, )):
lto_mode = True
reason = "known to be supported (Debian)"
elif env.gcc_mode and env.the_cc_name == "gnu-cc":
lto_mode = True
reason = "known to be supported (CondaCC)"
elif env.mingw_mode and env.clang_mode:
lto_mode = False
reason = "known to not be supported (new MinGW64 Clang)"
elif env.gcc_mode and env.mingw_mode and env.gcc_version >= (11, 2):
lto_mode = True
reason = "known to be supported (new MinGW64)"
else:
lto_mode = False
reason = "not known to be supported"
if lto_mode and env.gcc_mode and not env.clang_mode and env.gcc_version < (
4, 6):
scons_logger.warning("""\
The gcc compiler %s (version %s) doesn't have the sufficient \
version for lto mode (>= 4.6). Disabled.""" % (env["CXX"], env["CXXVERSION"]))
lto_mode = False
reason = "gcc 4.6 is doesn't have good enough LTO support"
if env.gcc_mode and lto_mode:
env.Append(CCFLAGS=["-flto"])
if env.clang_mode:
env.Append(LINKFLAGS=["-flto"])
else:
env.Append(CCFLAGS=["-fuse-linker-plugin", "-fno-fat-lto-objects"])
env.Append(LINKFLAGS=["-fuse-linker-plugin"])
env.Append(LINKFLAGS=["-flto=%d" % job_count])
# Need to tell the linker these things are OK.
env.Append(LINKFLAGS=["-fpartial-inlining", "-freorder-functions"])
# Tell compiler to use link time optimization for MSVC
if env.msvc_mode and lto_mode:
env.Append(CCFLAGS=["/GL"])
if not env.clangcl_mode:
env.Append(LINKFLAGS=["/LTCG"])
if orig_lto_mode == "auto":
scons_details_logger.info(
"LTO mode auto was resolved to mode: '%s' (%s)." %
("yes" if lto_mode else "no", reason))
env.lto_mode = lto_mode
# PGO configuration
_enablePgoSettings(env, pgo_mode)
def enableClcache(the_compiler, env, source_dir, python_prefix,
show_scons_mode):
# Many branches to deal with, pylint: disable=too-many-branches
# The ccache needs absolute path, otherwise it will not work.
clcache_logfile = os.path.abspath(
os.path.join(source_dir, "clcache-%d.txt" % os.getpid()))
# Our spawn function will pick it up from the output.
setEnvironmentVariable(env, "CLCACHE_LOG", clcache_logfile)
env["CLCACHE_LOG"] = clcache_logfile
clcache_binary = os.environ.get("NUITKA_CLCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if clcache_binary is None:
clcache_binary = getExecutablePath("clcache", env)
if clcache_binary is None:
for candidate in _getClcacheGuessedPaths(python_prefix):
if show_scons_mode:
scons_logger.info(
"Checking if clcache is at '%s' guessed path." %
candidate)
if os.path.exists(candidate):
clcache_binary = candidate
if show_scons_mode:
scons_logger.info(
"Using clcache '%s' from guessed path." %
clcache_binary)
break
if clcache_binary is None and os.name == "nt":
def checkClcache(install_dir):
candidate = os.path.join(install_dir, "scripts", "clcache.exe")
if show_scons_mode:
scons_logger.info(
"Checking if clcache is at '%s' python installation path."
% candidate)
if os.path.exists(candidate):
return True
candidate = getPythonInstallPathWindows(supported=("3.6", "3.7",
"3.8", "3.9"),
decider=checkClcache)
if candidate is not None:
clcache_binary = os.path.join(candidate, "scripts",
"clcache.exe")
if show_scons_mode:
scons_logger.info(
"Using clcache '%s' from registry path." %
clcache_binary)
else:
if show_scons_mode:
scons_logger.info(
"Using clcache '%s' from NUITKA_CLCACHE_BINARY environment variable."
% clcache_binary)
if clcache_binary is not None and os.path.exists(clcache_binary):
cl_binary = getExecutablePath(the_compiler, env)
# The compiler is passed via environment.
setEnvironmentVariable(env, "CLCACHE_CL", cl_binary)
env["CXX"] = env["CC"] = clcache_binary
if show_scons_mode:
scons_logger.info(
"Found clcache '%s' to cache C compilation result." %
clcache_binary)
scons_logger.info(
"Providing real cl.exe path '%s' via environment." % cl_binary)
result = True
else:
scons_logger.warning(
"Didn't find clcache for C level caching, follow Nuitka user manual description."
)
result = False
return result
def _injectCcache(the_compiler, cc_path, env, python_prefix, show_scons_mode):
# We deal with a lot of cases here, pylint: disable=too-many-branches
ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if ccache_binary is None:
ccache_binary = getExecutablePath("ccache", env=env)
if ccache_binary is None:
for candidate in _getCcacheGuessedPaths(python_prefix):
if show_scons_mode:
scons_logger.info(
"Checking if ccache is at '%s' guessed path." %
candidate)
if os.path.exists(candidate):
ccache_binary = candidate
if show_scons_mode:
scons_logger.info(
"Using ccache '%s' from guessed path." %
ccache_binary)
break
else:
if show_scons_mode:
scons_logger.info(
"Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable."
% ccache_binary)
if ccache_binary is not None and os.path.exists(ccache_binary):
# In case we are on Windows, make sure the Anaconda form runs outside of Anaconda
# environment, by adding DLL folder to PATH.
if os.name == "nt":
conda_dll_dir = os.path.normpath(
os.path.join(ccache_binary, "..", "..", "Library", "mingw-w64",
"bin"))
if os.path.exists(conda_dll_dir):
addToPATH(env, conda_dll_dir, prefix=False)
assert getExecutablePath(os.path.basename(the_compiler),
env=env) == cc_path
# Since we use absolute paths for CC, pass it like this, as ccache does not like absolute.
env["CXX"] = env["CC"] = "%s %s" % (ccache_binary,
os.path.basename(cc_path))
# Spare ccache the detection of the compiler, seems it will also misbehave when it's
# prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage.
env["LINK"] = cc_path
# Do not consider scons cache anymore.
if show_scons_mode:
scons_logger.info(
"Found ccache '%s' to cache C compilation result." %
ccache_binary)
scons_logger.info(
"Providing real CC path '%s' via PATH extension." % cc_path)
result = True
else:
if isWin32Windows():
scons_logger.warning(
"Didn't find ccache for C level caching, follow Nuitka user manual description."
)
result = False
return result
