def __TryGetEnvironmentVariable(
virtualVariantEnvironmentCache: VirtualVariantEnvironmentCache,
envVariable: str) -> str:
res = virtualVariantEnvironmentCache.TryGetCachedValue(envVariable)
if res is None:
res = IOUtil.GetEnvironmentVariable(envVariable)
return res
def __ExtractVariantIncludeDirs(
log: Log, localVariantInfo: LocalVariantInfo,
virtualVariantEnvironmentCache: VirtualVariantEnvironmentCache,
package: Package) -> List[str]:
"""
Extract all static and virtual include dirs and ensure that 'dynamic' environment variables gets cached
"""
if len(package.ResolvedAllVariantDict) <= 0:
return []
environmentVariableList = [] # type: List[str]
allIncludeDirs = [] # type: List[str]
for variant in package.ResolvedAllVariantDict.values():
if variant.Type == VariantType.Virtual:
if len(variant.Options) != 1:
raise Exception("Unsupported virtual variant type")
parsedDeps = [] # type: List[ParsedFormatString]
for externalDep in variant.Options[0].ExternalDependencies:
if externalDep.Include is not None:
if externalDep.Include.startswith("$("):
endIndex = externalDep.Include.find(')')
if endIndex < 0:
raise Exception(
"external include path invalid no ending ')' in '{0}'"
.format(externalDep.Include))
environmentVariableList.append(
externalDep.Include[:endIndex + 1])
allIncludeDirs.append(externalDep.Include)
else:
optionName = localVariantInfo.ResolvedVariantSettingsDict[
variant.Name]
selectedOption = variant.OptionDict[optionName]
for externalDep in selectedOption.ExternalDependencies:
if externalDep.Include is not None:
allIncludeDirs.append(externalDep.Include)
if package not in localVariantInfo.GeneratorReportDict:
raise Exception("Could not find a report for package '{0}".format(
package.Name))
report = localVariantInfo.GeneratorReportDict[package]
runCommand = []
if report.BuildReport is not None and report.BuildReport.BuildCommandReport is not None:
runInEnvScript = report.BuildReport.BuildCommandReport.RunInEnvScript
if runInEnvScript is not None:
if package.AbsolutePath is None:
raise Exception("internal error")
runCommand.append(IOUtil.Join(package.AbsolutePath,
runInEnvScript))
# Cache all environment
virtualVariantEnvironmentCache.CacheEnv(environmentVariableList,
runCommand)
#
return allIncludeDirs
def RunInAnotherThread(
packageQueue: Any, cancellationToken: SimpleCancellationToken,
mainLog: Log, toolConfig: ToolConfig, platformId: str,
pythonScriptRoot: str,
performClangTidyConfig: PerformClangTidyConfig,
clangExeInfo: ClangExeInfo,
customPackageFileFilter: Optional[CustomPackageFileFilter],
localVariantInfo: LocalVariantInfo) -> Tuple[int, int]:
threadId = threading.get_ident()
mainLog.LogPrintVerbose(4, "Starting thread {0}".format(threadId))
examinedCount = 0
processedCount = 0
keepWorking = True
package = None # type: Optional[Package]
try:
virtualVariantEnvironmentCache = VirtualVariantEnvironmentCache(
mainLog, pythonScriptRoot)
while keepWorking and not cancellationToken.IsCancelled():
try:
# Since the queue is preloaded this is ok
package = packageQueue.get_nowait()
except:
package = None
if package is None:
keepWorking = False
else:
if mainLog.Verbosity >= 4:
mainLog.LogPrint(
"- clang-tidy on package '{0}' on thread {1}".
format(package.Name, threadId))
else:
mainLog.LogPrint(
"- clang-tidy on package '{0}'".format(
package.Name))
captureLog = CaptureLog(mainLog.Title, mainLog.Verbosity)
try:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(
captureLog, package, performClangTidyConfig.
ClangTidyConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
processedCount += 1
_RunClangTidy(captureLog, toolConfig, platformId,
performClangTidyConfig, clangExeInfo,
package, filteredFiles, None,
localVariantInfo,
virtualVariantEnvironmentCache, True)
examinedCount += 1
finally:
try:
if len(captureLog.Captured) > 0:
capturedLog = "Package: '{0}' result:\n{1}".format(
package.Name,
"\n".join(captureLog.Captured))
mainLog.DoPrint(capturedLog)
except:
pass
except Exception as ex:
cancellationToken.Cancel()
mainLog.DoPrintError("Cancelling tasks due to exception: {0}")
raise
finally:
mainLog.LogPrintVerbose(4, "Ending thread {0}".format(threadId))
return (examinedCount, processedCount)
def ProcessAllPackages(
log: Log, toolConfig: ToolConfig, platformId: str,
pythonScriptRoot: str,
performClangTidyConfig: PerformClangTidyConfig,
clangExeInfo: ClangExeInfo, packageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
localVariantInfo: LocalVariantInfo, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
virtualVariantEnvironmentCache = VirtualVariantEnvironmentCache(
log, pythonScriptRoot)
# do standard serial processing
for package in packageList:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, performClangTidyConfig.
ClangTidyConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
_RunClangTidy(log, toolConfig, platformId,
performClangTidyConfig, clangExeInfo,
package, filteredFiles, None,
localVariantInfo,
virtualVariantEnvironmentCache)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangTidyHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
platformId, pythonScriptRoot,
performClangTidyConfig, clangExeInfo,
customPackageFileFilter, localVariantInfo)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount