def Process(log: Log, toolConfig: ToolConfig,
customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, packageList: List[Package],
repairEnabled: bool, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
for package in packageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
log.LogPrint("- {0}".format(package.Name))
_RunClangFormat(log, toolConfig, clangFormatConfiguration,
clangExeInfo, package, filteredFiles,
repairEnabled)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangFormatHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
customPackageFileFilter, clangFormatConfiguration,
clangExeInfo, repairEnabled)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount
def RunInAnotherThread(packageQueue: Any,
cancellationToken: SimpleCancellationToken,
mainLog: Log, toolConfig: ToolConfig, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, clangExeInfo: ClangExeInfo, repairEnabled: bool) -> Tuple[int,int]:
threadId = threading.get_ident()
mainLog.LogPrintVerbose(4, "Starting thread {0}".format(threadId))
examinedCount = 0
processedCount = 0
keepWorking = True
package = None # type: Optional[Package]
try:
while keepWorking and not cancellationToken.IsCancelled():
try:
# Since the queue is preloaded this is ok
package = packageQueue.get_nowait()
except:
package = None
if package is None:
keepWorking = False
else:
if mainLog.Verbosity >= 4:
mainLog.LogPrint("- clang-format on package '{0}' on thread {1}".format(package.Name, threadId))
else:
mainLog.LogPrint("- clang-format on package '{0}'".format(package.Name))
captureLog = CaptureLog(mainLog.Title, mainLog.Verbosity)
try:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(captureLog, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
processedCount += 1
_RunClangFormat(captureLog, toolConfig, clangFormatConfiguration, clangExeInfo, package, filteredFiles, repairEnabled)
examinedCount += 1
finally:
try:
if len(captureLog.Captured) > 0:
capturedLog = "Package: '{0}' result:\n{1}".format(package.Name, "\n".join(captureLog.Captured))
mainLog.DoPrint(capturedLog)
except:
pass
except Exception as ex:
cancellationToken.Cancel()
mainLog.DoPrintError("Cancelling tasks due to exception: {0}")
raise
finally:
mainLog.LogPrintVerbose(4, "Ending thread {0}".format(threadId))
return (examinedCount, processedCount)
def ProcessAllPackages(
log: Log, toolConfig: ToolConfig, platformId: str,
pythonScriptRoot: str,
performClangTidyConfig: PerformClangTidyConfig,
clangExeInfo: ClangExeInfo, packageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
localVariantInfo: LocalVariantInfo, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
virtualVariantEnvironmentCache = VirtualVariantEnvironmentCache(
log, pythonScriptRoot)
# do standard serial processing
for package in packageList:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, performClangTidyConfig.
ClangTidyConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
_RunClangTidy(log, toolConfig, platformId,
performClangTidyConfig, clangExeInfo,
package, filteredFiles, None,
localVariantInfo,
virtualVariantEnvironmentCache)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangTidyHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
platformId, pythonScriptRoot,
performClangTidyConfig, clangExeInfo,
customPackageFileFilter, localVariantInfo)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount