def Process(log: Log, toolConfig: ToolConfig,
customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, packageList: List[Package],
repairEnabled: bool, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
for package in packageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
log.LogPrint("- {0}".format(package.Name))
_RunClangFormat(log, toolConfig, clangFormatConfiguration,
clangExeInfo, package, filteredFiles,
repairEnabled)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangFormatHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
customPackageFileFilter, clangFormatConfiguration,
clangExeInfo, repairEnabled)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount
def ProcessAllPackages(
log: Log, toolConfig: ToolConfig, platformId: str,
pythonScriptRoot: str,
performClangTidyConfig: PerformClangTidyConfig,
clangExeInfo: ClangExeInfo, packageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
localVariantInfo: LocalVariantInfo, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
virtualVariantEnvironmentCache = VirtualVariantEnvironmentCache(
log, pythonScriptRoot)
# do standard serial processing
for package in packageList:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, performClangTidyConfig.
ClangTidyConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
_RunClangTidy(log, toolConfig, platformId,
performClangTidyConfig, clangExeInfo,
package, filteredFiles, None,
localVariantInfo,
virtualVariantEnvironmentCache)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangTidyHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
platformId, pythonScriptRoot,
performClangTidyConfig, clangExeInfo,
customPackageFileFilter, localVariantInfo)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount