def __FiltersPackagesByExtensions(
log: Log, packages: Union[List[Package], List[AppInfoPackage]],
extensionNameList: ExtensionListManager, featureNameList: List[str],
requirementTree: Union[RequirementTree, AppInfoGlobalRequirementTree]
) -> List[T]:
""" Filter packages by extensions.
If '*' is in the extensionNameList a clone of 'packages' will be returned
Else we return a list containing only the packages that can be build with the available extensions
"""
if extensionNameList.AllowAllExtensions:
log.LogPrint("Filtering by extensions: All")
return cast(List[T], list(packages))
else:
log.LogPrint("Filtering by extensions: {0}".format(", ".join([
str(qualifiedName)
for qualifiedName in extensionNameList.Content
])))
filteredPackageList = []
for package in packages:
if PackageFilter.__IsAllPackageExtensionAvailable(
package, requirementTree):
filteredPackageList.append(package)
elif package.Type == PackageType.Library or package.Type == PackageType.Executable or package.ResolvedDirectExperimentalRecipe != None:
missingNames = PackageFilter.__GetCompleteMissingExtensionNames(
package, requirementTree)
log.LogPrint(
"Could not build package '{0}' due to missing extension '{1}'"
.format(package.Name, ", ".join(missingNames)))
return cast(List[T], filteredPackageList)
def __FiltersPackagesByFeatures(log: Log,
packages: Union[List[Package],
List[AppInfoPackage]],
featureNameList: List[str]) -> List[T]:
""" Filter packages by features.
If '*' is in the featureNameList a clone of 'packages' will be returned
Else we return a list containing only the packages that can be build with the available features
"""
if '*' in featureNameList:
log.LogPrint("Filtering by features: All")
return cast(List[T], list(packages))
else:
log.LogPrint("Filtering by features: {0}".format(
", ".join(featureNameList)))
filteredPackageList = []
for package in packages:
if PackageFilter.__FeaturesAvailable(package, featureNameList):
filteredPackageList.append(package)
elif package.Type == PackageType.Library or package.Type == PackageType.Executable or package.ResolvedDirectExperimentalRecipe is not None:
missingFeatures = PackageFilter.__GetCompleteMissingFeatureNames(
package, featureNameList)
log.LogPrint(
"Could not build package '{0}' due to missing features '{1}'"
.format(package.Name, ", ".join(missingFeatures)))
return cast(List[T], filteredPackageList)
def __init__(self, log: Log, packageName: str, xmlExperimentalRecipe: XmlExperimentalRecipe, forceDisable: bool) -> None:
"""
forceDisable will not disable 'the external' recipe type used for build tools
"""
self._Log = log
self.SysXmlSource = xmlExperimentalRecipe
self.Version = xmlExperimentalRecipe.Version # type: Optional[Version]
self.ShortName = xmlExperimentalRecipe.ShortName
self.FullName = xmlExperimentalRecipe.FullName
self.AllowFind = xmlExperimentalRecipe.Find
determinedType = self.__DetermineRecipeType(xmlExperimentalRecipe)
self.Type = determinedType if not forceDisable or determinedType == RecipeType.External else RecipeType.Disabled
self.Pipeline = xmlExperimentalRecipe.Pipeline if self.Type == RecipeType.Build else None
self.ValidateInstallation = xmlExperimentalRecipe.ValidateInstallation if self.Type != RecipeType.Disabled else None
self.IsLocalSourceBuild = False
if (self.Pipeline is not None and len(self.Pipeline.CommandList) > 0 and
self.Pipeline.CommandList[0].CommandType == BuildRecipePipelineCommand.Source):
self.IsLocalSourceBuild = True
# The installation path of the package
self.ResolvedInstallLocation = None # type: Optional[ResolvedPath]
if self.Type == RecipeType.Undefined:
log.DoPrintWarning("No installation or validation available for package '{0}' recipe '{1}'".format(packageName, self.FullName))
if self.ValidateInstallation is None and self.Type != RecipeType.Disabled:
log.DoPrintWarning("No installation validation available for package '{0}' recipe '{1}'".format(packageName, self.FullName))
if forceDisable and log.Verbosity >= 4:
if self.Type == RecipeType.Disabled:
log.LogPrint(" - Force disabling recipe for package {0}".format(packageName))
else:
log.LogPrint(" - Force disabling recipe for package {0} ignored due to type".format(packageName))
def __init__(self, log: Log, rootDirs: List[ToolConfigRootDirectory],
basedUponXML: XmlExperimental, configFileName: str,
projectRootDirectory: str) -> None:
super(ToolConfigExperimental, self).__init__()
self.BasedOn = basedUponXML
self.DefaultThirdPartyInstallReadonlyCacheDirectory = None # type: Optional[ToolConfigExperimentalDefaultThirdPartyInstallDirectory]
self.AllowDownloads = basedUponXML.AllowDownloads
self.DisableDownloadEnv = basedUponXML.DisableDownloadEnv
self.DefaultThirdPartyInstallDirectory = ToolConfigExperimentalDefaultThirdPartyInstallDirectory(
log, basedUponXML.DefaultThirdPartyInstallDirectory,
"DefaultThirdPartyInstallDirectory", False)
self.DefaultThirdPartyInstallReadonlyCacheDirectory = self.__TryCreateReadonlyCache(
log, basedUponXML.DefaultThirdPartyInstallReadonlyCacheDirectory)
if log.Verbosity > 1:
log.LogPrint(
"DefaultThirdPartyInstallDirectory: {0} which resolves to '{1}'"
.format(self.DefaultThirdPartyInstallDirectory.Name,
self.DefaultThirdPartyInstallDirectory.ResolvedPath))
if not self.DefaultThirdPartyInstallReadonlyCacheDirectory is None:
log.LogPrint(
"DefaultThirdPartyInstallReadonlyCacheDirectory: {0} which resolves to '{1}'"
.format(
self.DefaultThirdPartyInstallReadonlyCacheDirectory.
Name,
self.DefaultThirdPartyInstallReadonlyCacheDirectory.
ResolvedPath))
def CheckWindowsGLES(log: Log) -> None:
log.LogPrint("- OpenGL ES environment")
try:
emulator = os.environ.get("FSL_GLES_NAME")
if not emulator:
raise EnvironmentError(
"OpenGLES emulation 'FSL_GLES_NAME' environment variable is not set."
)
emulator = emulator.lower()
if (emulator == 'arm' or emulator == 'default'):
CheckWindowsGLESArm(log)
elif (emulator == 'mesa'):
CheckWindowsGLESPowerVR(log)
elif (emulator == 'powervr'):
CheckWindowsGLESMesa(log)
elif (emulator == 'qualcomm'):
CheckWindowsGLESQualcomm(log)
elif (emulator == 'vivante'):
CheckWindowsGLESVivante(log)
else:
log.LogPrint("WARNING: Unknown emulator '{0}'".format(emulator))
except:
print(
"ERROR: CheckWindowsGLES failed.\nThis indicates that the OpenGLES emulation environment was not correctly setup and the code wont be able to locate the necessary OpenGLES header files. Possible causes:\n- Are you using the right emulator type: x64 or x86 see documentation.\n- Did you run the required setup scripts?\n- If building from visual studio did you launch the project via .StartProject.bat\n"
)
raise
def __AddParentFeatures(log: Log, featureNameList: List[str],
requirementTree: Union[RequirementTree, AppInfoGlobalRequirementTree],
useStrictFeatureWarning: bool) -> List[str]:
if '*' in featureNameList:
return featureNameList
featureNameList.sort()
if log.Verbosity > 1:
log.LogPrint("Automatically adding features to supplied feature list {0}".format(featureNameList))
featureNameSet = set(featureNameList)
for featureName in featureNameList:
if featureName in requirementTree.FeatureToNodeDict:
requirementNode = requirementTree.FeatureToNodeDict[featureName]
currentNode = requirementNode # type: Optional[Union[RequirementTreeNode, AppInfoGlobalRequirementTreeNode]]
while currentNode is not None:
if currentNode.Content is not None:
if not currentNode.Content.Name in featureNameSet:
featureNameSet.add(currentNode.Content.Name)
if log.Verbosity > 1 and requirementNode.Content is not None:
log.LogPrint("- '{0}' because '{1}' depends on it".format(currentNode.Content.Name, requirementNode.Content.Name))
currentNode = currentNode.Parent
else:
featureNameSet.remove(featureName)
if useStrictFeatureWarning:
log.DoPrintWarning("Unknown feature name '{0}' in filterNameList {1}".format(featureName, featureNameList))
else:
# For now just log a warning
log.LogPrintVerbose(5, "Unknown feature name '{0}' in filterNameList {1}".format(featureName, featureNameList))
resultList = list(featureNameSet)
resultList.sort()
return resultList
def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration,
cmakeConfig: GeneratorCMakeConfig, repairEnabled: bool,
buildThreads: int, useLegacyTidyMethod: bool) -> None:
""" RunClangFormat on a package at a time """
# Lookup the recommended build threads using the standard build algorithm
numBuildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(
buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
ninjaExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.NinjaRecipePackageName,
MagicValues.NinjaCommand)
log.LogPrint("ClangFormat version: {0}".format(clangExeInfo.Version))
log.LogPrint("Ninja version: {0}".format(ninjaExeInfo.Version))
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in formatPackageList
if PerformClangUtil.CanProcessPackage(package)
]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
#test = set()
#for entry in sortedPackages:
# if entry.Name in test:
# raise Exception("duplicated package")
# else:
# test.add(entry.Name)
if useLegacyTidyMethod:
count = PerformClangFormatHelper.Process(
log, toolConfig, customPackageFileFilter,
clangFormatConfiguration, clangExeInfo, sortedPackages,
repairEnabled, numBuildThreads)
else:
count = PerformClangFormatHelper2.Process(
log, toolConfig, customPackageFileFilter,
clangFormatConfiguration, cmakeConfig, clangExeInfo,
ninjaExeInfo, sortedPackages, repairEnabled, numBuildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning(
"No files processed, could not find a package that matches {0}"
.format(customPackageFileFilter))
def Process(log: Log, toolConfig: ToolConfig, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, cmakeConfig: GeneratorCMakeConfig,
clangFormatExeInfo: ClangExeInfo, ninjaExeInfo: ClangExeInfo, sortedPackageList: List[Package],
repairEnabled: bool, numBuildThreads: int) -> int:
totalProcessedCount = 0
logOutput = False
currentWorkingDirectory = BuildUtil.GetBuildDir(log, toolConfig, cmakeConfig.CheckDir)
currentWorkingDirectory = IOUtil.Join(currentWorkingDirectory, "format")
ninjaOutputFile = IOUtil.Join(currentWorkingDirectory, "build.ninja")
toolVersionOutputFile = IOUtil.Join(currentWorkingDirectory, "ToolVersions.txt")
IOUtil.SafeMakeDirs(currentWorkingDirectory)
log.LogPrint("Using path: '{0}'".format(currentWorkingDirectory))
log.LogPrint("Storing tool versions.")
PerformClangFormatHelper2.WriteToolVersionFile(log, toolVersionOutputFile, clangFormatExeInfo, ninjaExeInfo)
log.LogPrint("Generating ninja format file.")
#foundFixes = PerformClangTidyHelper2.FindFixes(log, clangTidyFixOutputFolder)
#if len(foundFixes) > 0:
# PerformClangTidyHelper2.DeleteFixes(log, clangTidyFixOutputFolder, foundFixes)
totalProcessedCount = PerformClangFormatHelper2.GenerateNinjaTidyFile(log, toolConfig, ninjaOutputFile, currentWorkingDirectory,
toolVersionOutputFile, clangFormatExeInfo, customPackageFileFilter,
clangFormatConfiguration, sortedPackageList, repairEnabled)
log.LogPrint("Executing ninja format file.")
RunHelper.RunNinja(log, ninjaExeInfo, ninjaOutputFile, currentWorkingDirectory, numBuildThreads, logOutput)
return totalProcessedCount
def _RunClangFormat(log: Log, toolConfig: ToolConfig,
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, package: Package,
filteredFiles: Optional[List[str]],
repairEnabled: bool) -> None:
if package.ResolvedBuildAllIncludeFiles is None or len(
package.ResolvedBuildAllIncludeFiles
) <= 0 or not package.AllowCheck or package.IsVirtual:
return
if package.AbsolutePath is None or package.ResolvedBuildSourceFiles is None:
raise Exception("Invalid package")
formatPackageConfig = FormatPackageConfig(log, package,
clangFormatConfiguration,
filteredFiles)
cmd = clangExeInfo.Command
buildCommand = [cmd, '-style=file']
if repairEnabled:
buildCommand.append('-i')
if len(clangFormatConfiguration.AdditionalUserArguments) > 0:
log.LogPrint("Adding user supplied arguments before '--' {0}".format(
clangFormatConfiguration.AdditionalUserArguments))
buildCommand += clangFormatConfiguration.AdditionalUserArguments
buildCommand += [
entry.ResolvedPath for entry in formatPackageConfig.AllFiles
]
currentWorkingDirectory = package.AbsolutePath
FileFinder.FindClosestFileInRoot(log, toolConfig, currentWorkingDirectory,
clangFormatConfiguration.CustomFormatFile)
try:
# if verbose enabled we log the clang-format version
if log.Verbosity >= 4:
log.LogPrint("Running command '{0}' in cwd: {1}".format(
buildCommand, currentWorkingDirectory))
result = subprocess.call(buildCommand, cwd=currentWorkingDirectory)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(" ".join(buildCommand), result,
currentWorkingDirectory))
raise ExitException(result)
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(" ".join(buildCommand), currentWorkingDirectory))
raise
def RunInAnotherThread(packageQueue: Any,
cancellationToken: SimpleCancellationToken,
mainLog: Log, toolConfig: ToolConfig, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, clangExeInfo: ClangExeInfo, repairEnabled: bool) -> Tuple[int,int]:
threadId = threading.get_ident()
mainLog.LogPrintVerbose(4, "Starting thread {0}".format(threadId))
examinedCount = 0
processedCount = 0
keepWorking = True
package = None # type: Optional[Package]
try:
while keepWorking and not cancellationToken.IsCancelled():
try:
# Since the queue is preloaded this is ok
package = packageQueue.get_nowait()
except:
package = None
if package is None:
keepWorking = False
else:
if mainLog.Verbosity >= 4:
mainLog.LogPrint("- clang-format on package '{0}' on thread {1}".format(package.Name, threadId))
else:
mainLog.LogPrint("- clang-format on package '{0}'".format(package.Name))
captureLog = CaptureLog(mainLog.Title, mainLog.Verbosity)
try:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(captureLog, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
processedCount += 1
_RunClangFormat(captureLog, toolConfig, clangFormatConfiguration, clangExeInfo, package, filteredFiles, repairEnabled)
examinedCount += 1
finally:
try:
if len(captureLog.Captured) > 0:
capturedLog = "Package: '{0}' result:\n{1}".format(package.Name, "\n".join(captureLog.Captured))
mainLog.DoPrint(capturedLog)
except:
pass
except Exception as ex:
cancellationToken.Cancel()
mainLog.DoPrintError("Cancelling tasks due to exception: {0}")
raise
finally:
mainLog.LogPrintVerbose(4, "Ending thread {0}".format(threadId))
return (examinedCount, processedCount)
def __init__(self, log: Log, errorHelpManager: ErrorHelpManager,
recipeFilterManager: RecipeFilterManager,
experimental: Optional[ToolConfigExperimental],
generator: GeneratorPlugin) -> None:
if generator.CMakeConfig is None:
raise Exception("Invalid generator")
recipeBuilderSetup = None
allowDownloads = False
if experimental is not None:
targetLocation = ResolvedPath(
experimental.DefaultThirdPartyInstallDirectory.DynamicName,
experimental.DefaultThirdPartyInstallDirectory.ResolvedPath)
installReadonlySource = experimental.DefaultThirdPartyInstallReadonlyCacheDirectory
readonlyCachePath = None if installReadonlySource is None else installReadonlySource.ResolvedPath # type: Optional[str]
recipeBuilderSetup = RecipeBuilderSetup(targetLocation,
readonlyCachePath)
if IOUtil.TryGetEnvironmentVariable(
experimental.DisableDownloadEnv) is not None:
allowDownloads = False
log.LogPrint(
"Downloads disabled since the environment variable {0} was defined"
.format(experimental.DisableDownloadEnv))
elif experimental.AllowDownloads:
allowDownloads = True
else:
log.LogPrint(
"Downloads disabled since the project has it disabled by default"
)
validVariableDict = {} # type: Dict[str, object]
validVariableDict['PlatformName'] = generator.PlatformName
validVariableDict['IsCMakeBuild'] = generator.IsCMake
generatorInfo = GeneratorInfo(generator.IsCMake,
generator.CMakeConfig.AllowFindPackage,
validVariableDict)
super().__init__(log, errorHelpManager, generator.PlatformName,
generator.PlatformName, generatorInfo,
generator.CMakeConfig, recipeBuilderSetup)
#allowDownload=True, disableDownloadEnv=None
self.Log = log
self.Generator = generator
# for now the generator is also the platform and the existing code use that name so maintain compatibility for now
self.Platform = generator
self.AllowDownloads = allowDownloads
self.RecipeFilterManager = recipeFilterManager
def __TryGetValidateInstallation(self, log: Log, xmlElement: ET.Element) -> Optional[XmlRecipeInstallation]:
child = self._TryGetElement(xmlElement, 'Installation')
if child is None:
if log.Verbosity >= 2:
log.LogPrint("The Installation element is missing for recipe {0}".format(self.Name))
return None
return XmlRecipeInstallation(log, child)
def ProcessInstallDirClaim(log: Log,
targetPath: str,
sdkPath: str,
forceClaimInstallArea: bool,
installAreaInfoPath: str) -> None:
filePath = IOUtil.Join(targetPath, installAreaInfoPath)
# Beware that this method to claim a area is in no way secure
# It will basically only be useful after the claim file has been created
# So we start with a possible race condition, but any future attempts to use the directory
# will catch that multiple repos are trying to reuse the same install area
fileContent = IOUtil.TryReadFile(filePath)
if fileContent is None:
log.LogPrint("Install area '{0}' is unclaimed, claiming it".format(targetPath))
BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea)
return
jsonBuildInfoDict = json.loads(fileContent)
if not BuildAreaInfoFile.IsDictValid(jsonBuildInfoDict):
if not forceClaimInstallArea:
raise Exception("Install area '{0}' contained an invalid file '{1}', did you try to run concurrent builds using the same install directory or did it get corrupted? Delete the file to allow the build to continue or use --ForceClaimInstallArea to do so automatically. You could also set up a readonly cache area to reuse between repos see the documentation for more info.".format(targetPath, filePath))
log.DoPrintWarning("Install area '{0}' contained an invalid file '{1}', as --ForceClaimInstallArea was specified it was overwritten and '{2}' now controls it".format(targetPath, filePath, sdkPath))
jsonBuildInfoDict = BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea, logWarning=False)
buildAreaInfoFile = BuildAreaInfoFile(jsonBuildInfoDict)
if buildAreaInfoFile.SDKPath != sdkPath:
if not forceClaimInstallArea:
raise Exception("The Install area at '{0}' is already claimed by the sdk at '{1}' so the sdk at '{2}' can not reuse it as it could give concurrency issues if multiple builds execute at the same time. If you are sure that you are not doing concurrent builds and you just want to use the area for a new SDK you can force claim it with --ForceClaimInstallArea. You could also set up a readonly cache area to reuse between repos see the documentation for more info.".format(targetPath, buildAreaInfoFile.SDKPath, sdkPath))
log.DoPrintWarning("The Install area at '{0}' was already claimed by the sdk at '{1}' but '{2}' took control of it as --ForceClaimInstallArea was specified.".format(targetPath, buildAreaInfoFile.SDKPath, sdkPath))
jsonBuildInfoDict = BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea, logWarning=False)
def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration,
repairEnabled: bool) -> None:
""" RunClangFormat on a package at a time """
clangExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
if log.Verbosity >= 1:
PerformClangUtil.ShowVersion(log, clangExeInfo)
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in formatPackageList
if PerformClangUtil.CanProcessPackage(package)
]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
for package in sortedPackages:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
_RunClangFormat(log, toolConfig, clangFormatConfiguration,
clangExeInfo, package, filteredFiles,
repairEnabled)
def __BuildInstanceGraph(
log: Log, queue: PackageResolveQueue) -> ResolvedPackageGraph:
graph = ResolvedPackageGraph()
instanceDict = dict(
) # type: Dict[PackageName, ResolvedPackageInstance]
resolver = PackageResolver(log)
while len(queue) > 0:
currentPackage = queue.Dequeue()
if currentPackage.Type != PackageType.TopLevel:
if log.Verbosity >= LocalVerbosityLevel.Info:
log.LogPrint(
"Resolving package: {0}".format(currentPackage))
resolvedPackageTemplate = resolver.Resolve(currentPackage)
PackageGraphBuilder.__AddTemplateNodeAndEdges(
graph, resolvedPackageTemplate)
if len(resolvedPackageTemplate.InstanceConfigs) <= 0:
raise Exception(
"ERROR: no instance configs provided by template")
isFlavorSingleton = len(
resolvedPackageTemplate.InstanceConfigs) <= 1
for instanceConfig in resolvedPackageTemplate.InstanceConfigs:
packageInstance = PackageGraphBuilder.__CreateResolvedPackageInstance(
log, instanceDict, resolvedPackageTemplate,
instanceConfig,
isFlavorSingleton) # type: ResolvedPackageInstance
PackageGraphBuilder.__AddInstanceNodeAndEdges(
graph, packageInstance)
elif len(queue) != 0:
raise Exception("internal error")
return graph
def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration, repairEnabled: bool, buildThreads: int) -> None:
""" RunClangFormat on a package at a time """
# Lookup the recommended build threads using the standard build algorithm
buildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(packageRecipeResultManager, clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
if log.Verbosity >= 1:
PerformClangUtil.ShowVersion(log, clangExeInfo)
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [package for package in formatPackageList if PerformClangUtil.CanProcessPackage(package)]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
count = PerformClangFormatHelper.Process(log, toolConfig, customPackageFileFilter, clangFormatConfiguration, clangExeInfo, sortedPackages,
repairEnabled, buildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning("No files processed, could not find a package that matches {0}".format(customPackageFileFilter))
def __ApplyClangTidy(self, log: Log, toolConfig: ToolConfig, localToolConfig: LocalToolConfig,
packageRecipeResultManager: Optional[PackageRecipeResultManager],
topLevelPackage: Optional[Package], tidyPackageList: List[Package], generator: GeneratorPlugin,
config: Config, generatorContext: GeneratorContext, customPackageFileFilter: Optional[CustomPackageFileFilter]) -> None:
"""
The only reason we take optionals here is because they are optional in the main program, so its just easier to do the check here
:currentDirPath: is used to process the '--file' argument
"""
if toolConfig.ClangTidyConfiguration is None or packageRecipeResultManager is None or topLevelPackage is None:
raise Exception("internal error")
# Generate the build files (so that the run scripts exist)
log.LogPrint("Generating build files")
generator.Generate(generatorContext, config, tidyPackageList)
clangFormatFilename = None if toolConfig.ClangFormatConfiguration is None else toolConfig.ClangFormatConfiguration.CustomFormatFile
pythonScriptRoot = IOUtil.Join(config.SDKPath, ".Config")
performClangTidyConfig = PerformClangTidyConfig(toolConfig.ClangTidyConfiguration, localToolConfig.ClangTidyArgs,
localToolConfig.ClangTidyPostfixArgs, localToolConfig.TidyOverrideChecks,
localToolConfig.ClangTidyProfile, not localToolConfig.ClangTidyNoDynamicVariantCache,
localToolConfig.Repair)
PerformClangTidy.Run(log, toolConfig, generatorContext.Generator.PlatformId, topLevelPackage, tidyPackageList,
localToolConfig.BuildVariantsDict, pythonScriptRoot, generatorContext, config.SDKConfigTemplatePath,
packageRecipeResultManager, performClangTidyConfig, customPackageFileFilter, clangFormatFilename,
localToolConfig.BuildThreads, localToolConfig.Legacy)
def Scan(log: Log, scanPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
repairEnabled: bool, thirdpartyExceptionDir: Optional[str],
checkType: CheckType, disableWrite: bool) -> None:
"""
Run through all source files that are part of the packages and check for common errors
:param scanPackageList: the packages that will be scanned.
"""
log.LogPrint("Running source scan")
extensionList = __g_includeExtensionList + __g_sourceExtensionList
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in scanPackageList
if PerformClangUtil.CanProcessPackage(package)
]
totalErrorCount = 0
for package in finalPackageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, extensionList)
if customPackageFileFilter is None or filteredFiles is not None:
totalErrorCount += __ScanFiles(log, package, filteredFiles,
repairEnabled,
thirdpartyExceptionDir, checkType,
disableWrite)
if totalErrorCount > 0 and not repairEnabled:
log.DoPrint(
"BEWARE: If you have made a backup of your files you can try to auto correct the errors with '--Repair' but do so at your own peril"
)
def __FilterNotUserRequested(log: Log, sourcePackageBuildOrder: List[PreResolvePackageResult],
requestedPackages: Optional[List[PreResolvePackageResult]]) -> List[PreResolvePackageResult]:
if requestedPackages is None or len(requestedPackages) <= 0:
return sourcePackageBuildOrder
userRequestedPackageNameSet = set()
for preResolvePackageResult in requestedPackages:
package = preResolvePackageResult.SourcePackage
if not package.NameInfo.FullName in userRequestedPackageNameSet:
userRequestedPackageNameSet.add(package.NameInfo.FullName)
for depPreResolvePackageResult in preResolvePackageResult.ResolvedBuildOrder:
if not depPreResolvePackageResult.SourcePackage.NameInfo.FullName in userRequestedPackageNameSet:
userRequestedPackageNameSet.add(depPreResolvePackageResult.SourcePackage.NameInfo.FullName)
# This one liner
# return [entry for entry in sourcePackageBuildOrder if entry.SourcePackage.NameInfo.FullName in userRequestedPackageNameSet]
# Was expanded because we need to log information
logResult = log.Verbosity > 2
result = []
for entry in sourcePackageBuildOrder:
if entry.SourcePackage.NameInfo.FullName in userRequestedPackageNameSet:
result.append(entry)
elif logResult:
log.LogPrint("- {0} (Removed because it was not requested by the user)".format(entry.SourcePackage.NameInfo.FullName))
return result
def ShowRecipeList(log: Log, topLevelPackage: Package,
requestedFiles: Optional[List[str]]) -> None:
#requestedPackages = PackageUtil.GetPackageListFromFilenames(topLevelPackage, requestedFiles, True)
recipeBuildOrder = topLevelPackage.ResolvedExperimentalRecipeBuildOrder
if recipeBuildOrder is None or len(recipeBuildOrder) <= 0:
log.LogPrint("No recipes found")
return
recipePackages = list(
topLevelPackage.ResolvedExperimentalRecipeBuildOrder)
recipePackages.sort(
key=lambda s: s.ResolvedDirectExperimentalRecipe.FullName.lower()
if s.ResolvedDirectExperimentalRecipe is not None else s.Name.
lower())
strAddIndent = " "
print("Recipes")
for package in recipePackages:
packageRecipe = package.ResolvedDirectExperimentalRecipe
if packageRecipe is not None and packageRecipe.Type != RecipeType.External:
print("{0}{1} (introduced by: {2}), type: {3}".format(
strAddIndent, packageRecipe.FullName, package.Name,
RecipeType.ToString(packageRecipe.Type)))
print("External")
for package in recipePackages:
packageRecipe = package.ResolvedDirectExperimentalRecipe
if packageRecipe is not None and packageRecipe.Type == RecipeType.External:
print("{0}{1} (introduced by: {2})".format(
strAddIndent, packageRecipe.FullName, package.Name))
def GenerateNinjaTidyFile(log: Log, toolConfig: ToolConfig, ninjaOutputFile: str, outputFolder: str, toolVersionOutputFile: str,
clangFormatExeInfo: ClangExeInfo, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, sortedPackageList: List[Package], repairEnabled: bool) -> int:
totalProcessedCount = 0
toolProjectContextsDict = PackagePathUtil.CreateToolProjectContextsDict(toolConfig.ProjectInfo)
with io.StringIO() as ninjaFile:
ninjaFile.writelines("ninja_required_version = 1.8\n")
writer = Writer(ninjaFile, 149)
formatCommand = "{0} $in --style=file".format(clangFormatExeInfo.Command)
if repairEnabled:
formatCommand += " -i"
if len(clangFormatConfiguration.AdditionalUserArguments) > 0:
log.LogPrint("Adding user supplied arguments before '--' {0}".format(clangFormatConfiguration.AdditionalUserArguments))
formatCommand += " {0}".format(" ".join(clangFormatConfiguration.AdditionalUserArguments))
writer.rule(name=PerformClangFormatHelper2.RULE_FORMAT,
command=formatCommand,
restat=True)
for package in sortedPackageList:
PerformClangFormatHelper2.AddPackage(log, toolConfig, toolVersionOutputFile, writer, package, outputFolder, toolProjectContextsDict,
customPackageFileFilter, clangFormatConfiguration)
totalProcessedCount += 1
# finally we write the ninja file
IOUtil.WriteFileIfChanged(ninjaOutputFile, ninjaFile.getvalue())
writer.close()
return totalProcessedCount
def Process(log: Log, toolConfig: ToolConfig,
customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, packageList: List[Package],
repairEnabled: bool, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
for package in packageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
log.LogPrint("- {0}".format(package.Name))
_RunClangFormat(log, toolConfig, clangFormatConfiguration,
clangExeInfo, package, filteredFiles,
repairEnabled)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangFormatHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
customPackageFileFilter, clangFormatConfiguration,
clangExeInfo, repairEnabled)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount
def __AppendMakeThreads(log: Log, rBuildCommandList: List[str], buildThreads: int) -> None:
if buildThreads == BuildThreads.Auto:
buildThreads = multiprocessing.cpu_count()
if buildThreads != BuildThreads.NotDefined and buildThreads > 0:
log.LogPrint("Builder using BuildThreads: {0}".format(buildThreads))
rBuildCommandList += ['-j', str(buildThreads)]
def TryLoadBuildInformation(log: Log, sourcePackage: Package,
path: str) -> Optional[JsonDictType]:
try:
if not PackageRecipeUtil.HasBuildPipeline(sourcePackage):
return None
sourceRecipe = sourcePackage.ResolvedDirectExperimentalRecipe
if sourceRecipe is None or sourceRecipe.ResolvedInstallLocation is None:
raise Exception("Invalid recipe")
# Generally this should not be called if there is no pipeline
srcFilePath = IOUtil.Join(
sourceRecipe.ResolvedInstallLocation.ResolvedPath, path)
fileContent = IOUtil.TryReadFile(srcFilePath)
if fileContent is None:
log.LogPrint(
"Package build information for package {0} not found in the expected file '{1}'"
.format(sourcePackage.Name, srcFilePath))
return None
jsonBuildInfoDict = json.loads(fileContent)
if not BuildInfoFile.IsDictValid(jsonBuildInfoDict):
log.LogPrint(
"Package build information for package {0} found in file '{1}' is invalid"
.format(sourcePackage.Name, srcFilePath))
return None
# Decode the complex element to a object of the right type
jsonBuildInfoDict[
BuildInfoFileElements.
ContentState] = BuildInfoComplexJsonDecoder.DecodeJson(
jsonBuildInfoDict[BuildInfoFileElements.ContentState])
if BuildInfoFileElements.SourceState in jsonBuildInfoDict:
jsonBuildInfoDict[
BuildInfoFileElements.
SourceState] = BuildInfoComplexJsonDecoder.DecodeJson(
jsonBuildInfoDict[BuildInfoFileElements.SourceState])
return cast(JsonDictType, jsonBuildInfoDict)
except Exception as ex:
log.LogPrintWarning(
"TryLoadBuildInformation failed for package '{0}' with {1}".
format(sourcePackage.Name, ex))
return None
def __ValidateIncludeGuard(log: Log, sourceFile: SourceFile, shortFile: str, repairEnabled: bool) -> bool:
if len(sourceFile.LinesModded) < 2:
return False
currentLine0 = sourceFile.LinesModded[0].strip()
currentLine1 = sourceFile.LinesModded[1].strip()
guard = __GenerateIncludeGuardName(sourceFile.Package, shortFile)
line0Valid = "#ifndef %s" % (guard)
line1Valid = "#define %s" % (guard)
if currentLine0 == line0Valid and currentLine1 == line1Valid:
return True
# check that the file starts with the guard statements
prefix0 = "#ifndef "
prefix1 = "#define "
if not currentLine0.startswith(prefix0):
log.DoPrint("Line 0 does not start with '%s' in '%s'" % (prefix0, os.path.normpath(sourceFile.FileName)))
if repairEnabled:
log.LogPrint("Because of this repair was not attempted.")
return False
if not currentLine1.startswith(prefix1):
log.DoPrint("Line 1 does not start with '%s' in '%s'" % (prefix1, os.path.normpath(sourceFile.FileName)))
if repairEnabled:
log.LogPrint("Because of this repair was not attempted.")
return False
# validate that the #ifndef and define works on the same string
userDef0 = currentLine0[len(prefix0):].strip()
userDef1 = currentLine1[len(prefix1):].strip()
if userDef0 != userDef1:
log.DoPrint("The include guards do not appear to match '%s' != '%s' in '%s'" % (userDef0, userDef1, os.path.normpath(sourceFile.FileName)))
log.LogPrint("- Line 0 '%s'" % (userDef0))
log.LogPrint("- Line 1 '%s'" % (userDef1))
if repairEnabled:
log.LogPrint("Because of this repair was not attempted.")
return False
# So we should be sure that the guard is just the incorrect name, so list it
log.DoPrint("Wrong include guard: '%s' expected '%s'" % (os.path.normpath(sourceFile.FileName), guard))
if currentLine0 != line0Valid:
log.LogPrint("- Expected '%s'" % (line0Valid))
log.LogPrint("- Was '%s'" % (currentLine0))
elif currentLine1 != line1Valid:
log.LogPrint("- Expected '%s'" % (line1Valid))
log.LogPrint("- Was '%s'" % (currentLine1))
if not repairEnabled:
return False
log.DoPrint("Include guard corrected")
# We are allowed to repair the content, so lets do that
sourceFile.LinesModded[0] = line0Valid
sourceFile.LinesModded[1] = line1Valid
return False
def AddBuildThreads(log: Log, rArgumentList: List[str], generatorOriginalName: str, buildThreads: int, external: bool = False) -> None:
platformName = generatorOriginalName
if (platformName == PlatformNameString.QNX or
platformName == PlatformNameString.YOCTO or
platformName == PlatformNameString.UBUNTU or
(external and platformName == PlatformNameString.ANDROID)):
PlatformBuildUtil.__AppendMakeThreads(log, rArgumentList, buildThreads)
elif platformName == PlatformNameString.WINDOWS:
if buildThreads == BuildThreads.Auto:
log.LogPrint("Builder using BuildThreads: auto")
rArgumentList += ['/maxcpucount']
elif buildThreads != BuildThreads.NotDefined and buildThreads > 0:
log.LogPrint("Builder using BuildThreads: {0}".format(buildThreads))
rArgumentList += ['/maxcpucount:{0}'.format(buildThreads)]
else:
if buildThreads != BuildThreads.NotDefined:
log.LogPrintWarning("The builder ignored 'BuildThreads'")
def Run(log: Log, createInfo: OpenProjectCreateInfo) -> None:
log.LogPrintVerbose(
1, "Configuring and launching Visual Studio Code for path '{0}'".
format(createInfo.SourcePath))
buildPlatformType = PlatformUtil.DetectBuildPlatformType()
vsCodeConfigPath = IOUtil.Join(createInfo.SourcePath, ".vscode")
launchFilePath = IOUtil.Join(vsCodeConfigPath, "launch.json")
settingsFilePath = IOUtil.Join(vsCodeConfigPath, "settings.json")
IOUtil.SafeMakeDirs(vsCodeConfigPath)
log.LogPrintVerbose(
1, "- Patching settings at '{0}'".format(settingsFilePath))
log.PushIndent()
try:
VSCodeSettingsJsonUtil.Patch(log, settingsFilePath,
createInfo.CMakeInfo)
finally:
log.PopIndent()
exeInfo = createInfo.ExeInfo
if exeInfo is not None:
if log.Verbosity >= 1:
log.LogPrint("- Patching launch settings at '{0}'".format(
launchFilePath))
log.LogPrint(" - Exe: '{0}'".format(exeInfo.Executable))
log.LogPrint(" - Cwd: '{0}'".format(
exeInfo.CurrentWorkingDirectory))
if not VSCodeLaunchJsonUtil.TryPatch(
launchFilePath, buildPlatformType, exeInfo.Executable,
exeInfo.CurrentWorkingDirectory):
log.LogPrintVerbose(
1, "WARNING Failed to patch launch file '{0}'".format(
launchFilePath))
else:
log.LogPrintVerbose(1, "- Launch: No executable information found")
log.PushIndent()
try:
OpenProjectUtil.__RunVSCode(log, buildPlatformType,
createInfo.SourcePath)
finally:
log.PopIndent()
def LogVariantSettings(log: Log, variantSettingsDict: Dict[str, str]) -> None:
if len(variantSettingsDict) <= 0:
return
names = list(variantSettingsDict.keys())
names.sort()
result = []
for name in names:
result.append("{0}={1}".format(name, variantSettingsDict[name]))
log.LogPrint("Variant settings: {0}".format(", ".join(result)))
def __Load(self, log: Log, path: str) -> None:
if not os.path.exists(path):
return
content = IOUtil.ReadFile(path)
self.Dirs = {}
self.Entries = {}
self.Removed = {}
self.IsNew = True
cacheHeader = GetCacheHeader()
# if the file starts with a invalid header, we ignore it
if not content.startswith(cacheHeader):
log.LogPrint("Cache header invalid, ignoring cache")
return
entries = [] # type: List[ContentState]
dirs = []
lines = content.splitlines(False)
if len(lines) < 3:
log.LogPrintWarning("Cache at '{0}' is invalid, ignoring it.".format(path))
return
for line in lines:
line = line.strip()
if not line.startswith('#'):
elements = line.split(GLOBAL_SEP)
if len(elements) != 5:
log.LogPrint("Cache entry invalid, ignoring cache")
return
contentState = ContentState()
contentState.Name = elements[0]
contentState.Length = int(elements[1])
contentState.ModifiedDate = elements[2]
contentState.Checksum = elements[3]
contentState.TagChecksum = elements[4]
if contentState.Length >= 0:
entries.append(contentState)
else:
dirs.append(contentState)
for entry in entries:
self.Add(entry)
for entry in dirs:
self.AddDir(entry)
self.IsNew = False
def __ScanForTemplates(self, log: Log, path: str) -> List[XmlNewVSProjectTemplateFile]:
files = IOUtil.GetFilePaths(path, self.TemplateFileName)
templateList = [] # type: List[XmlNewVSProjectTemplateFile]
for filename in files:
template = self.__TryLoadTemplate(log, filename)
if template is not None:
templateList.append(template)
else:
log.LogPrint("Failed to load template '{0}'".format(filename))
return templateList