def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration,
cmakeConfig: GeneratorCMakeConfig, repairEnabled: bool,
buildThreads: int, useLegacyTidyMethod: bool) -> None:
""" RunClangFormat on a package at a time """
# Lookup the recommended build threads using the standard build algorithm
numBuildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(
buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
ninjaExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.NinjaRecipePackageName,
MagicValues.NinjaCommand)
log.LogPrint("ClangFormat version: {0}".format(clangExeInfo.Version))
log.LogPrint("Ninja version: {0}".format(ninjaExeInfo.Version))
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in formatPackageList
if PerformClangUtil.CanProcessPackage(package)
]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
#test = set()
#for entry in sortedPackages:
# if entry.Name in test:
# raise Exception("duplicated package")
# else:
# test.add(entry.Name)
if useLegacyTidyMethod:
count = PerformClangFormatHelper.Process(
log, toolConfig, customPackageFileFilter,
clangFormatConfiguration, clangExeInfo, sortedPackages,
repairEnabled, numBuildThreads)
else:
count = PerformClangFormatHelper2.Process(
log, toolConfig, customPackageFileFilter,
clangFormatConfiguration, cmakeConfig, clangExeInfo,
ninjaExeInfo, sortedPackages, repairEnabled, numBuildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning(
"No files processed, could not find a package that matches {0}"
.format(customPackageFileFilter))
def ProcessInstallDirClaim(log: Log,
targetPath: str,
sdkPath: str,
forceClaimInstallArea: bool,
installAreaInfoPath: str) -> None:
filePath = IOUtil.Join(targetPath, installAreaInfoPath)
# Beware that this method to claim a area is in no way secure
# It will basically only be useful after the claim file has been created
# So we start with a possible race condition, but any future attempts to use the directory
# will catch that multiple repos are trying to reuse the same install area
fileContent = IOUtil.TryReadFile(filePath)
if fileContent is None:
log.LogPrint("Install area '{0}' is unclaimed, claiming it".format(targetPath))
BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea)
return
jsonBuildInfoDict = json.loads(fileContent)
if not BuildAreaInfoFile.IsDictValid(jsonBuildInfoDict):
if not forceClaimInstallArea:
raise Exception("Install area '{0}' contained an invalid file '{1}', did you try to run concurrent builds using the same install directory or did it get corrupted? Delete the file to allow the build to continue or use --ForceClaimInstallArea to do so automatically. You could also set up a readonly cache area to reuse between repos see the documentation for more info.".format(targetPath, filePath))
log.DoPrintWarning("Install area '{0}' contained an invalid file '{1}', as --ForceClaimInstallArea was specified it was overwritten and '{2}' now controls it".format(targetPath, filePath, sdkPath))
jsonBuildInfoDict = BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea, logWarning=False)
buildAreaInfoFile = BuildAreaInfoFile(jsonBuildInfoDict)
if buildAreaInfoFile.SDKPath != sdkPath:
if not forceClaimInstallArea:
raise Exception("The Install area at '{0}' is already claimed by the sdk at '{1}' so the sdk at '{2}' can not reuse it as it could give concurrency issues if multiple builds execute at the same time. If you are sure that you are not doing concurrent builds and you just want to use the area for a new SDK you can force claim it with --ForceClaimInstallArea. You could also set up a readonly cache area to reuse between repos see the documentation for more info.".format(targetPath, buildAreaInfoFile.SDKPath, sdkPath))
log.DoPrintWarning("The Install area at '{0}' was already claimed by the sdk at '{1}' but '{2}' took control of it as --ForceClaimInstallArea was specified.".format(targetPath, buildAreaInfoFile.SDKPath, sdkPath))
jsonBuildInfoDict = BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea, logWarning=False)
def __init__(self, log: Log, packageName: str, xmlExperimentalRecipe: XmlExperimentalRecipe, forceDisable: bool) -> None:
"""
forceDisable will not disable 'the external' recipe type used for build tools
"""
self._Log = log
self.SysXmlSource = xmlExperimentalRecipe
self.Version = xmlExperimentalRecipe.Version # type: Optional[Version]
self.ShortName = xmlExperimentalRecipe.ShortName
self.FullName = xmlExperimentalRecipe.FullName
self.AllowFind = xmlExperimentalRecipe.Find
determinedType = self.__DetermineRecipeType(xmlExperimentalRecipe)
self.Type = determinedType if not forceDisable or determinedType == RecipeType.External else RecipeType.Disabled
self.Pipeline = xmlExperimentalRecipe.Pipeline if self.Type == RecipeType.Build else None
self.ValidateInstallation = xmlExperimentalRecipe.ValidateInstallation if self.Type != RecipeType.Disabled else None
self.IsLocalSourceBuild = False
if (self.Pipeline is not None and len(self.Pipeline.CommandList) > 0 and
self.Pipeline.CommandList[0].CommandType == BuildRecipePipelineCommand.Source):
self.IsLocalSourceBuild = True
# The installation path of the package
self.ResolvedInstallLocation = None # type: Optional[ResolvedPath]
if self.Type == RecipeType.Undefined:
log.DoPrintWarning("No installation or validation available for package '{0}' recipe '{1}'".format(packageName, self.FullName))
if self.ValidateInstallation is None and self.Type != RecipeType.Disabled:
log.DoPrintWarning("No installation validation available for package '{0}' recipe '{1}'".format(packageName, self.FullName))
if forceDisable and log.Verbosity >= 4:
if self.Type == RecipeType.Disabled:
log.LogPrint(" - Force disabling recipe for package {0}".format(packageName))
else:
log.LogPrint(" - Force disabling recipe for package {0} ignored due to type".format(packageName))
def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration, repairEnabled: bool, buildThreads: int) -> None:
""" RunClangFormat on a package at a time """
# Lookup the recommended build threads using the standard build algorithm
buildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(packageRecipeResultManager, clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
if log.Verbosity >= 1:
PerformClangUtil.ShowVersion(log, clangExeInfo)
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [package for package in formatPackageList if PerformClangUtil.CanProcessPackage(package)]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
count = PerformClangFormatHelper.Process(log, toolConfig, customPackageFileFilter, clangFormatConfiguration, clangExeInfo, sortedPackages,
repairEnabled, buildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning("No files processed, could not find a package that matches {0}".format(customPackageFileFilter))
def __init__(self, log: Log, packageName: str,
xmlExperimentalRecipe: XmlExperimentalRecipe) -> None:
self._Log = log
self.XmlSource = xmlExperimentalRecipe
self.Name = xmlExperimentalRecipe.Name # type: str
self.Type = self.__DetermineRecipeType(xmlExperimentalRecipe)
self.Pipeline = xmlExperimentalRecipe.Pipeline if self.Type == RecipeType.Build else None
self.ValidateInstallation = xmlExperimentalRecipe.ValidateInstallation
self.IsLocalSourceBuild = False
if (self.Pipeline is not None and len(self.Pipeline.CommandList) > 0
and self.Pipeline.CommandList[0].CommandType
== BuildRecipePipelineCommand.Source):
self.IsLocalSourceBuild = True
# The installation path of the package
self.ResolvedInstallPath = None # type: Optional[str]
if self.Type == RecipeType.Undefined:
log.DoPrintWarning(
"No installation or validation available for package '{0}' recipe '{1}'"
.format(packageName, self.Name))
if self.ValidateInstallation is None:
log.DoPrintWarning(
"No installation validation available for package '{0}' recipe '{1}'"
.format(packageName, self.Name))
def TryLoad(log: Log,
cacheFilename: str) -> Optional['BuildConfigureCache']:
try:
strJson = IOUtil.TryReadFile(cacheFilename)
if strJson is None:
return None
jsonDict = json.loads(strJson)
if jsonDict["Version"] != 1:
raise Exception("Unsupported version")
jsonFileHashDict = jsonDict["FileHashDict"]
finalDict = {} # type: Dict[str,str]
for key, value in jsonFileHashDict.items():
if not isinstance(key, str) or not isinstance(value, str):
raise Exception("json decode failed")
finalDict[key] = value
finalCommandList = [] # type: List[str]
jsonCommandList = jsonDict["CommandList"]
for value in jsonCommandList:
if not isinstance(value, str):
raise Exception("json decode failed")
finalCommandList.append(value)
return BuildConfigureCache(finalDict, finalCommandList)
except:
log.DoPrintWarning(
"Failed to decode cache file '{0}'".format(cacheFilename))
return None
def TryRun(log: Log, cmdList: List[str]) -> Optional[str]:
"""
Run the command and capture the output
:return: the captured output on sucess, None if it failed
"""
try:
if cmdList[0].endswith(
'.py') and PlatformUtil.DetectBuildPlatformType(
) == BuildPlatformType.Windows:
cmdList[0] = cmdList[0][:-3] + ".bat"
with subprocess.Popen(cmdList,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=True) as proc:
output = proc.stdout.read().strip()
proc.stdout.close()
result = proc.wait()
if result != 0:
LocalUtil.DumpCapture(log, 4, output)
log.LogPrintWarning(
"The command '{0}' failed with '{1}'".format(
" ".join(cmdList), result))
return None
if isinstance(output, str):
return output
return None
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'.".format(
" ".join(cmdList)))
return None
def __AddParentFeatures(log: Log, featureNameList: List[str],
requirementTree: Union[RequirementTree, AppInfoGlobalRequirementTree],
useStrictFeatureWarning: bool) -> List[str]:
if '*' in featureNameList:
return featureNameList
featureNameList.sort()
if log.Verbosity > 1:
log.LogPrint("Automatically adding features to supplied feature list {0}".format(featureNameList))
featureNameSet = set(featureNameList)
for featureName in featureNameList:
if featureName in requirementTree.FeatureToNodeDict:
requirementNode = requirementTree.FeatureToNodeDict[featureName]
currentNode = requirementNode # type: Optional[Union[RequirementTreeNode, AppInfoGlobalRequirementTreeNode]]
while currentNode is not None:
if currentNode.Content is not None:
if not currentNode.Content.Name in featureNameSet:
featureNameSet.add(currentNode.Content.Name)
if log.Verbosity > 1 and requirementNode.Content is not None:
log.LogPrint("- '{0}' because '{1}' depends on it".format(currentNode.Content.Name, requirementNode.Content.Name))
currentNode = currentNode.Parent
else:
featureNameSet.remove(featureName)
if useStrictFeatureWarning:
log.DoPrintWarning("Unknown feature name '{0}' in filterNameList {1}".format(featureName, featureNameList))
else:
# For now just log a warning
log.LogPrintVerbose(5, "Unknown feature name '{0}' in filterNameList {1}".format(featureName, featureNameList))
resultList = list(featureNameSet)
resultList.sort()
return resultList
def _RunClangFormat(log: Log, toolConfig: ToolConfig,
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, package: Package,
filteredFiles: Optional[List[str]],
repairEnabled: bool) -> None:
if package.ResolvedBuildAllIncludeFiles is None or len(
package.ResolvedBuildAllIncludeFiles
) <= 0 or not package.AllowCheck or package.IsVirtual:
return
if package.AbsolutePath is None or package.ResolvedBuildSourceFiles is None:
raise Exception("Invalid package")
formatPackageConfig = FormatPackageConfig(log, package,
clangFormatConfiguration,
filteredFiles)
cmd = clangExeInfo.Command
buildCommand = [cmd, '-style=file']
if repairEnabled:
buildCommand.append('-i')
if len(clangFormatConfiguration.AdditionalUserArguments) > 0:
log.LogPrint("Adding user supplied arguments before '--' {0}".format(
clangFormatConfiguration.AdditionalUserArguments))
buildCommand += clangFormatConfiguration.AdditionalUserArguments
buildCommand += [
entry.ResolvedPath for entry in formatPackageConfig.AllFiles
]
currentWorkingDirectory = package.AbsolutePath
FileFinder.FindClosestFileInRoot(log, toolConfig, currentWorkingDirectory,
clangFormatConfiguration.CustomFormatFile)
try:
# if verbose enabled we log the clang-format version
if log.Verbosity >= 4:
log.LogPrint("Running command '{0}' in cwd: {1}".format(
buildCommand, currentWorkingDirectory))
result = subprocess.call(buildCommand, cwd=currentWorkingDirectory)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(" ".join(buildCommand), result,
currentWorkingDirectory))
raise ExitException(result)
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(" ".join(buildCommand), currentWorkingDirectory))
raise
def ShowVersion(log: Log, clangExeInfo: ClangExeInfo) -> None:
try:
log.LogPrint("Listing version")
cmd = clangExeInfo.Command
versionCommand = [cmd, '-version']
result = subprocess.call(versionCommand)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'.".format(
" ".join(versionCommand), result))
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'.".format(
" ".join(versionCommand)))
raise
def ExtendEnvironmentDictWithVariants(log: Log,
buildEnv: Dict[str, str],
package: Package,
userVariantSettingDict: Dict[str, str]) -> None:
for key, value in list(userVariantSettingDict.items()):
variant = BuildVariantUtil.TryLocateVariant(package, key)
if variant is not None:
if variant.Type == VariantType.Virtual or (value in variant.OptionDict):
envName = "{0}{1}".format(GEN_BUILD_ENV_VARIANT_SETTING, key.upper())
if envName in buildEnv:
raise Exception("The environment variable {0} has allready been defined".format(envName))
buildEnv[envName] = value
else:
validValues = list(variant.OptionDict.keys())
validValues.sort()
log.DoPrintWarning("Variant '{0}' expects one of the following values: '{1}' not '{2}'".format(key, ','.join(validValues), value))
def __RemoveInvalidInstallation(
self, log: Log, sourceRecipe: PackageExperimentalRecipe) -> None:
if sourceRecipe is None or sourceRecipe.ResolvedInstallPath is None or sourceRecipe.Pipeline is None:
return
# the external installs dont have a associated pipeline so this should be safe
# but we check the build type as well just to be safe
if IOUtil.IsDirectory(sourceRecipe.ResolvedInstallPath):
if sourceRecipe.Type != RecipeType.Build:
log.DoPrintWarning(
"The sourceRecipe type was not of the expected type, aborting delete to be safe"
)
return
log.LogPrint("Removing invalid content at '{0}'".format(
sourceRecipe.ResolvedInstallPath))
IOUtil.SafeRemoveDirectoryTree(sourceRecipe.ResolvedInstallPath)
def ClaimInstallDirNow(log: Log,
targetPath: str,
dstFilePath: str,
sdkPath: str,
forceClaimInstallArea: bool,
logWarning: bool = True) -> JsonDictType:
# Since we are claiming the install area, we check to see if its empty as expected
files = IOUtil.GetFilePaths(targetPath, None)
# Then give the user error about the files at the install area can be lost unless it is changed
if len(files) > 0:
if not forceClaimInstallArea:
raise Exception("The install area at '{0}' was unclaimed, but it was not empty. To allow the tool to use the directory and do with its content as it see fit.\nYou need to rerun the command with the --ForceClaimInstallArea parameter to allow it, but BEWARE that doing so means the content of '{0}' can be lost".format(targetPath))
if logWarning:
log.DoPrintWarning("The install area was not empty but the user enabled --ForceClaimInstallArea and allowed the tool to use it, the files there could be lost because of it.")
# then save the claim file
return BuildAreaInfoFileUtil.SaveInstallAreaInfo(dstFilePath, sdkPath)
def __DoResolveAllPackageDependencies(self, log: Log,
package: Package) -> None:
# FIX: we are doing some of the same checks twice here
addedDict = {} # type: Dict[str, PackageDependency]
# First we resolve all direct dependencies
for dep in package.ResolvedDirectDependencies:
if not dep.Name in addedDict:
package.ResolvedAllDependencies.append(dep)
addedDict[dep.Name] = dep
else:
# The package was already added so we need to check if this dependency is less restrictive than the old one
oldDep = addedDict[dep.Name]
if dep.Access.value < oldDep.Access.value:
package.ResolvedAllDependencies.remove(oldDep)
addedDict[dep.Name] = dep
# Then we pull in the children's dependencies
resolvedDirectDependencies = list(package.ResolvedDirectDependencies)
for directDep in resolvedDirectDependencies:
for dep in directDep.Package.ResolvedAllDependencies:
# ensure that anything we get via a non public access type keeps gets a access type that is >= directDep.Access
if dep.Access.value < directDep.Access.value or dep.Access == AccessType.Private:
#dep = PackageDependency(dep.Package, directDep.Access)
dep = PackageDependency(dep.Package, AccessType.Link)
if not dep.Name in addedDict:
package.ResolvedAllDependencies.append(dep)
addedDict[dep.Name] = dep
else:
# The package was already added so we need to check if this dependency is less restrictive than the old one
oldDep = addedDict[dep.Name]
if dep.Access.value < oldDep.Access.value:
package.ResolvedAllDependencies.remove(oldDep)
package.ResolvedAllDependencies.append(dep)
addedDict[dep.Name] = dep
foundDep = self.__TryFindDep(
package.ResolvedDirectDependencies, dep)
if foundDep is not None:
log.DoPrintWarning(
"Requested dependency access to '{0}', overwritten by dependency from '{1}'"
.format(dep.Name, directDep.Name))
package.ResolvedDirectDependencies.remove(foundDep)
package.ResolvedDirectDependencies.append(dep)
package.ResolvedDirectDependencies.sort(key=lambda s: s.Name.lower())
package.ResolvedAllDependencies.sort(key=lambda s: s.Name.lower())
def __init__(self, log: Log,
platform: GeneratorPluginBase,
configTypeName: str,
packageConfigDict: Dict[str, ToolConfigPackageConfiguration],
genFilename: str,
testModeEnabled: bool) -> None:
self.Log = log
self.ConfigTypeName = configTypeName
# The locations available in this configuration
self.PackageLocations = [] if not configTypeName in packageConfigDict else packageConfigDict[configTypeName].Locations # type: List[ToolConfigPackageLocation]
self.GenFilename = genFilename
self.PackageLocationCache = PackageLocationCache(log, self.PackageLocations, self.GenFilename)
self.__TestModeEnabled = testModeEnabled
self.__InitialSearchLocations = self.__BuildInitialSearchLocations(self.PackageLocations) # type: List[ToolConfigPackageLocation]
if not configTypeName in packageConfigDict:
log.DoPrintWarning("The configuration name '{0}' is unknown, expected one of {1}".format(configTypeName, list(packageConfigDict.keys())))
def __ScanFiles(log: Log, package: Package, filteredFiles: Optional[List[str]],
repairEnabled: bool, thirdpartyExceptionDir: Optional[str],
checkType: CheckType, disableWrite: bool) -> int:
"""
:param filteredFiles: a optional list of specifc files to scan in this package (if supplied the rest should be ignored)
"""
if not package.AllowCheck and checkType == CheckType.Normal:
return 0
if package.AbsolutePath is None:
log.DoPrintWarning("package did not contain a abs path")
return 0
allowedFileSet = None if filteredFiles is None else set(filteredFiles)
errorCount = 0
if package.ResolvedBuildAllIncludeFiles is not None:
for fileName in package.ResolvedBuildAllIncludeFiles:
fullPath = IOUtil.Join(package.AbsolutePath, fileName)
# Only process files with the expected extension
if allowedFileSet is None or fullPath in allowedFileSet:
if __IsValidExtension(fileName, __g_includeExtensionList):
if not __ProcessIncludeFile(
log, package, fullPath, repairEnabled,
thirdpartyExceptionDir, disableWrite):
errorCount += 1
if package.ResolvedBuildSourceFiles is not None:
for fileName in package.ResolvedBuildSourceFiles:
fullPath = IOUtil.Join(package.AbsolutePath, fileName)
if allowedFileSet is None or fullPath in allowedFileSet:
if __IsValidExtension(fileName, __g_includeExtensionList):
if not __ProcessIncludeFile(
log, package, fullPath, repairEnabled,
thirdpartyExceptionDir, disableWrite):
errorCount += 1
elif __IsValidExtension(fileName, __g_sourceExtensionList):
if not __ProcessSourceFile(
log, package, fullPath, repairEnabled,
thirdpartyExceptionDir, disableWrite):
errorCount += 1
return errorCount
def TryLoad(log: Log,
cacheFilename: str) -> Optional['BuildConfigureCache']:
try:
strJson = IOUtil.TryReadFile(cacheFilename)
if strJson is None:
return None
jsonDict = json.loads(strJson)
if jsonDict["Version"] != BuildConfigureCache.CURRENT_VERSION:
raise Exception("Unsupported version")
jsonEnvironmentDict = jsonDict["EnvironmentDict"]
finalEnvironmentDict = {} # type: Dict[str,str]
for key, value in jsonEnvironmentDict.items():
if not isinstance(key, str) or not isinstance(value, str):
raise Exception("json decode failed")
finalEnvironmentDict[key] = value
jsonFileHashDict = jsonDict["FileHashDict"]
finalDict = {} # type: Dict[str,str]
for key, value in jsonFileHashDict.items():
if not isinstance(key, str) or not isinstance(value, str):
raise Exception("json decode failed")
finalDict[key] = value
finalCommandList = [] # type: List[str]
jsonCommandList = jsonDict["CommandList"]
for value in jsonCommandList:
if not isinstance(value, str):
raise Exception("json decode failed")
finalCommandList.append(value)
platformName = jsonDict["PlatformName"] # type: str
fslBuildVersion = jsonDict["FslBuildVersion"] # type: str
allowFindPackage = jsonDict["AllowFindPackage"] # type: str
return BuildConfigureCache(finalEnvironmentDict, finalDict,
finalCommandList, platformName,
fslBuildVersion, allowFindPackage)
except:
log.DoPrintWarning(
"Failed to decode cache file '{0}'".format(cacheFilename))
return None
def TryLoad(log: Log,
cacheFilename: str) -> Optional['JsonProjectIdCache']:
try:
strJson = IOUtil.TryReadFile(cacheFilename)
if strJson is None:
return None
jsonDict = json.loads(strJson)
if jsonDict["Version"] != JsonProjectIdCache.CURRENT_VERSION:
raise Exception("Unsupported version")
jsonProjectIdDict = jsonDict["ProjectIdDict"]
finalDict = {} # type: Dict[str,str]
for key, value in jsonProjectIdDict.items():
if not isinstance(key, str) or not isinstance(value, str):
raise Exception("json decode failed")
finalDict[key] = value
return JsonProjectIdCache(finalDict)
except:
log.DoPrintWarning(
"Failed to decode cache file '{0}'".format(cacheFilename))
return None
def TryAddBuildThreads(log: Log, rArgumentList: List[str],
generatorOriginalName: str, numBuildThreads: int,
buildTypeInfo: PlatformBuildTypeInfo,
cmakeConfig: Optional[GeneratorCMakeConfig],
external: bool) -> bool:
if buildTypeInfo == PlatformBuildTypeInfo.Legacy:
# This is the old add thread arguments code from a simpler time when we only supported one build system per platform
platformName = generatorOriginalName
if (platformName == PlatformNameString.QNX
or platformName == PlatformNameString.YOCTO
or platformName == PlatformNameString.UBUNTU
or platformName == PlatformNameString.FREERTOS or
(external and platformName == PlatformNameString.ANDROID)):
PlatformBuildUtil.AddMakeThreadArgument(
rArgumentList, numBuildThreads)
return True
elif platformName == PlatformNameString.WINDOWS:
PlatformBuildUtil.AddVisualStudioThreadArgument(
rArgumentList, numBuildThreads)
return True
return False
# We are doing somekind of cmake based build.
if cmakeConfig is None:
raise Exception(
"buildTypeInfo {0} requires a cmakeConfig to be supplied".
format(buildTypeInfo))
if buildTypeInfo == PlatformBuildTypeInfo.CMake:
return PlatformBuildUtil.TryAddCMakeThreadArgument(
rArgumentList, numBuildThreads, cmakeConfig)
elif buildTypeInfo == PlatformBuildTypeInfo.CMakeCustom:
return PlatformBuildUtil.TryAddCustomCMakeThreadArgument(
rArgumentList, numBuildThreads, cmakeConfig)
log.DoPrintWarning("Unknown build type info {0}".format(buildTypeInfo))
return False
def __RunVSCode(log: Log, buildPlatformType: BuildPlatformType,
sourcePath: str) -> None:
try:
if log.Verbosity >= 1:
log.LogPrint(
"Opening visual studio code in '{0}'".format(sourcePath))
codeCmd = OpenProjectUtil.__GetCodeCmd(buildPlatformType)
vsCodeCommand = [codeCmd, '.']
result = subprocess.call(vsCodeCommand, cwd=sourcePath)
if result != 0:
log.LogPrintWarning(
"The open vscode command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(
OpenProjectUtil.__SafeJoinCommandArguments(
vsCodeCommand), result, sourcePath))
raise ExitException(result)
except FileNotFoundError:
log.DoPrintWarning(
"The open vscode command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(
OpenProjectUtil.__SafeJoinCommandArguments(vsCodeCommand),
sourcePath))
raise
def RunNinja(log: Log, ninjaExeInfo: ClangExeInfo, ninjaFile: str,
currentWorkingDirectory: str, numBuildThreads: int,
logOutput: bool) -> None:
buildCommand = [ninjaExeInfo.Command, "-f", ninjaFile]
#buildCommand += ["-d", "explain"]
if numBuildThreads > 0:
buildCommand += ["-j", str(numBuildThreads)]
try:
if log.Verbosity >= 4:
log.LogPrint("Running command '{0}' in cwd: {1}".format(
buildCommand, currentWorkingDirectory))
result = RunHelper.RunNow(log, buildCommand,
currentWorkingDirectory, logOutput)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(" ".join(buildCommand), result,
currentWorkingDirectory))
raise ExitException(result)
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(" ".join(buildCommand), currentWorkingDirectory))
raise
def _RunClangFormat(log: Log, toolConfig: ToolConfig,
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, package: Package,
filteredFiles: Optional[List[str]],
repairEnabled: bool) -> None:
if not package.ResolvedBuildAllIncludeFiles or not package.AllowCheck or package.IsVirtual:
return
if package.AbsolutePath is None or package.ResolvedBuildSourceFiles is None:
raise Exception("Invalid package")
allFiles = [] # type: List[str]
if filteredFiles is None:
for fileName in package.ResolvedBuildAllIncludeFiles:
fullPath = IOUtil.Join(package.AbsolutePath, fileName)
# Only process files with the expected extension
if PerformClangUtil.IsValidExtension(
fileName, clangFormatConfiguration.FileExtensions):
allFiles.append(fileName)
for fileName in package.ResolvedBuildSourceFiles:
fullPath = IOUtil.Join(package.AbsolutePath, fileName)
if PerformClangUtil.IsValidExtension(
fileName, clangFormatConfiguration.FileExtensions):
allFiles.append(fileName)
if package.ResolvedContentFiles is not None:
for resolvedPath in package.ResolvedContentFiles:
if PerformClangUtil.IsValidExtension(
resolvedPath.ResolvedPath,
clangFormatConfiguration.FileExtensions):
allFiles.append(resolvedPath.ResolvedPath)
if package.ResolvedContentBuilderSyncInputFiles is not None:
for resolvedPath in package.ResolvedContentBuilderSyncInputFiles:
if PerformClangUtil.IsValidExtension(
resolvedPath.ResolvedPath,
clangFormatConfiguration.FileExtensions):
allFiles.append(resolvedPath.ResolvedPath)
if package.ResolvedContentBuilderBuildInputFiles is not None:
for resolvedPath in package.ResolvedContentBuilderBuildInputFiles:
if PerformClangUtil.IsValidExtension(
resolvedPath.ResolvedPath,
clangFormatConfiguration.FileExtensions):
allFiles.append(resolvedPath.ResolvedPath)
else:
allFiles += filteredFiles
cmd = clangExeInfo.ClangCommand
buildCommand = [cmd, '-style=file']
if repairEnabled:
buildCommand.append('-i')
if len(clangFormatConfiguration.AdditionalUserArguments) > 0:
log.LogPrint("Adding user supplied arguments before '--' {0}".format(
clangFormatConfiguration.AdditionalUserArguments))
buildCommand += clangFormatConfiguration.AdditionalUserArguments
buildCommand += allFiles
currentWorkingDirectory = package.AbsolutePath
FileFinder.FindClosestFileInRoot(log, toolConfig, currentWorkingDirectory,
clangFormatConfiguration.CustomFormatFile)
try:
# if verbose enabled we log the clang-format version
if log.Verbosity >= 4:
log.LogPrint("Running command '{0}' in cwd: {1}".format(
buildCommand, currentWorkingDirectory))
result = subprocess.call(buildCommand, cwd=currentWorkingDirectory)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(" ".join(buildCommand), result,
currentWorkingDirectory))
sys.exit(result)
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(" ".join(buildCommand), currentWorkingDirectory))
raise
def __DoBuildPackagesInOrder(log: Log, configSDKPath: str, configIsDryRun: bool, generatorContext: GeneratorContext,
resolvedBuildOrder: List[Package], builderSettings: BuilderSettings,
packageRecipeResultManager: PackageRecipeResultManager) -> None:
if not generatorContext.RecipePathBuilder.IsEnabled:
log.LogPrintVerbose(3, "External building has been disabled in the Project.gen file")
return
if generatorContext.RecipePathBuilder.TargetLocation is None:
raise Exception("Invalid path builder")
# Claim the 'package' install directory to prevent multiple builds from using the same
# as it would give concurrency issues
BuildAreaInfoFileUtil.ProcessInstallDirClaim(log, generatorContext.RecipePathBuilder.TargetLocation.ResolvedPath, configSDKPath,
builderSettings.ForceClaimInstallArea, __g_installAreaInformationFilename)
if resolvedBuildOrder is None:
log.LogPrintVerbose(2, "No recipes to build")
return
# Filter all packages that don't have a experimental recipe
resolvedBuildOrder = [entry for entry in resolvedBuildOrder if not entry.ResolvedDirectExperimentalRecipe is None]
if len(resolvedBuildOrder) == 0:
log.LogPrintVerbose(2, "No recipes to build")
return
recipePackageStateCache = RecipePackageStateCache(log)
validationEngine = ValidationEngine(log, generatorContext.VariableProcessor, packageRecipeResultManager, generatorContext.ErrorHelpManager)
missingPackagesInBuildOrder = __FindMissingInstallations(log, validationEngine, resolvedBuildOrder, recipePackageStateCache,
generatorContext.CMakeConfig)
builder = PipelineCommandBuilder(generatorContext, builderSettings.CheckBuildCommands, builderSettings.BuildThreads)
recipeRecords = __CreatePipelines(log, builder, missingPackagesInBuildOrder)
for recipeRecord in recipeRecords:
log.LogPrint("Package location: {0}".format(recipeRecord.SourcePackage.AbsolutePath))
try:
log.PushIndent()
if not recipeRecord.SourcePackage.ResolvedPlatformDirectSupported:
raise Exception("The package '{0}' is not supported on this platform".format(recipeRecord.SourcePackage.Name))
if not recipeRecord.Pipeline is None:
log.DoPrint("Building package: {0}".format(recipeRecord.SourcePackage.Name))
if builderSettings.PreDeleteBuild:
# We clear the build path to prepare for a new build
IOUtil.SafeRemoveDirectoryTree(recipeRecord.Pipeline.BuildPath)
for command in recipeRecord.Pipeline.CommandList:
if not configIsDryRun:
command.Execute()
# We finished building, so lets save some information about what we did
BuildInfoFileUtil.SaveBuildInformation(log, recipeRecord, recipePackageStateCache, generatorContext.CMakeConfig,
__g_BuildPackageInformationFilename)
if builderSettings.PostDeleteBuild:
# We clear the build path if a build is successfull
IOUtil.SafeRemoveDirectoryTree(recipeRecord.Pipeline.BuildPath, True)
else:
# Since we are trying to build this it means that the installation validation failed earlier and
# we apparently have no pipelines that could remedy it, so force the install validation to occur so
# we fail early as 'dependent' pipes might fail to build due to this
# generatorContext.RecipeFilterManager
if generatorContext.RecipeFilterManager.AllRecipesEnabled or recipeRecord.SourcePackage.Name in generatorContext.RecipeFilterManager.ContentDict:
log.DoPrintWarning("Missing installation of package '{0}' and no recipe for solving it is available".format(recipeRecord.SourcePackage.Name))
else:
log.LogPrintVerbose(4, "Package '{0}' recipe not enabled".format(recipeRecord.SourcePackage.Name))
validationEngine.Process(recipeRecord.SourcePackage)
finally:
log.PopIndent()
validationEngine.Process(recipeRecord.SourcePackage)
packageCount = len(recipeRecords)
if packageCount > 0:
log.LogPrint("Build {0} packages".format(packageCount))
else:
log.LogPrintVerbose(2, "No recipe was build!")
def _RunClangTidy(
log: Log,
toolConfig: ToolConfig,
platformId: str,
performClangTidyConfig: PerformClangTidyConfig,
clangExeInfo: ClangExeInfo,
package: Package,
filteredFiles: Optional[List[str]],
clangFormatFilename: Optional[str],
localVariantInfo: LocalVariantInfo,
virtualVariantEnvironmentCache: VirtualVariantEnvironmentCache,
logOutput: bool = False) -> None:
if package.ResolvedBuildAllIncludeFiles is None:
raise Exception("invalid package")
log.LogPrint("- {0}".format(package.Name))
clangTidyConfiguration = performClangTidyConfig.ClangTidyConfiguration
if package.AbsolutePath is None or package.ResolvedBuildSourceFiles is None:
raise Exception("Invalid package")
allFiles = [] # type: List[str]
if filteredFiles is None:
for fileName in package.ResolvedBuildAllIncludeFiles:
fullPath = IOUtil.Join(package.AbsolutePath, fileName)
# Only process files with the expected extension
if PerformClangUtil.IsValidExtension(
fileName, clangTidyConfiguration.FileExtensions):
allFiles.append(fileName)
for fileName in package.ResolvedBuildSourceFiles:
fullPath = IOUtil.Join(package.AbsolutePath, fileName)
if PerformClangUtil.IsValidExtension(
fileName, clangTidyConfiguration.FileExtensions):
allFiles.append(fileName)
else:
for fileName in filteredFiles:
if PerformClangUtil.IsValidExtension(
fileName, clangTidyConfiguration.FileExtensions):
allFiles.append(fileName)
# ensure we process the files in the same order every time
allFiles.sort()
platformCompilerFlags = [] # type: List[str]
platformDefineCommands = [] # type: List[str]
platformStrictChecks = [] # type: List[str]
if platformId in clangTidyConfiguration.PlatformDict:
clangPlatformConfig = clangTidyConfiguration.PlatformDict[platformId]
platformCompilerFlags = clangPlatformConfig.Compiler.Flags
for platformDefine in clangPlatformConfig.Defines.All:
platformDefineCommands.append('-D')
platformDefineCommands.append(platformDefine)
# We default to release for now
for platformDefine in clangPlatformConfig.Defines.Release:
platformDefineCommands.append('-D')
platformDefineCommands.append(platformDefine)
for strictCheck in clangPlatformConfig.StrictChecks:
platformStrictChecks.append(strictCheck)
includePaths = __BuildClangTidyPackageIncludePaths(
log, localVariantInfo, virtualVariantEnvironmentCache, package)
packageDefineCommands = __BuildClangTidyPackageDefines(
log, localVariantInfo, package)
cmd = clangExeInfo.ClangCommand
buildCommand = [cmd]
if performClangTidyConfig.Repair:
buildCommand.append('-fix')
#buildCommand.append('-fix-errors')
#buildCommand.append('-header-filter=.*')
usingCheckCommand = False
if len(performClangTidyConfig.OverrideChecks) > 0:
newOverrideChecks = ",".join(performClangTidyConfig.OverrideChecks)
log.LogPrintVerbose(
2, "Overriding checks checks '{0}'".format(newOverrideChecks))
if performClangTidyConfig.StrictChecks:
log.DoPrintWarning(
"Ignoreing strict checks because 'override' is enabled")
buildCommand.append("--checks")
buildCommand.append(newOverrideChecks)
usingCheckCommand = True
elif performClangTidyConfig.StrictChecks and len(platformStrictChecks) > 0:
newStrictChecks = ",".join(platformStrictChecks)
log.LogPrintVerbose(
2, "Adding strict checks '{0}'".format(newStrictChecks))
buildCommand.append("--checks")
buildCommand.append(newStrictChecks)
usingCheckCommand = True
if len(performClangTidyConfig.AdditionalUserArguments) > 0:
log.LogPrintVerbose(
2, "Adding user supplied arguments before '--' {0}".format(
performClangTidyConfig.AdditionalUserArguments))
if usingCheckCommand and '--checks' in performClangTidyConfig.AdditionalUserArguments:
log.DoPrintWarning(
"another command is adding '--checks' so it could conflict with the user supplied argument"
)
buildCommand += performClangTidyConfig.AdditionalUserArguments
buildCommand += allFiles
buildCommand.append('--')
if len(platformCompilerFlags) > 0:
buildCommand += __LookupEnvironmentVariables(
platformCompilerFlags, virtualVariantEnvironmentCache)
if len(platformDefineCommands) > 0:
buildCommand += platformDefineCommands
if len(includePaths) > 0:
buildCommand += includePaths
if len(packageDefineCommands) > 0:
buildCommand += packageDefineCommands
if len(performClangTidyConfig.PostfixArguments) > 0:
log.LogPrintVerbose(
2, "Adding user supplied arguments after '--' {0}".format(
performClangTidyConfig.PostfixArguments))
buildCommand += performClangTidyConfig.PostfixArguments
currentWorkingDirectory = package.AbsolutePath
if clangFormatFilename is not None:
FileFinder.FindClosestFileInRoot(log, toolConfig,
currentWorkingDirectory,
clangFormatFilename)
FileFinder.FindClosestFileInRoot(log, toolConfig, currentWorkingDirectory,
clangTidyConfiguration.CustomTidyFile)
try:
if log.Verbosity >= 4:
log.LogPrint("Running command '{0}' in cwd: {1}".format(
buildCommand, currentWorkingDirectory))
result = __RunNow(log, buildCommand, currentWorkingDirectory,
logOutput)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(" ".join(buildCommand), result,
currentWorkingDirectory))
raise ExitException(result)
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(" ".join(buildCommand), currentWorkingDirectory))
raise
def Run(log: Log, toolConfig: ToolConfig, platformId: str,
topLevelPackage: Package, tidyPackageList: List[Package],
userBuildVariantsDict: Dict[str, str], pythonScriptRoot: str,
generatorContext: GeneratorContext, sdkConfigTemplatePath: str,
packageRecipeResultManager: PackageRecipeResultManager,
performClangTidyConfig: PerformClangTidyConfig,
customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatFilename: Optional[str], buildThreads: int) -> None:
"""
RunClangTidy on a package at a time
:param topLevelPackage: the top level system package
:param tidyPackageList: the packages to 'tidy'
:param userBuildVariantsDict: the variant configuration supplied by the user
:param performClangTidyConfig:
:param customPackageFileFilter:
:param clangFormatFilename:
:param repairEnabled:
"""
# Lookup the recommended build threads using the standard build algorithm
buildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(
buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
performClangTidyConfig.ClangTidyConfiguration.RecipePackageName,
MagicValues.ClangTidyCommand)
BuildVariantUtil.ValidateUserVariantSettings(log, topLevelPackage,
userBuildVariantsDict)
BuildVariantUtil.LogVariantSettings(log, userBuildVariantsDict)
resolvedVariantSettingsDict = BuildVariantUtil.CreateCompleteStaticVariantSettings(
topLevelPackage.ResolvedAllVariantDict, userBuildVariantsDict)
if log.Verbosity >= 1:
PerformClangUtil.ShowVersion(log, clangExeInfo)
log.LogPrint("Running clang-tidy")
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in tidyPackageList
if PerformClangUtil.CanProcessPackage(package)
]
generatorConfig = GeneratorConfig(sdkConfigTemplatePath, toolConfig)
generatorReportDict = generatorContext.Generator.GenerateReport(
log, generatorConfig, finalPackageList)
# Validate report dict
for package in finalPackageList:
if package not in generatorReportDict:
raise Exception(
"Generator report is missing information for package '{0}'"
.format(package.Name))
localVariantInfo = LocalVariantInfo(resolvedVariantSettingsDict,
generatorReportDict,
pythonScriptRoot)
count = PerformClangTidyHelper.ProcessAllPackages(
log, toolConfig, platformId, pythonScriptRoot,
performClangTidyConfig, clangExeInfo, finalPackageList,
customPackageFileFilter, localVariantInfo, buildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning(
"No files processed, could not find a package that matches {0}"
.format(customPackageFileFilter))