def Scan(log: Log, scanPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
repairEnabled: bool, thirdpartyExceptionDir: Optional[str],
checkType: CheckType, disableWrite: bool) -> None:
"""
Run through all source files that are part of the packages and check for common errors
:param scanPackageList: the packages that will be scanned.
"""
log.LogPrint("Running source scan")
extensionList = __g_includeExtensionList + __g_sourceExtensionList
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in scanPackageList
if PerformClangUtil.CanProcessPackage(package)
]
totalErrorCount = 0
for package in finalPackageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, extensionList)
if customPackageFileFilter is None or filteredFiles is not None:
totalErrorCount += __ScanFiles(log, package, filteredFiles,
repairEnabled,
thirdpartyExceptionDir, checkType,
disableWrite)
if totalErrorCount > 0 and not repairEnabled:
log.DoPrint(
"BEWARE: If you have made a backup of your files you can try to auto correct the errors with '--Repair' but do so at your own peril"
)
def ProcessInstallDirClaim(log: Log,
targetPath: str,
sdkPath: str,
forceClaimInstallArea: bool,
installAreaInfoPath: str) -> None:
filePath = IOUtil.Join(targetPath, installAreaInfoPath)
# Beware that this method to claim a area is in no way secure
# It will basically only be useful after the claim file has been created
# So we start with a possible race condition, but any future attempts to use the directory
# will catch that multiple repos are trying to reuse the same install area
fileContent = IOUtil.TryReadFile(filePath)
if fileContent is None:
log.LogPrint("Install area '{0}' is unclaimed, claiming it".format(targetPath))
BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea)
return
jsonBuildInfoDict = json.loads(fileContent)
if not BuildAreaInfoFile.IsDictValid(jsonBuildInfoDict):
if not forceClaimInstallArea:
raise Exception("Install area '{0}' contained an invalid file '{1}', did you try to run concurrent builds using the same install directory or did it get corrupted? Delete the file to allow the build to continue or use --ForceClaimInstallArea to do so automatically. You could also set up a readonly cache area to reuse between repos see the documentation for more info.".format(targetPath, filePath))
log.DoPrintWarning("Install area '{0}' contained an invalid file '{1}', as --ForceClaimInstallArea was specified it was overwritten and '{2}' now controls it".format(targetPath, filePath, sdkPath))
jsonBuildInfoDict = BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea, logWarning=False)
buildAreaInfoFile = BuildAreaInfoFile(jsonBuildInfoDict)
if buildAreaInfoFile.SDKPath != sdkPath:
if not forceClaimInstallArea:
raise Exception("The Install area at '{0}' is already claimed by the sdk at '{1}' so the sdk at '{2}' can not reuse it as it could give concurrency issues if multiple builds execute at the same time. If you are sure that you are not doing concurrent builds and you just want to use the area for a new SDK you can force claim it with --ForceClaimInstallArea. You could also set up a readonly cache area to reuse between repos see the documentation for more info.".format(targetPath, buildAreaInfoFile.SDKPath, sdkPath))
log.DoPrintWarning("The Install area at '{0}' was already claimed by the sdk at '{1}' but '{2}' took control of it as --ForceClaimInstallArea was specified.".format(targetPath, buildAreaInfoFile.SDKPath, sdkPath))
jsonBuildInfoDict = BuildAreaInfoFileUtil.ClaimInstallDirNow(log, targetPath, filePath, sdkPath, forceClaimInstallArea, logWarning=False)
def __init__(self, log: Log, packageName: str,
xmlExperimentalRecipe: XmlExperimentalRecipe) -> None:
self._Log = log
self.XmlSource = xmlExperimentalRecipe
self.Name = xmlExperimentalRecipe.Name # type: str
self.Type = self.__DetermineRecipeType(xmlExperimentalRecipe)
self.Pipeline = xmlExperimentalRecipe.Pipeline if self.Type == RecipeType.Build else None
self.ValidateInstallation = xmlExperimentalRecipe.ValidateInstallation
self.IsLocalSourceBuild = False
if (self.Pipeline is not None and len(self.Pipeline.CommandList) > 0
and self.Pipeline.CommandList[0].CommandType
== BuildRecipePipelineCommand.Source):
self.IsLocalSourceBuild = True
# The installation path of the package
self.ResolvedInstallPath = None # type: Optional[str]
if self.Type == RecipeType.Undefined:
log.DoPrintWarning(
"No installation or validation available for package '{0}' recipe '{1}'"
.format(packageName, self.Name))
if self.ValidateInstallation is None:
log.DoPrintWarning(
"No installation validation available for package '{0}' recipe '{1}'"
.format(packageName, self.Name))
def TryRun(log: Log, cmdList: List[str]) -> Optional[str]:
"""
Run the command and capture the output
:return: the captured output on sucess, None if it failed
"""
try:
if cmdList[0].endswith(
'.py') and PlatformUtil.DetectBuildPlatformType(
) == BuildPlatformType.Windows:
cmdList[0] = cmdList[0][:-3] + ".bat"
with subprocess.Popen(cmdList,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=True) as proc:
output = proc.stdout.read().strip()
proc.stdout.close()
result = proc.wait()
if result != 0:
LocalUtil.DumpCapture(log, 4, output)
log.LogPrintWarning(
"The command '{0}' failed with '{1}'".format(
" ".join(cmdList), result))
return None
if isinstance(output, str):
return output
return None
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'.".format(
" ".join(cmdList)))
return None
def __AddParentFeatures(log: Log, featureNameList: List[str],
requirementTree: Union[RequirementTree, AppInfoGlobalRequirementTree],
useStrictFeatureWarning: bool) -> List[str]:
if '*' in featureNameList:
return featureNameList
featureNameList.sort()
if log.Verbosity > 1:
log.LogPrint("Automatically adding features to supplied feature list {0}".format(featureNameList))
featureNameSet = set(featureNameList)
for featureName in featureNameList:
if featureName in requirementTree.FeatureToNodeDict:
requirementNode = requirementTree.FeatureToNodeDict[featureName]
currentNode = requirementNode # type: Optional[Union[RequirementTreeNode, AppInfoGlobalRequirementTreeNode]]
while currentNode is not None:
if currentNode.Content is not None:
if not currentNode.Content.Name in featureNameSet:
featureNameSet.add(currentNode.Content.Name)
if log.Verbosity > 1 and requirementNode.Content is not None:
log.LogPrint("- '{0}' because '{1}' depends on it".format(currentNode.Content.Name, requirementNode.Content.Name))
currentNode = currentNode.Parent
else:
featureNameSet.remove(featureName)
if useStrictFeatureWarning:
log.DoPrintWarning("Unknown feature name '{0}' in filterNameList {1}".format(featureName, featureNameList))
else:
# For now just log a warning
log.LogPrintVerbose(5, "Unknown feature name '{0}' in filterNameList {1}".format(featureName, featureNameList))
resultList = list(featureNameSet)
resultList.sort()
return resultList
def __init__(self, log: Log, packageName: str, xmlExperimentalRecipe: XmlExperimentalRecipe, forceDisable: bool) -> None:
"""
forceDisable will not disable 'the external' recipe type used for build tools
"""
self._Log = log
self.SysXmlSource = xmlExperimentalRecipe
self.Version = xmlExperimentalRecipe.Version # type: Optional[Version]
self.ShortName = xmlExperimentalRecipe.ShortName
self.FullName = xmlExperimentalRecipe.FullName
self.AllowFind = xmlExperimentalRecipe.Find
determinedType = self.__DetermineRecipeType(xmlExperimentalRecipe)
self.Type = determinedType if not forceDisable or determinedType == RecipeType.External else RecipeType.Disabled
self.Pipeline = xmlExperimentalRecipe.Pipeline if self.Type == RecipeType.Build else None
self.ValidateInstallation = xmlExperimentalRecipe.ValidateInstallation if self.Type != RecipeType.Disabled else None
self.IsLocalSourceBuild = False
if (self.Pipeline is not None and len(self.Pipeline.CommandList) > 0 and
self.Pipeline.CommandList[0].CommandType == BuildRecipePipelineCommand.Source):
self.IsLocalSourceBuild = True
# The installation path of the package
self.ResolvedInstallLocation = None # type: Optional[ResolvedPath]
if self.Type == RecipeType.Undefined:
log.DoPrintWarning("No installation or validation available for package '{0}' recipe '{1}'".format(packageName, self.FullName))
if self.ValidateInstallation is None and self.Type != RecipeType.Disabled:
log.DoPrintWarning("No installation validation available for package '{0}' recipe '{1}'".format(packageName, self.FullName))
if forceDisable and log.Verbosity >= 4:
if self.Type == RecipeType.Disabled:
log.LogPrint(" - Force disabling recipe for package {0}".format(packageName))
else:
log.LogPrint(" - Force disabling recipe for package {0} ignored due to type".format(packageName))
def BuildContentState(self, log: Log,
pathFileRecord: PathRecord,
allowCaching: bool, allowNew: bool,
cachedSyncState: Optional['SyncState'] = None) -> ContentState:
fileState = ContentState()
fileState.Name = pathFileRecord.RelativePath
fileState.Length = os.path.getsize(pathFileRecord.ResolvedPath)
fileState.ModifiedDate = self.__FileModificationDate(pathFileRecord.ResolvedPath)
fileState.TagChecksum = '0'
cachedState = cachedSyncState.TryGetFileState(fileState) if cachedSyncState is not None else None
if allowCaching and cachedState is not None and fileState.Length == cachedState.Length and fileState.ModifiedDate == cachedState.ModifiedDate:
fileState.Checksum = cachedState.Checksum
fileState.TagChecksum = cachedState.TagChecksum
log.LogPrintVerbose(2, "Using cached checksum for '{0}'".format(fileState.Name))
else:
log.LogPrintVerbose(2, "Calculating checksum for '{0}'".format(fileState.Name))
fileState.Checksum = IOUtil.HashFile(pathFileRecord.ResolvedPath)
# Mark the entry as being new
#if (cachedState is None or CacheState.New) and allowNew:
if cachedState is None and allowNew:
fileState.CacheState = CacheState.New
elif cachedState is not None and not fileState.IsSameState(cachedState):
fileState.CacheState = CacheState.Modified
fileState.ModificationComment = fileState.GetDifferenceString(cachedState)
else:
fileState.CacheState = CacheState.Unmodified
return fileState
def ShowExtensionList(log: Log, topLevelPackage: Package,
requestedFiles: Optional[List[str]]) -> None:
# As the packages in requestedFiles might have been filtered at this point (and any issues already caught), we just ignore not found
requestedPackages = PackageUtil.GetPackageListFromFilenames(
topLevelPackage, requestedFiles, True)
requirements = RequirementFilter.GetRequirementList(
topLevelPackage, requestedPackages,
PackageRequirementTypeString.Extension)
if len(requirements) <= 0:
log.DoPrint("Extensions: None")
return
log.DoPrint("Extensions:")
# Pretty print useful information in name sorted order
requirements.sort(key=lambda s: s.Id)
currentIndent = " "
for requirement in requirements:
strFormat = "{0}- '{1}'"
if len(requirement.Version) > 0:
strFormat += " V{2}"
if len(requirement.Extends) > 0:
strFormat += " extends '{3}'"
strFormat += " (introduced by package: {4})"
log.DoPrint(
strFormat.format(currentIndent, requirement.Name,
requirement.Version, requirement.Extends,
", ".join(requirement.IntroducedByPackages)))
def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration,
cmakeConfig: GeneratorCMakeConfig, repairEnabled: bool,
buildThreads: int, useLegacyTidyMethod: bool) -> None:
""" RunClangFormat on a package at a time """
# Lookup the recommended build threads using the standard build algorithm
numBuildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(
buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
ninjaExeInfo = PerformClangUtil.LookupRecipeResults(
packageRecipeResultManager,
clangFormatConfiguration.NinjaRecipePackageName,
MagicValues.NinjaCommand)
log.LogPrint("ClangFormat version: {0}".format(clangExeInfo.Version))
log.LogPrint("Ninja version: {0}".format(ninjaExeInfo.Version))
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in formatPackageList
if PerformClangUtil.CanProcessPackage(package)
]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
#test = set()
#for entry in sortedPackages:
# if entry.Name in test:
# raise Exception("duplicated package")
# else:
# test.add(entry.Name)
if useLegacyTidyMethod:
count = PerformClangFormatHelper.Process(
log, toolConfig, customPackageFileFilter,
clangFormatConfiguration, clangExeInfo, sortedPackages,
repairEnabled, numBuildThreads)
else:
count = PerformClangFormatHelper2.Process(
log, toolConfig, customPackageFileFilter,
clangFormatConfiguration, cmakeConfig, clangExeInfo,
ninjaExeInfo, sortedPackages, repairEnabled, numBuildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning(
"No files processed, could not find a package that matches {0}"
.format(customPackageFileFilter))
def Run(log: Log, toolConfig: ToolConfig, formatPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
packageRecipeResultManager: PackageRecipeResultManager,
clangFormatConfiguration: ClangFormatConfiguration, repairEnabled: bool, buildThreads: int) -> None:
""" RunClangFormat on a package at a time """
# Lookup the recommended build threads using the standard build algorithm
buildThreads = PlatformBuildUtil.GetRecommendedBuildThreads(buildThreads)
clangExeInfo = PerformClangUtil.LookupRecipeResults(packageRecipeResultManager, clangFormatConfiguration.RecipePackageName,
MagicValues.ClangFormatCommand)
if log.Verbosity >= 1:
PerformClangUtil.ShowVersion(log, clangExeInfo)
log.LogPrint("Running clang-format")
# Filter the package list so it only contains things we can process
finalPackageList = [package for package in formatPackageList if PerformClangUtil.CanProcessPackage(package)]
sortedPackages = list(finalPackageList)
sortedPackages.sort(key=lambda s: s.Name.lower())
count = PerformClangFormatHelper.Process(log, toolConfig, customPackageFileFilter, clangFormatConfiguration, clangExeInfo, sortedPackages,
repairEnabled, buildThreads)
if count == 0:
if customPackageFileFilter is None:
log.DoPrintWarning("No files processed")
else:
log.DoPrintWarning("No files processed, could not find a package that matches {0}".format(customPackageFileFilter))
def CheckWindowsGLES(log: Log) -> None:
log.LogPrint("- OpenGL ES environment")
try:
emulator = os.environ.get("FSL_GLES_NAME")
if not emulator:
raise EnvironmentError(
"OpenGLES emulation 'FSL_GLES_NAME' environment variable is not set."
)
emulator = emulator.lower()
if (emulator == 'arm' or emulator == 'default'):
CheckWindowsGLESArm(log)
elif (emulator == 'mesa'):
CheckWindowsGLESPowerVR(log)
elif (emulator == 'powervr'):
CheckWindowsGLESMesa(log)
elif (emulator == 'qualcomm'):
CheckWindowsGLESQualcomm(log)
elif (emulator == 'vivante'):
CheckWindowsGLESVivante(log)
else:
log.LogPrint("WARNING: Unknown emulator '{0}'".format(emulator))
except:
print(
"ERROR: CheckWindowsGLES failed.\nThis indicates that the OpenGLES emulation environment was not correctly setup and the code wont be able to locate the necessary OpenGLES header files. Possible causes:\n- Are you using the right emulator type: x64 or x86 see documentation.\n- Did you run the required setup scripts?\n- If building from visual studio did you launch the project via .StartProject.bat\n"
)
raise
def Process(log: Log, toolConfig: ToolConfig, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, cmakeConfig: GeneratorCMakeConfig,
clangFormatExeInfo: ClangExeInfo, ninjaExeInfo: ClangExeInfo, sortedPackageList: List[Package],
repairEnabled: bool, numBuildThreads: int) -> int:
totalProcessedCount = 0
logOutput = False
currentWorkingDirectory = BuildUtil.GetBuildDir(log, toolConfig, cmakeConfig.CheckDir)
currentWorkingDirectory = IOUtil.Join(currentWorkingDirectory, "format")
ninjaOutputFile = IOUtil.Join(currentWorkingDirectory, "build.ninja")
toolVersionOutputFile = IOUtil.Join(currentWorkingDirectory, "ToolVersions.txt")
IOUtil.SafeMakeDirs(currentWorkingDirectory)
log.LogPrint("Using path: '{0}'".format(currentWorkingDirectory))
log.LogPrint("Storing tool versions.")
PerformClangFormatHelper2.WriteToolVersionFile(log, toolVersionOutputFile, clangFormatExeInfo, ninjaExeInfo)
log.LogPrint("Generating ninja format file.")
#foundFixes = PerformClangTidyHelper2.FindFixes(log, clangTidyFixOutputFolder)
#if len(foundFixes) > 0:
# PerformClangTidyHelper2.DeleteFixes(log, clangTidyFixOutputFolder, foundFixes)
totalProcessedCount = PerformClangFormatHelper2.GenerateNinjaTidyFile(log, toolConfig, ninjaOutputFile, currentWorkingDirectory,
toolVersionOutputFile, clangFormatExeInfo, customPackageFileFilter,
clangFormatConfiguration, sortedPackageList, repairEnabled)
log.LogPrint("Executing ninja format file.")
RunHelper.RunNinja(log, ninjaExeInfo, ninjaOutputFile, currentWorkingDirectory, numBuildThreads, logOutput)
return totalProcessedCount
def __FiltersPackagesByExtensions(
log: Log, packages: Union[List[Package], List[AppInfoPackage]],
extensionNameList: ExtensionListManager, featureNameList: List[str],
requirementTree: Union[RequirementTree, AppInfoGlobalRequirementTree]
) -> List[T]:
""" Filter packages by extensions.
If '*' is in the extensionNameList a clone of 'packages' will be returned
Else we return a list containing only the packages that can be build with the available extensions
"""
if extensionNameList.AllowAllExtensions:
log.LogPrint("Filtering by extensions: All")
return cast(List[T], list(packages))
else:
log.LogPrint("Filtering by extensions: {0}".format(", ".join([
str(qualifiedName)
for qualifiedName in extensionNameList.Content
])))
filteredPackageList = []
for package in packages:
if PackageFilter.__IsAllPackageExtensionAvailable(
package, requirementTree):
filteredPackageList.append(package)
elif package.Type == PackageType.Library or package.Type == PackageType.Executable or package.ResolvedDirectExperimentalRecipe != None:
missingNames = PackageFilter.__GetCompleteMissingExtensionNames(
package, requirementTree)
log.LogPrint(
"Could not build package '{0}' due to missing extension '{1}'"
.format(package.Name, ", ".join(missingNames)))
return cast(List[T], filteredPackageList)
def __FiltersPackagesByFeatures(log: Log,
packages: Union[List[Package],
List[AppInfoPackage]],
featureNameList: List[str]) -> List[T]:
""" Filter packages by features.
If '*' is in the featureNameList a clone of 'packages' will be returned
Else we return a list containing only the packages that can be build with the available features
"""
if '*' in featureNameList:
log.LogPrint("Filtering by features: All")
return cast(List[T], list(packages))
else:
log.LogPrint("Filtering by features: {0}".format(
", ".join(featureNameList)))
filteredPackageList = []
for package in packages:
if PackageFilter.__FeaturesAvailable(package, featureNameList):
filteredPackageList.append(package)
elif package.Type == PackageType.Library or package.Type == PackageType.Executable or package.ResolvedDirectExperimentalRecipe is not None:
missingFeatures = PackageFilter.__GetCompleteMissingFeatureNames(
package, featureNameList)
log.LogPrint(
"Could not build package '{0}' due to missing features '{1}'"
.format(package.Name, ", ".join(missingFeatures)))
return cast(List[T], filteredPackageList)
def ShowRequirementList(log: Log,
basicConfig: BasicConfig,
topLevelPackage: Package,
requestedFiles: Optional[List[str]],
showFeaturesOnly: bool = False) -> None:
message = "Requirements" if not showFeaturesOnly else "Features"
filterName = None if not showFeaturesOnly else PackageRequirementTypeString.Feature
# As the packages in requestedFiles might have been filtered at this point (and any issues already caught), we just ignore not found
requestedPackages = PackageUtil.GetPackageListFromFilenames(
topLevelPackage, requestedFiles, True)
requirements = RequirementFilter.GetRequirementList(
topLevelPackage, requestedPackages, filterName)
if len(requirements) <= 0:
log.DoPrint("{0}: None".format(message))
return
log.DoPrint("{0}:".format(message))
rootNode = RequirementTree(requirements).RootNode
strAddIndent = " "
# We only show the type group info when there is more than one
#showTypeGroup = len(rootNode.Children) > 1
baseIndent = "" #strAddIndent if len(rootNode.Children) > 1 else ""
sortedFeatures = list(rootNode.Children)
sortedFeatures.sort(
key=lambda s: None if s.Content is None else s.Content.Id)
for sortedFeature in sortedFeatures:
__PrintRequirementsNode(log, sortedFeature, baseIndent, strAddIndent)
def __init__(self, log: Log, rootDirs: List[ToolConfigRootDirectory],
basedUponXML: XmlExperimental, configFileName: str,
projectRootDirectory: str) -> None:
super(ToolConfigExperimental, self).__init__()
self.BasedOn = basedUponXML
self.DefaultThirdPartyInstallReadonlyCacheDirectory = None # type: Optional[ToolConfigExperimentalDefaultThirdPartyInstallDirectory]
self.AllowDownloads = basedUponXML.AllowDownloads
self.DisableDownloadEnv = basedUponXML.DisableDownloadEnv
self.DefaultThirdPartyInstallDirectory = ToolConfigExperimentalDefaultThirdPartyInstallDirectory(
log, basedUponXML.DefaultThirdPartyInstallDirectory,
"DefaultThirdPartyInstallDirectory", False)
self.DefaultThirdPartyInstallReadonlyCacheDirectory = self.__TryCreateReadonlyCache(
log, basedUponXML.DefaultThirdPartyInstallReadonlyCacheDirectory)
if log.Verbosity > 1:
log.LogPrint(
"DefaultThirdPartyInstallDirectory: {0} which resolves to '{1}'"
.format(self.DefaultThirdPartyInstallDirectory.Name,
self.DefaultThirdPartyInstallDirectory.ResolvedPath))
if not self.DefaultThirdPartyInstallReadonlyCacheDirectory is None:
log.LogPrint(
"DefaultThirdPartyInstallReadonlyCacheDirectory: {0} which resolves to '{1}'"
.format(
self.DefaultThirdPartyInstallReadonlyCacheDirectory.
Name,
self.DefaultThirdPartyInstallReadonlyCacheDirectory.
ResolvedPath))
def BuildGeneratorCMakeConfig(log: Log, toolVersion: Version, platformName: str, buildVariantConfig: BuildVariantConfig,
userCMakeConfig: Optional[UserCMakeConfig], cmakeConfiguration: CMakeConfiguration,
defaultCompilerVersion: int, isCheckMode: bool) -> GeneratorCMakeConfig:
"""
Build the CMake config based on the supplied parameters and the default settings from the toolconfig
"""
# Setup default configuration
buildDir = IOUtil.Join(cmakeConfiguration.DefaultBuildDir, platformName)
generatorName = ""
installPrefix = cmakeConfiguration.DefaultInstallPrefix
# Give the platform a chance to override the config
platformConfig = cmakeConfiguration.TryGetPlatformConfig(platformName)
allowFindPackage = True
if platformConfig is not None:
if platformConfig.DefaultGeneratorName is not None:
generatorName = platformConfig.DefaultGeneratorName
if platformConfig.DefaultInstallPrefix is not None:
installPrefix = platformConfig.DefaultInstallPrefix
if platformConfig.AllowFindPackage is not None:
allowFindPackage = platformConfig.AllowFindPackage
log.LogPrintVerbose(2, "project defined AllowFindPackage to {0}".format(allowFindPackage))
# Apply the commandline overrides (so the user gets the final say)
buildDirSetByUser = False
if userCMakeConfig is not None:
if userCMakeConfig.BuildDir is not None:
buildDir = userCMakeConfig.BuildDir
buildDirSetByUser = True
if userCMakeConfig.GeneratorName is not None:
generatorName = userCMakeConfig.GeneratorName
if userCMakeConfig.InstallPrefix is not None:
installPrefix = userCMakeConfig.InstallPrefix
if userCMakeConfig.AllowFindPackage is not None:
allowFindPackage = userCMakeConfig.AllowFindPackage
log.LogPrintVerbose(2, "Command line set AllowFindPackage to {0}".format(allowFindPackage))
# If we still dont have a generator name then try to select a good default
if len(generatorName) <= 0:
# Try to determine the default generator name for the platform
generatorName = CMakeHelper.GetPlatformDefaultCMakeGenerator(platformName, defaultCompilerVersion)
cmakeVersion = CMakeUtil.GetVersion()
cmakeConfigGlobalArgs = [] if userCMakeConfig is None else shlex.split(userCMakeConfig.ConfigGlobalArgs)
cmakeConfigAppArgs = [] if userCMakeConfig is None else shlex.split(userCMakeConfig.ConfigAppArgs)
checkDir = IOUtil.Join(buildDir, 'fsl')
if isCheckMode:
buildDir = checkDir
return GeneratorCMakeConfig(toolVersion, platformName, buildVariantConfig, buildDir, buildDirSetByUser, checkDir, generatorName, installPrefix,
cmakeVersion, cmakeConfigGlobalArgs, cmakeConfigAppArgs, allowFindPackage)
def _RunClangFormat(log: Log, toolConfig: ToolConfig,
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, package: Package,
filteredFiles: Optional[List[str]],
repairEnabled: bool) -> None:
if package.ResolvedBuildAllIncludeFiles is None or len(
package.ResolvedBuildAllIncludeFiles
) <= 0 or not package.AllowCheck or package.IsVirtual:
return
if package.AbsolutePath is None or package.ResolvedBuildSourceFiles is None:
raise Exception("Invalid package")
formatPackageConfig = FormatPackageConfig(log, package,
clangFormatConfiguration,
filteredFiles)
cmd = clangExeInfo.Command
buildCommand = [cmd, '-style=file']
if repairEnabled:
buildCommand.append('-i')
if len(clangFormatConfiguration.AdditionalUserArguments) > 0:
log.LogPrint("Adding user supplied arguments before '--' {0}".format(
clangFormatConfiguration.AdditionalUserArguments))
buildCommand += clangFormatConfiguration.AdditionalUserArguments
buildCommand += [
entry.ResolvedPath for entry in formatPackageConfig.AllFiles
]
currentWorkingDirectory = package.AbsolutePath
FileFinder.FindClosestFileInRoot(log, toolConfig, currentWorkingDirectory,
clangFormatConfiguration.CustomFormatFile)
try:
# if verbose enabled we log the clang-format version
if log.Verbosity >= 4:
log.LogPrint("Running command '{0}' in cwd: {1}".format(
buildCommand, currentWorkingDirectory))
result = subprocess.call(buildCommand, cwd=currentWorkingDirectory)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'. It was run with CWD: '{2}'"
.format(" ".join(buildCommand), result,
currentWorkingDirectory))
raise ExitException(result)
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'. It was run with CWD: '{1}'"
.format(" ".join(buildCommand), currentWorkingDirectory))
raise
def __CreatePipelines(log: Log,
builder: PipelineCommandBuilder,
resolvedBuildOrder: List[Package]) -> List[RecipeRecord]:
pipelines = [] # type: List[RecipeRecord]
for package in resolvedBuildOrder:
log.LogPrint("Creating package {0} build pipelines".format(package.Name))
try:
log.PushIndent()
record = RecipeRecord(log, builder, package)
pipelines.append(record)
finally:
log.PopIndent()
return pipelines
def Resolve(log: Log,
allPackages: List[UnresolvedBasicPackage],
dump: bool = False) -> List[ResolvedPackageInstance]:
log.LogPrintVerbose(LocalVerbosityLevel.Info,
"Initial package resolve")
log.PushIndent()
try:
res = [] # type: List[ResolvedPackageInstance]
# Build a package dictionary for quick lookup
allPackageDict = dict() # type: Dict[str, UnresolvedBasicPackage]
if len(allPackages) > 0:
allPackages = list(allPackages)
allPackages.sort(key=lambda s: s.Name.Value.upper())
log.LogPrintVerbose(LocalVerbosityLevel.Debug,
"Available packages")
#numTopLevel = 0
#for package in allPackages:
# if package.Type == PackageType.TopLevel:
# numTopLevel = numTopLevel + 1
# else:
# log.LogPrintVerbose(LocalVerbosityLevel.Debug, "- {0}".format(package))
# allPackageDict[package.Name.Value] = package
#if numTopLevel != 1:
# raise Exception("Missing a top level package");
#graph = PackageGraphBuilder.Build(log, list(allPackageDict.values()))
for package in allPackages:
if package.Type == PackageType.TopLevel:
raise Exception("Unsupported package type")
if log.Verbosity >= LocalVerbosityLevel.Debug:
log.LogPrint("- {0}".format(package))
allPackageDict[package.Name.Value] = package
graph = PackageGraphBuilder.Build(log, allPackages)
for node in graph.DebugNodes():
packageInstance = node.Source
if isinstance(packageInstance, ResolvedPackageInstance):
res.append(packageInstance)
if dump:
DotUtil.ToFile(log, "AllDependencies", graph)
return res
finally:
log.PopIndent()
def __init__(self, log: Log, xmlElement: ET.Element,
defaultName: str) -> None:
super().__init__(log, xmlElement)
self.ShortName = self._ReadAttrib(xmlElement, 'Name', defaultName)
self.Version = self._TryReadAttribAsVersion(
xmlElement, 'Version') # type: Optional[Version]
self.Pipeline = self.__TryGetPipeline(xmlElement)
self.ValidateInstallation = self.__TryGetValidateInstallation(
log, xmlElement)
self.ExternalInstallDirectory = self._TryReadAttrib(
xmlElement, 'ExternalInstallDirectory')
self.FindVersion = self._TryReadAttribAsVersion(
xmlElement, 'FindVersion') # type: Optional[Version]
self.FindTargetName = self._TryReadAttrib(xmlElement, 'FindTargetName')
findResult = self._TryReadBoolAttrib(xmlElement, "Find", None)
self.Find = False if findResult is None else findResult
if self.FindVersion is not None:
if findResult is not None and findResult == False:
self.FindVersion = None
log.LogPrintVerbose(
2,
"Recipe specified Find=False, so discarding the specified FindVersion '{0}'"
.format(self.FindVersion))
else:
self.Find = True
if self.Version is not None and not self.FindVersion.IsCompatible(
self.Version):
raise Exception(
"Recipe '{0}' version {1} is not compatible with the specified FindVersion '{2}'."
.format(self.ShortName, self.Version,
self.FindVersion))
if self.FindTargetName is not None:
if findResult is not None and findResult == False:
self.FindTargetName = None
log.LogPrintVerbose(
2,
"Recipe specified Find=False, so discarding the specified FindTargetName '{0}'"
.format(self.FindTargetName))
else:
self.Find = True
#if self.Pipeline is None and self.ExternalInstallDirectory is None:
# raise Exception("Recipe '{0}' has to have either a pipeline or a ExternalInstallDirectory defined.".format(self.ShortName))
if not self.Pipeline is None and not self.ExternalInstallDirectory is None:
raise Exception(
"Recipe '{0}' can only a pipeline or a ExternalInstallDirectory defined not both."
.format(self.ShortName))
self.FullName = self.__GenerateName(self.ShortName, self.Version)
def Build(
log: Log,
allPackages: List[UnresolvedBasicPackage]) -> ResolvedPackageGraph:
buildOrder = PackageBuildOrder.ResolveBuildOrder(
log, allPackages) # type: List[UnresolvedBasicPackage]
log.LogPrintVerbose(LocalVerbosityLevel.Info,
"Building instance graph")
log.PushIndent()
try:
queue = PackageResolveQueue(buildOrder)
return PackageGraphBuilder.__BuildInstanceGraph(log, queue)
finally:
log.PopIndent()
def __init__(self, log: Log, errorHelpManager: ErrorHelpManager,
recipeFilterManager: RecipeFilterManager,
experimental: Optional[ToolConfigExperimental],
generator: GeneratorPlugin) -> None:
if generator.CMakeConfig is None:
raise Exception("Invalid generator")
recipeBuilderSetup = None
allowDownloads = False
if experimental is not None:
targetLocation = ResolvedPath(
experimental.DefaultThirdPartyInstallDirectory.DynamicName,
experimental.DefaultThirdPartyInstallDirectory.ResolvedPath)
installReadonlySource = experimental.DefaultThirdPartyInstallReadonlyCacheDirectory
readonlyCachePath = None if installReadonlySource is None else installReadonlySource.ResolvedPath # type: Optional[str]
recipeBuilderSetup = RecipeBuilderSetup(targetLocation,
readonlyCachePath)
if IOUtil.TryGetEnvironmentVariable(
experimental.DisableDownloadEnv) is not None:
allowDownloads = False
log.LogPrint(
"Downloads disabled since the environment variable {0} was defined"
.format(experimental.DisableDownloadEnv))
elif experimental.AllowDownloads:
allowDownloads = True
else:
log.LogPrint(
"Downloads disabled since the project has it disabled by default"
)
validVariableDict = {} # type: Dict[str, object]
validVariableDict['PlatformName'] = generator.PlatformName
validVariableDict['IsCMakeBuild'] = generator.IsCMake
generatorInfo = GeneratorInfo(generator.IsCMake,
generator.CMakeConfig.AllowFindPackage,
validVariableDict)
super().__init__(log, errorHelpManager, generator.PlatformName,
generator.PlatformName, generatorInfo,
generator.CMakeConfig, recipeBuilderSetup)
#allowDownload=True, disableDownloadEnv=None
self.Log = log
self.Generator = generator
# for now the generator is also the platform and the existing code use that name so maintain compatibility for now
self.Platform = generator
self.AllowDownloads = allowDownloads
self.RecipeFilterManager = recipeFilterManager
def ShowVersion(log: Log, clangExeInfo: ClangExeInfo) -> None:
try:
log.LogPrint("Listing version")
cmd = clangExeInfo.Command
versionCommand = [cmd, '-version']
result = subprocess.call(versionCommand)
if result != 0:
log.LogPrintWarning(
"The command '{0}' failed with '{1}'.".format(
" ".join(versionCommand), result))
except FileNotFoundError:
log.DoPrintWarning(
"The command '{0}' failed with 'file not found'.".format(
" ".join(versionCommand)))
raise
def GenerateNinjaTidyFile(log: Log, toolConfig: ToolConfig, ninjaOutputFile: str, outputFolder: str, toolVersionOutputFile: str,
clangFormatExeInfo: ClangExeInfo, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, sortedPackageList: List[Package], repairEnabled: bool) -> int:
totalProcessedCount = 0
toolProjectContextsDict = PackagePathUtil.CreateToolProjectContextsDict(toolConfig.ProjectInfo)
with io.StringIO() as ninjaFile:
ninjaFile.writelines("ninja_required_version = 1.8\n")
writer = Writer(ninjaFile, 149)
formatCommand = "{0} $in --style=file".format(clangFormatExeInfo.Command)
if repairEnabled:
formatCommand += " -i"
if len(clangFormatConfiguration.AdditionalUserArguments) > 0:
log.LogPrint("Adding user supplied arguments before '--' {0}".format(clangFormatConfiguration.AdditionalUserArguments))
formatCommand += " {0}".format(" ".join(clangFormatConfiguration.AdditionalUserArguments))
writer.rule(name=PerformClangFormatHelper2.RULE_FORMAT,
command=formatCommand,
restat=True)
for package in sortedPackageList:
PerformClangFormatHelper2.AddPackage(log, toolConfig, toolVersionOutputFile, writer, package, outputFolder, toolProjectContextsDict,
customPackageFileFilter, clangFormatConfiguration)
totalProcessedCount += 1
# finally we write the ninja file
IOUtil.WriteFileIfChanged(ninjaOutputFile, ninjaFile.getvalue())
writer.close()
return totalProcessedCount
def Process(log: Log, toolConfig: ToolConfig,
customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration,
clangExeInfo: ClangExeInfo, packageList: List[Package],
repairEnabled: bool, buildThreads: int) -> int:
totalProcessedCount = 0
if buildThreads <= 1 or len(packageList) <= 1:
for package in packageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
totalProcessedCount += 1
log.LogPrint("- {0}".format(package.Name))
_RunClangFormat(log, toolConfig, clangFormatConfiguration,
clangExeInfo, package, filteredFiles,
repairEnabled)
else:
# Do some threading magic. We can do this because each package touches separate files
packageQueue = queue.Queue(len(packageList)) # type: Any
for package in packageList:
packageQueue.put(package)
# Ensure we dont start threads we dont need
finalBuildThreads = buildThreads if len(
packageList) >= buildThreads else len(packageList)
exceptionList = []
cancellationToken = SimpleCancellationToken()
totalExaminedCount = 0
try:
with ThreadPoolExecutor(
max_workers=finalBuildThreads) as executor:
futures = []
for index in range(0, buildThreads):
taskFuture = executor.submit(
PerformClangFormatHelper.RunInAnotherThread,
packageQueue, cancellationToken, log, toolConfig,
customPackageFileFilter, clangFormatConfiguration,
clangExeInfo, repairEnabled)
futures.append(taskFuture)
# Wait for all futures to finish
for future in futures:
try:
examinedCount, processedCount = future.result()
totalExaminedCount += examinedCount
totalProcessedCount += processedCount
except Exception as ex:
cancellationToken.Cancel()
exceptionList.append(ex)
finally:
cancellationToken.Cancel()
if len(exceptionList) > 0:
raise AggregateException(exceptionList)
if totalExaminedCount != len(packageList):
raise Exception(
"internal error: we did not process the expected amount of packages Expected: {0} processed {1}"
.format(len(packageList), totalExaminedCount))
return totalProcessedCount
def _DoGenerateReport(
self, log: Log, generatorConfig: GeneratorConfig,
packageList: List[Package]
) -> Dict[Package, PackageGeneratorReport]:
log.LogPrintWarning(
"Generator {0} does not support build reports".format(self.Name))
return {}
def Build(log: Log,
configBuildDir: str,
configDisableWrite: bool,
toolConfig: ToolConfig,
packagePath: PackagePath,
featureList: List[str],
outputPath: Optional[str] = None) -> None:
currentPath = packagePath.AbsoluteDirPath
contentBuildDir = ToolSharedValues.CONTENT_BUILD_FOLDER_NAME
contentBuildPath = IOUtil.Join(currentPath, contentBuildDir)
contentOutputPath = GetContentOutputPath(
packagePath) if outputPath is None else outputPath
if not IOUtil.IsDirectory(contentBuildPath):
log.LogPrintVerbose(
1,
"No '{0}' directory present at '{1}' so there is no content to process."
.format(contentBuildDir, currentPath))
return
packageBuildPath = IOUtil.Join(currentPath, configBuildDir)
if not configDisableWrite:
IOUtil.SafeMakeDirs(packageBuildPath)
contentProcessorManager = GetContentProcessorManager(
log, toolConfig, featureList)
Builder(log, configDisableWrite, toolConfig, packageBuildPath,
contentBuildPath, contentOutputPath, contentProcessorManager)
def PrintExecutableSkipReason(log: Log, fullPackageList: List[Package], filteredPackageList: List[Package]) -> None:
for package in fullPackageList:
if package.Type == PackageType.Executable:
if package.ResolvedPlatformNotSupported:
notSupported = LocalUtil.BuildListOfDirectlyNotSupported(package)
notSupportedNames = Util.ExtractNames(notSupported)
log.DoPrint("{0} was marked as not supported on this platform by package: {1}".format(package.Name, notSupportedNames))
def ShowRecipeList(log: Log, topLevelPackage: Package,
requestedFiles: Optional[List[str]]) -> None:
#requestedPackages = PackageUtil.GetPackageListFromFilenames(topLevelPackage, requestedFiles, True)
recipeBuildOrder = topLevelPackage.ResolvedExperimentalRecipeBuildOrder
if recipeBuildOrder is None or len(recipeBuildOrder) <= 0:
log.LogPrint("No recipes found")
return
recipePackages = list(
topLevelPackage.ResolvedExperimentalRecipeBuildOrder)
recipePackages.sort(
key=lambda s: s.ResolvedDirectExperimentalRecipe.FullName.lower()
if s.ResolvedDirectExperimentalRecipe is not None else s.Name.
lower())
strAddIndent = " "
print("Recipes")
for package in recipePackages:
packageRecipe = package.ResolvedDirectExperimentalRecipe
if packageRecipe is not None and packageRecipe.Type != RecipeType.External:
print("{0}{1} (introduced by: {2}), type: {3}".format(
strAddIndent, packageRecipe.FullName, package.Name,
RecipeType.ToString(packageRecipe.Type)))
print("External")
for package in recipePackages:
packageRecipe = package.ResolvedDirectExperimentalRecipe
if packageRecipe is not None and packageRecipe.Type == RecipeType.External:
print("{0}{1} (introduced by: {2})".format(
strAddIndent, packageRecipe.FullName, package.Name))