def ShowRequirementList(log: Log,
basicConfig: BasicConfig,
topLevelPackage: Package,
requestedFiles: Optional[List[str]],
showFeaturesOnly: bool = False) -> None:
message = "Requirements" if not showFeaturesOnly else "Features"
filterName = None if not showFeaturesOnly else PackageRequirementTypeString.Feature
# As the packages in requestedFiles might have been filtered at this point (and any issues already caught), we just ignore not found
requestedPackages = PackageUtil.GetPackageListFromFilenames(
topLevelPackage, requestedFiles, True)
requirements = RequirementFilter.GetRequirementList(
topLevelPackage, requestedPackages, filterName)
if len(requirements) <= 0:
log.DoPrint("{0}: None".format(message))
return
log.DoPrint("{0}:".format(message))
rootNode = RequirementTree(requirements).RootNode
strAddIndent = " "
# We only show the type group info when there is more than one
#showTypeGroup = len(rootNode.Children) > 1
baseIndent = "" #strAddIndent if len(rootNode.Children) > 1 else ""
sortedFeatures = list(rootNode.Children)
sortedFeatures.sort(
key=lambda s: None if s.Content is None else s.Content.Id)
for sortedFeature in sortedFeatures:
__PrintRequirementsNode(log, sortedFeature, baseIndent, strAddIndent)
def ShowExtensionList(log: Log, topLevelPackage: Package,
requestedFiles: Optional[List[str]]) -> None:
# As the packages in requestedFiles might have been filtered at this point (and any issues already caught), we just ignore not found
requestedPackages = PackageUtil.GetPackageListFromFilenames(
topLevelPackage, requestedFiles, True)
requirements = RequirementFilter.GetRequirementList(
topLevelPackage, requestedPackages,
PackageRequirementTypeString.Extension)
if len(requirements) <= 0:
log.DoPrint("Extensions: None")
return
log.DoPrint("Extensions:")
# Pretty print useful information in name sorted order
requirements.sort(key=lambda s: s.Id)
currentIndent = " "
for requirement in requirements:
strFormat = "{0}- '{1}'"
if len(requirement.Version) > 0:
strFormat += " V{2}"
if len(requirement.Extends) > 0:
strFormat += " extends '{3}'"
strFormat += " (introduced by package: {4})"
log.DoPrint(
strFormat.format(currentIndent, requirement.Name,
requirement.Version, requirement.Extends,
", ".join(requirement.IntroducedByPackages)))
def __ValidateIncludeGuard(log: Log, sourceFile: SourceFile, shortFile: str, repairEnabled: bool) -> bool:
if len(sourceFile.LinesModded) < 2:
return False
currentLine0 = sourceFile.LinesModded[0].strip()
currentLine1 = sourceFile.LinesModded[1].strip()
guard = __GenerateIncludeGuardName(sourceFile.Package, shortFile)
line0Valid = "#ifndef %s" % (guard)
line1Valid = "#define %s" % (guard)
if currentLine0 == line0Valid and currentLine1 == line1Valid:
return True
# check that the file starts with the guard statements
prefix0 = "#ifndef "
prefix1 = "#define "
if not currentLine0.startswith(prefix0):
log.DoPrint("Line 0 does not start with '%s' in '%s'" % (prefix0, os.path.normpath(sourceFile.FileName)))
if repairEnabled:
log.LogPrint("Because of this repair was not attempted.")
return False
if not currentLine1.startswith(prefix1):
log.DoPrint("Line 1 does not start with '%s' in '%s'" % (prefix1, os.path.normpath(sourceFile.FileName)))
if repairEnabled:
log.LogPrint("Because of this repair was not attempted.")
return False
# validate that the #ifndef and define works on the same string
userDef0 = currentLine0[len(prefix0):].strip()
userDef1 = currentLine1[len(prefix1):].strip()
if userDef0 != userDef1:
log.DoPrint("The include guards do not appear to match '%s' != '%s' in '%s'" % (userDef0, userDef1, os.path.normpath(sourceFile.FileName)))
log.LogPrint("- Line 0 '%s'" % (userDef0))
log.LogPrint("- Line 1 '%s'" % (userDef1))
if repairEnabled:
log.LogPrint("Because of this repair was not attempted.")
return False
# So we should be sure that the guard is just the incorrect name, so list it
log.DoPrint("Wrong include guard: '%s' expected '%s'" % (os.path.normpath(sourceFile.FileName), guard))
if currentLine0 != line0Valid:
log.LogPrint("- Expected '%s'" % (line0Valid))
log.LogPrint("- Was '%s'" % (currentLine0))
elif currentLine1 != line1Valid:
log.LogPrint("- Expected '%s'" % (line1Valid))
log.LogPrint("- Was '%s'" % (currentLine1))
if not repairEnabled:
return False
log.DoPrint("Include guard corrected")
# We are allowed to repair the content, so lets do that
sourceFile.LinesModded[0] = line0Valid
sourceFile.LinesModded[1] = line1Valid
return False
def Scan(log: Log, scanPackageList: List[Package],
customPackageFileFilter: Optional[CustomPackageFileFilter],
repairEnabled: bool, thirdpartyExceptionDir: Optional[str],
checkType: CheckType, disableWrite: bool) -> None:
"""
Run through all source files that are part of the packages and check for common errors
:param scanPackageList: the packages that will be scanned.
"""
log.LogPrint("Running source scan")
extensionList = __g_includeExtensionList + __g_sourceExtensionList
# Filter the package list so it only contains things we can process
finalPackageList = [
package for package in scanPackageList
if PerformClangUtil.CanProcessPackage(package)
]
totalErrorCount = 0
for package in finalPackageList:
filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage(
log, package, extensionList)
if customPackageFileFilter is None or filteredFiles is not None:
totalErrorCount += __ScanFiles(log, package, filteredFiles,
repairEnabled,
thirdpartyExceptionDir, checkType,
disableWrite)
if totalErrorCount > 0 and not repairEnabled:
log.DoPrint(
"BEWARE: If you have made a backup of your files you can try to auto correct the errors with '--Repair' but do so at your own peril"
)
def PrintExecutableSkipReason(log: Log, fullPackageList: List[Package], filteredPackageList: List[Package]) -> None:
for package in fullPackageList:
if package.Type == PackageType.Executable:
if package.ResolvedPlatformNotSupported:
notSupported = LocalUtil.BuildListOfDirectlyNotSupported(package)
notSupportedNames = Util.ExtractNames(notSupported)
log.DoPrint("{0} was marked as not supported on this platform by package: {1}".format(package.Name, notSupportedNames))
def ShowBuildVariantList(log: Log, generator: GeneratorPluginBase2) -> None:
# This is kind of a hack to list this here (its also not a real variant inside our model)
generatorVariants = [
variant for variant in generator.GetVariants()
if variant.Type == BuildVariantType.Static
]
if len(generatorVariants) <= 0:
log.DoPrint("Build variants: None")
return
log.DoPrint("Build variants:")
generatorVariants.sort(key=lambda s: s.Name.lower())
for variantInfo in generatorVariants:
log.DoPrint(
" {0}={1} (Introduced by native build system generator)".format(
variantInfo.Name, variantInfo.Description))
def __Repair(log: Log, sourceFile: SourceFile, asciiRepair: bool, disableWrite: bool) -> None:
strContent = "\n".join(sourceFile.LinesModded)
if asciiRepair:
strContent = __Decoded(strContent)
if strContent != sourceFile.Content:
log.DoPrint("Repaired '%s'" % (os.path.normpath(sourceFile.FileName)))
if not disableWrite:
IOUtil.WriteFile(sourceFile.FileName, strContent)
def __BuildAndRunPackages(self, log: Log, buildConfig: BuildConfigRecord,
buildContext: LocalBuildContext,
resolvedBuildOrderBuildable: List[Package],
originalBuildArgs: List[str],
builderCanBuildContent: bool,
runValidationChecks: bool, allowBuild: bool,
isDryRun: bool,
generatorConfig: GeneratorConfig) -> None:
for package in resolvedBuildOrderBuildable:
if allowBuild:
log.LogPrint("Building package: {0}".format(package.Name))
log.LogPrint("Package location: {0}".format(
package.AbsolutePath))
if not isDryRun:
buildEnv = self.__CreateBuildEnv(log, buildConfig,
buildContext, package,
builderCanBuildContent)
buildConfig.BuildArgs = list(originalBuildArgs)
if log.Verbosity > 4:
log.DoPrint("Package build arguments1: {0}".format(
buildConfig.BuildArgs))
log.DoPrint("General build arguments2: {0}".format(
originalBuildArgs))
if runValidationChecks:
self.__RunValidationChecks(buildConfig, package)
# Do the actual package build and then run the package if so desired
if allowBuild:
self.__BuildPackage(buildContext, buildConfig, buildEnv,
package)
# Run commands
strRunCommands = buildConfig.RunCommand
runCommands = None # type: Optional[List[str]]
if strRunCommands is not None:
userRunCommands = shlex.split(strRunCommands)
runCommands = self.__TryGenerateRunCommandForExecutable(
buildContext, package, buildConfig, userRunCommands,
generatorConfig)
self.__RunPackage(buildContext, package, buildEnv,
runCommands)
def __CheckTabs(log: Log, sourceFile: SourceFile, repairEnabled: bool, thirdpartyExceptionDir: Optional[str]) -> bool:
if __g_thirdParty in sourceFile.FileName or (thirdpartyExceptionDir is not None and sourceFile.BasePath is not None and sourceFile.BasePath.startswith(thirdpartyExceptionDir)):
return True
tabCount = 0
for line in sourceFile.LinesModded:
tabCount += line.count('\t')
if tabCount == 0:
return True
log.DoPrint("Found %s tab characters in '%s'" % (tabCount, os.path.normpath(sourceFile.FileName)))
return False
def __FiltersPackagesBySupported(log: Log, packages: Union[List[Package], List[AppInfoPackage]]) -> List[T]:
""" Remove packages that are marked as not supported by the platform
"""
packageList = []
for package in packages:
if not package.ResolvedPlatformNotSupported:
packageList.append(package)
elif not package.Type == PackageType.TopLevel and log.IsVerbose and isinstance(package, Package):
notSupported = LocalUtil.BuildListOfDirectlyNotSupported(package)
notSupportedNames = Util.ExtractNames(notSupported)
log.DoPrint("Skipping {0} since its marked as not supported on this platform by package: {1}".format(package.Name, notSupportedNames))
return cast(List[T], packageList)
def __CheckASCII(log: Log, sourceFile: SourceFile, repairEnabled: bool) -> bool:
errorCount = 0
for index, line in enumerate(sourceFile.LinesOriginal):
if not __IsAscii(line):
posX = __IndexOfNonAscii(line, 0)
while posX >= 0:
ch = hex(ord(line[posX])) if index >= 0 else '-failed-'
log.DoPrint("Non ASCII character '{0}' encountered at X:{1}, Y:{2} in '{3}'".format(ch, posX+1, index+1, os.path.normpath(sourceFile.FileName)))
errorCount = errorCount + 1
#if repairEnabled:
# line[posX] = ' ' # disabled because its too dangerous
posX = __IndexOfNonAscii(line, posX+1)
return errorCount == 0
def RunInAnotherThread(packageQueue: Any,
cancellationToken: SimpleCancellationToken,
mainLog: Log, toolConfig: ToolConfig, customPackageFileFilter: Optional[CustomPackageFileFilter],
clangFormatConfiguration: ClangFormatConfiguration, clangExeInfo: ClangExeInfo, repairEnabled: bool) -> Tuple[int,int]:
threadId = threading.get_ident()
mainLog.LogPrintVerbose(4, "Starting thread {0}".format(threadId))
examinedCount = 0
processedCount = 0
keepWorking = True
package = None # type: Optional[Package]
try:
while keepWorking and not cancellationToken.IsCancelled():
try:
# Since the queue is preloaded this is ok
package = packageQueue.get_nowait()
except:
package = None
if package is None:
keepWorking = False
else:
if mainLog.Verbosity >= 4:
mainLog.LogPrint("- clang-format on package '{0}' on thread {1}".format(package.Name, threadId))
else:
mainLog.LogPrint("- clang-format on package '{0}'".format(package.Name))
captureLog = CaptureLog(mainLog.Title, mainLog.Verbosity)
try:
filteredFiles = None
if customPackageFileFilter is not None:
filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(captureLog, package, clangFormatConfiguration.FileExtensions)
if customPackageFileFilter is None or filteredFiles is not None:
processedCount += 1
_RunClangFormat(captureLog, toolConfig, clangFormatConfiguration, clangExeInfo, package, filteredFiles, repairEnabled)
examinedCount += 1
finally:
try:
if len(captureLog.Captured) > 0:
capturedLog = "Package: '{0}' result:\n{1}".format(package.Name, "\n".join(captureLog.Captured))
mainLog.DoPrint(capturedLog)
except:
pass
except Exception as ex:
cancellationToken.Cancel()
mainLog.DoPrintError("Cancelling tasks due to exception: {0}")
raise
finally:
mainLog.LogPrintVerbose(4, "Ending thread {0}".format(threadId))
return (examinedCount, processedCount)
def RunNow(log: Log, buildCommand: List[str], currentWorkingDirectory: str,
logOutput: bool) -> int:
if not logOutput:
return subprocess.call(buildCommand, cwd=currentWorkingDirectory)
try:
with subprocess.Popen(buildCommand,
cwd=currentWorkingDirectory,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=True) as proc:
output = proc.stdout.read().strip()
proc.stdout.close()
return proc.wait()
finally:
if output is not None and len(output) > 0:
log.DoPrint(output)
def __PrintRequirementsNode(log: Log, node: RequirementTreeNode,
currentIndent: str, strAddIndent: str) -> None:
if node.Content is None:
raise Exception("Invalid node")
strFormat = "{0}-"
if node.Content.Type == PackageRequirementTypeString.Feature:
strFormat += " '{2}'"
else:
strFormat += " {1}: '{2}'"
if len(node.Content.Version) > 0:
strFormat += " V{3}"
#if len(node.Content.Extends) > 0:
# strFormat += " extends '{4}'"
strFormat += " (introduced by package: {5})"
log.DoPrint(
strFormat.format(currentIndent, node.Content.Type, node.Content.Name,
node.Content.Version, node.Content.Extends,
", ".join(node.Content.IntroducedByPackages)))
# Group by type
dictGroup = {} # type: Dict[str, List[RequirementTreeNode]]
for childNode in node.Children:
if childNode.Content is None:
raise Exception("Invalid node")
if childNode.Content.Type in dictGroup:
dictGroup[childNode.Content.Type].append(childNode)
else:
dictGroup[childNode.Content.Type] = [childNode]
# Pretty print useful information in name sorted order
sortedGroupIds = list(dictGroup.keys())
# sort by type name, but make sure that new 'features' go last
sortedGroupIds.sort(key=lambda s: s if s != PackageRequirementTypeString.
Feature else '{{feature}}')
for groupId in sortedGroupIds:
groupedRequirements = dictGroup[groupId]
groupedRequirements.sort(
key=lambda s: None if s.Content is None else s.Content.Id)
for childNode in groupedRequirements:
__PrintRequirementsNode(log, childNode,
currentIndent + strAddIndent, strAddIndent)
def ShowVariantList(log: Log, topLevelPackage: Package,
requestedFiles: Optional[List[str]],
generator: GeneratorPluginBase2) -> None:
variantDict = BuildVariantUtil.BuildCompleteVariantDict(topLevelPackage)
# This is kind of a hack to list this here (its also not a real variant inside our model)
generatorVariants = generator.GetVariants()
if len(variantDict) <= 0 and len(generatorVariants) <= 0:
log.DoPrint("Variants: None")
return
# Pretty print useful information
log.DoPrint("Variants:")
generatorVariants.sort(key=lambda s: s.Name.lower())
for variantInfo in generatorVariants:
if variantInfo.Type == BuildVariantType.Static:
log.DoPrint(
" {0}={1} (Introduced by native build system generator)".
format(variantInfo.Name, variantInfo.Description))
else:
log.DoPrint(" {0}={1} (Introduced by native build system)".format(
variantInfo.Name, variantInfo.Description))
variantNames = list(variantDict.keys())
variantNames.sort()
for variantName in variantNames:
variant = variantDict[variantName]
optionNames = list(variant.OptionDict.keys())
optionNames.sort()
if variant.Type == VariantType.Virtual:
log.DoPrint(
" {0}={1} *Virtual* (Introduced by package: {2})".format(
variant.PurifiedName, ', '.join(optionNames),
variant.IntroducedByPackageName))
else:
log.DoPrint((" {0}={1} (Introduced by package: {2})".format(
variant.PurifiedName, ', '.join(optionNames),
variant.IntroducedByPackageName)))
def __DoBuildPackagesInOrder(log: Log, configSDKPath: str, configIsDryRun: bool, generatorContext: GeneratorContext,
resolvedBuildOrder: List[Package], builderSettings: BuilderSettings,
packageRecipeResultManager: PackageRecipeResultManager) -> None:
if not generatorContext.RecipePathBuilder.IsEnabled:
log.LogPrintVerbose(3, "External building has been disabled in the Project.gen file")
return
if generatorContext.RecipePathBuilder.TargetLocation is None:
raise Exception("Invalid path builder")
# Claim the 'package' install directory to prevent multiple builds from using the same
# as it would give concurrency issues
BuildAreaInfoFileUtil.ProcessInstallDirClaim(log, generatorContext.RecipePathBuilder.TargetLocation.ResolvedPath, configSDKPath,
builderSettings.ForceClaimInstallArea, __g_installAreaInformationFilename)
if resolvedBuildOrder is None:
log.LogPrintVerbose(2, "No recipes to build")
return
# Filter all packages that don't have a experimental recipe
resolvedBuildOrder = [entry for entry in resolvedBuildOrder if not entry.ResolvedDirectExperimentalRecipe is None]
if len(resolvedBuildOrder) == 0:
log.LogPrintVerbose(2, "No recipes to build")
return
recipePackageStateCache = RecipePackageStateCache(log)
validationEngine = ValidationEngine(log, generatorContext.VariableProcessor, packageRecipeResultManager, generatorContext.ErrorHelpManager)
missingPackagesInBuildOrder = __FindMissingInstallations(log, validationEngine, resolvedBuildOrder, recipePackageStateCache,
generatorContext.CMakeConfig)
builder = PipelineCommandBuilder(generatorContext, builderSettings.CheckBuildCommands, builderSettings.BuildThreads)
recipeRecords = __CreatePipelines(log, builder, missingPackagesInBuildOrder)
for recipeRecord in recipeRecords:
log.LogPrint("Package location: {0}".format(recipeRecord.SourcePackage.AbsolutePath))
try:
log.PushIndent()
if not recipeRecord.SourcePackage.ResolvedPlatformDirectSupported:
raise Exception("The package '{0}' is not supported on this platform".format(recipeRecord.SourcePackage.Name))
if not recipeRecord.Pipeline is None:
log.DoPrint("Building package: {0}".format(recipeRecord.SourcePackage.Name))
if builderSettings.PreDeleteBuild:
# We clear the build path to prepare for a new build
IOUtil.SafeRemoveDirectoryTree(recipeRecord.Pipeline.BuildPath)
for command in recipeRecord.Pipeline.CommandList:
if not configIsDryRun:
command.Execute()
# We finished building, so lets save some information about what we did
BuildInfoFileUtil.SaveBuildInformation(log, recipeRecord, recipePackageStateCache, generatorContext.CMakeConfig,
__g_BuildPackageInformationFilename)
if builderSettings.PostDeleteBuild:
# We clear the build path if a build is successfull
IOUtil.SafeRemoveDirectoryTree(recipeRecord.Pipeline.BuildPath, True)
else:
# Since we are trying to build this it means that the installation validation failed earlier and
# we apparently have no pipelines that could remedy it, so force the install validation to occur so
# we fail early as 'dependent' pipes might fail to build due to this
# generatorContext.RecipeFilterManager
if generatorContext.RecipeFilterManager.AllRecipesEnabled or recipeRecord.SourcePackage.Name in generatorContext.RecipeFilterManager.ContentDict:
log.DoPrintWarning("Missing installation of package '{0}' and no recipe for solving it is available".format(recipeRecord.SourcePackage.Name))
else:
log.LogPrintVerbose(4, "Package '{0}' recipe not enabled".format(recipeRecord.SourcePackage.Name))
validationEngine.Process(recipeRecord.SourcePackage)
finally:
log.PopIndent()
validationEngine.Process(recipeRecord.SourcePackage)
packageCount = len(recipeRecords)
if packageCount > 0:
log.LogPrint("Build {0} packages".format(packageCount))
else:
log.LogPrintVerbose(2, "No recipe was build!")
