def ShowRequirementList(log: Log, basicConfig: BasicConfig, topLevelPackage: Package, requestedFiles: Optional[List[str]], showFeaturesOnly: bool = False) -> None: message = "Requirements" if not showFeaturesOnly else "Features" filterName = None if not showFeaturesOnly else PackageRequirementTypeString.Feature # As the packages in requestedFiles might have been filtered at this point (and any issues already caught), we just ignore not found requestedPackages = PackageUtil.GetPackageListFromFilenames( topLevelPackage, requestedFiles, True) requirements = RequirementFilter.GetRequirementList( topLevelPackage, requestedPackages, filterName) if len(requirements) <= 0: log.DoPrint("{0}: None".format(message)) return log.DoPrint("{0}:".format(message)) rootNode = RequirementTree(requirements).RootNode strAddIndent = " " # We only show the type group info when there is more than one #showTypeGroup = len(rootNode.Children) > 1 baseIndent = "" #strAddIndent if len(rootNode.Children) > 1 else "" sortedFeatures = list(rootNode.Children) sortedFeatures.sort( key=lambda s: None if s.Content is None else s.Content.Id) for sortedFeature in sortedFeatures: __PrintRequirementsNode(log, sortedFeature, baseIndent, strAddIndent)
def ShowExtensionList(log: Log, topLevelPackage: Package, requestedFiles: Optional[List[str]]) -> None: # As the packages in requestedFiles might have been filtered at this point (and any issues already caught), we just ignore not found requestedPackages = PackageUtil.GetPackageListFromFilenames( topLevelPackage, requestedFiles, True) requirements = RequirementFilter.GetRequirementList( topLevelPackage, requestedPackages, PackageRequirementTypeString.Extension) if len(requirements) <= 0: log.DoPrint("Extensions: None") return log.DoPrint("Extensions:") # Pretty print useful information in name sorted order requirements.sort(key=lambda s: s.Id) currentIndent = " " for requirement in requirements: strFormat = "{0}- '{1}'" if len(requirement.Version) > 0: strFormat += " V{2}" if len(requirement.Extends) > 0: strFormat += " extends '{3}'" strFormat += " (introduced by package: {4})" log.DoPrint( strFormat.format(currentIndent, requirement.Name, requirement.Version, requirement.Extends, ", ".join(requirement.IntroducedByPackages)))
def __ValidateIncludeGuard(log: Log, sourceFile: SourceFile, shortFile: str, repairEnabled: bool) -> bool: if len(sourceFile.LinesModded) < 2: return False currentLine0 = sourceFile.LinesModded[0].strip() currentLine1 = sourceFile.LinesModded[1].strip() guard = __GenerateIncludeGuardName(sourceFile.Package, shortFile) line0Valid = "#ifndef %s" % (guard) line1Valid = "#define %s" % (guard) if currentLine0 == line0Valid and currentLine1 == line1Valid: return True # check that the file starts with the guard statements prefix0 = "#ifndef " prefix1 = "#define " if not currentLine0.startswith(prefix0): log.DoPrint("Line 0 does not start with '%s' in '%s'" % (prefix0, os.path.normpath(sourceFile.FileName))) if repairEnabled: log.LogPrint("Because of this repair was not attempted.") return False if not currentLine1.startswith(prefix1): log.DoPrint("Line 1 does not start with '%s' in '%s'" % (prefix1, os.path.normpath(sourceFile.FileName))) if repairEnabled: log.LogPrint("Because of this repair was not attempted.") return False # validate that the #ifndef and define works on the same string userDef0 = currentLine0[len(prefix0):].strip() userDef1 = currentLine1[len(prefix1):].strip() if userDef0 != userDef1: log.DoPrint("The include guards do not appear to match '%s' != '%s' in '%s'" % (userDef0, userDef1, os.path.normpath(sourceFile.FileName))) log.LogPrint("- Line 0 '%s'" % (userDef0)) log.LogPrint("- Line 1 '%s'" % (userDef1)) if repairEnabled: log.LogPrint("Because of this repair was not attempted.") return False # So we should be sure that the guard is just the incorrect name, so list it log.DoPrint("Wrong include guard: '%s' expected '%s'" % (os.path.normpath(sourceFile.FileName), guard)) if currentLine0 != line0Valid: log.LogPrint("- Expected '%s'" % (line0Valid)) log.LogPrint("- Was '%s'" % (currentLine0)) elif currentLine1 != line1Valid: log.LogPrint("- Expected '%s'" % (line1Valid)) log.LogPrint("- Was '%s'" % (currentLine1)) if not repairEnabled: return False log.DoPrint("Include guard corrected") # We are allowed to repair the content, so lets do that sourceFile.LinesModded[0] = line0Valid sourceFile.LinesModded[1] = line1Valid return False
def Scan(log: Log, scanPackageList: List[Package], customPackageFileFilter: Optional[CustomPackageFileFilter], repairEnabled: bool, thirdpartyExceptionDir: Optional[str], checkType: CheckType, disableWrite: bool) -> None: """ Run through all source files that are part of the packages and check for common errors :param scanPackageList: the packages that will be scanned. """ log.LogPrint("Running source scan") extensionList = __g_includeExtensionList + __g_sourceExtensionList # Filter the package list so it only contains things we can process finalPackageList = [ package for package in scanPackageList if PerformClangUtil.CanProcessPackage(package) ] totalErrorCount = 0 for package in finalPackageList: filteredFiles = None if customPackageFileFilter is None else customPackageFileFilter.TryLocateFilePatternInPackage( log, package, extensionList) if customPackageFileFilter is None or filteredFiles is not None: totalErrorCount += __ScanFiles(log, package, filteredFiles, repairEnabled, thirdpartyExceptionDir, checkType, disableWrite) if totalErrorCount > 0 and not repairEnabled: log.DoPrint( "BEWARE: If you have made a backup of your files you can try to auto correct the errors with '--Repair' but do so at your own peril" )
def PrintExecutableSkipReason(log: Log, fullPackageList: List[Package], filteredPackageList: List[Package]) -> None: for package in fullPackageList: if package.Type == PackageType.Executable: if package.ResolvedPlatformNotSupported: notSupported = LocalUtil.BuildListOfDirectlyNotSupported(package) notSupportedNames = Util.ExtractNames(notSupported) log.DoPrint("{0} was marked as not supported on this platform by package: {1}".format(package.Name, notSupportedNames))
def ShowBuildVariantList(log: Log, generator: GeneratorPluginBase2) -> None: # This is kind of a hack to list this here (its also not a real variant inside our model) generatorVariants = [ variant for variant in generator.GetVariants() if variant.Type == BuildVariantType.Static ] if len(generatorVariants) <= 0: log.DoPrint("Build variants: None") return log.DoPrint("Build variants:") generatorVariants.sort(key=lambda s: s.Name.lower()) for variantInfo in generatorVariants: log.DoPrint( " {0}={1} (Introduced by native build system generator)".format( variantInfo.Name, variantInfo.Description))
def __Repair(log: Log, sourceFile: SourceFile, asciiRepair: bool, disableWrite: bool) -> None: strContent = "\n".join(sourceFile.LinesModded) if asciiRepair: strContent = __Decoded(strContent) if strContent != sourceFile.Content: log.DoPrint("Repaired '%s'" % (os.path.normpath(sourceFile.FileName))) if not disableWrite: IOUtil.WriteFile(sourceFile.FileName, strContent)
def __BuildAndRunPackages(self, log: Log, buildConfig: BuildConfigRecord, buildContext: LocalBuildContext, resolvedBuildOrderBuildable: List[Package], originalBuildArgs: List[str], builderCanBuildContent: bool, runValidationChecks: bool, allowBuild: bool, isDryRun: bool, generatorConfig: GeneratorConfig) -> None: for package in resolvedBuildOrderBuildable: if allowBuild: log.LogPrint("Building package: {0}".format(package.Name)) log.LogPrint("Package location: {0}".format( package.AbsolutePath)) if not isDryRun: buildEnv = self.__CreateBuildEnv(log, buildConfig, buildContext, package, builderCanBuildContent) buildConfig.BuildArgs = list(originalBuildArgs) if log.Verbosity > 4: log.DoPrint("Package build arguments1: {0}".format( buildConfig.BuildArgs)) log.DoPrint("General build arguments2: {0}".format( originalBuildArgs)) if runValidationChecks: self.__RunValidationChecks(buildConfig, package) # Do the actual package build and then run the package if so desired if allowBuild: self.__BuildPackage(buildContext, buildConfig, buildEnv, package) # Run commands strRunCommands = buildConfig.RunCommand runCommands = None # type: Optional[List[str]] if strRunCommands is not None: userRunCommands = shlex.split(strRunCommands) runCommands = self.__TryGenerateRunCommandForExecutable( buildContext, package, buildConfig, userRunCommands, generatorConfig) self.__RunPackage(buildContext, package, buildEnv, runCommands)
def __CheckTabs(log: Log, sourceFile: SourceFile, repairEnabled: bool, thirdpartyExceptionDir: Optional[str]) -> bool: if __g_thirdParty in sourceFile.FileName or (thirdpartyExceptionDir is not None and sourceFile.BasePath is not None and sourceFile.BasePath.startswith(thirdpartyExceptionDir)): return True tabCount = 0 for line in sourceFile.LinesModded: tabCount += line.count('\t') if tabCount == 0: return True log.DoPrint("Found %s tab characters in '%s'" % (tabCount, os.path.normpath(sourceFile.FileName))) return False
def __FiltersPackagesBySupported(log: Log, packages: Union[List[Package], List[AppInfoPackage]]) -> List[T]: """ Remove packages that are marked as not supported by the platform """ packageList = [] for package in packages: if not package.ResolvedPlatformNotSupported: packageList.append(package) elif not package.Type == PackageType.TopLevel and log.IsVerbose and isinstance(package, Package): notSupported = LocalUtil.BuildListOfDirectlyNotSupported(package) notSupportedNames = Util.ExtractNames(notSupported) log.DoPrint("Skipping {0} since its marked as not supported on this platform by package: {1}".format(package.Name, notSupportedNames)) return cast(List[T], packageList)
def __CheckASCII(log: Log, sourceFile: SourceFile, repairEnabled: bool) -> bool: errorCount = 0 for index, line in enumerate(sourceFile.LinesOriginal): if not __IsAscii(line): posX = __IndexOfNonAscii(line, 0) while posX >= 0: ch = hex(ord(line[posX])) if index >= 0 else '-failed-' log.DoPrint("Non ASCII character '{0}' encountered at X:{1}, Y:{2} in '{3}'".format(ch, posX+1, index+1, os.path.normpath(sourceFile.FileName))) errorCount = errorCount + 1 #if repairEnabled: # line[posX] = ' ' # disabled because its too dangerous posX = __IndexOfNonAscii(line, posX+1) return errorCount == 0
def RunInAnotherThread(packageQueue: Any, cancellationToken: SimpleCancellationToken, mainLog: Log, toolConfig: ToolConfig, customPackageFileFilter: Optional[CustomPackageFileFilter], clangFormatConfiguration: ClangFormatConfiguration, clangExeInfo: ClangExeInfo, repairEnabled: bool) -> Tuple[int,int]: threadId = threading.get_ident() mainLog.LogPrintVerbose(4, "Starting thread {0}".format(threadId)) examinedCount = 0 processedCount = 0 keepWorking = True package = None # type: Optional[Package] try: while keepWorking and not cancellationToken.IsCancelled(): try: # Since the queue is preloaded this is ok package = packageQueue.get_nowait() except: package = None if package is None: keepWorking = False else: if mainLog.Verbosity >= 4: mainLog.LogPrint("- clang-format on package '{0}' on thread {1}".format(package.Name, threadId)) else: mainLog.LogPrint("- clang-format on package '{0}'".format(package.Name)) captureLog = CaptureLog(mainLog.Title, mainLog.Verbosity) try: filteredFiles = None if customPackageFileFilter is not None: filteredFiles = customPackageFileFilter.TryLocateFilePatternInPackage(captureLog, package, clangFormatConfiguration.FileExtensions) if customPackageFileFilter is None or filteredFiles is not None: processedCount += 1 _RunClangFormat(captureLog, toolConfig, clangFormatConfiguration, clangExeInfo, package, filteredFiles, repairEnabled) examinedCount += 1 finally: try: if len(captureLog.Captured) > 0: capturedLog = "Package: '{0}' result:\n{1}".format(package.Name, "\n".join(captureLog.Captured)) mainLog.DoPrint(capturedLog) except: pass except Exception as ex: cancellationToken.Cancel() mainLog.DoPrintError("Cancelling tasks due to exception: {0}") raise finally: mainLog.LogPrintVerbose(4, "Ending thread {0}".format(threadId)) return (examinedCount, processedCount)
def RunNow(log: Log, buildCommand: List[str], currentWorkingDirectory: str, logOutput: bool) -> int: if not logOutput: return subprocess.call(buildCommand, cwd=currentWorkingDirectory) try: with subprocess.Popen(buildCommand, cwd=currentWorkingDirectory, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True) as proc: output = proc.stdout.read().strip() proc.stdout.close() return proc.wait() finally: if output is not None and len(output) > 0: log.DoPrint(output)
def __PrintRequirementsNode(log: Log, node: RequirementTreeNode, currentIndent: str, strAddIndent: str) -> None: if node.Content is None: raise Exception("Invalid node") strFormat = "{0}-" if node.Content.Type == PackageRequirementTypeString.Feature: strFormat += " '{2}'" else: strFormat += " {1}: '{2}'" if len(node.Content.Version) > 0: strFormat += " V{3}" #if len(node.Content.Extends) > 0: # strFormat += " extends '{4}'" strFormat += " (introduced by package: {5})" log.DoPrint( strFormat.format(currentIndent, node.Content.Type, node.Content.Name, node.Content.Version, node.Content.Extends, ", ".join(node.Content.IntroducedByPackages))) # Group by type dictGroup = {} # type: Dict[str, List[RequirementTreeNode]] for childNode in node.Children: if childNode.Content is None: raise Exception("Invalid node") if childNode.Content.Type in dictGroup: dictGroup[childNode.Content.Type].append(childNode) else: dictGroup[childNode.Content.Type] = [childNode] # Pretty print useful information in name sorted order sortedGroupIds = list(dictGroup.keys()) # sort by type name, but make sure that new 'features' go last sortedGroupIds.sort(key=lambda s: s if s != PackageRequirementTypeString. Feature else '{{feature}}') for groupId in sortedGroupIds: groupedRequirements = dictGroup[groupId] groupedRequirements.sort( key=lambda s: None if s.Content is None else s.Content.Id) for childNode in groupedRequirements: __PrintRequirementsNode(log, childNode, currentIndent + strAddIndent, strAddIndent)
def ShowVariantList(log: Log, topLevelPackage: Package, requestedFiles: Optional[List[str]], generator: GeneratorPluginBase2) -> None: variantDict = BuildVariantUtil.BuildCompleteVariantDict(topLevelPackage) # This is kind of a hack to list this here (its also not a real variant inside our model) generatorVariants = generator.GetVariants() if len(variantDict) <= 0 and len(generatorVariants) <= 0: log.DoPrint("Variants: None") return # Pretty print useful information log.DoPrint("Variants:") generatorVariants.sort(key=lambda s: s.Name.lower()) for variantInfo in generatorVariants: if variantInfo.Type == BuildVariantType.Static: log.DoPrint( " {0}={1} (Introduced by native build system generator)". format(variantInfo.Name, variantInfo.Description)) else: log.DoPrint(" {0}={1} (Introduced by native build system)".format( variantInfo.Name, variantInfo.Description)) variantNames = list(variantDict.keys()) variantNames.sort() for variantName in variantNames: variant = variantDict[variantName] optionNames = list(variant.OptionDict.keys()) optionNames.sort() if variant.Type == VariantType.Virtual: log.DoPrint( " {0}={1} *Virtual* (Introduced by package: {2})".format( variant.PurifiedName, ', '.join(optionNames), variant.IntroducedByPackageName)) else: log.DoPrint((" {0}={1} (Introduced by package: {2})".format( variant.PurifiedName, ', '.join(optionNames), variant.IntroducedByPackageName)))
def __DoBuildPackagesInOrder(log: Log, configSDKPath: str, configIsDryRun: bool, generatorContext: GeneratorContext, resolvedBuildOrder: List[Package], builderSettings: BuilderSettings, packageRecipeResultManager: PackageRecipeResultManager) -> None: if not generatorContext.RecipePathBuilder.IsEnabled: log.LogPrintVerbose(3, "External building has been disabled in the Project.gen file") return if generatorContext.RecipePathBuilder.TargetLocation is None: raise Exception("Invalid path builder") # Claim the 'package' install directory to prevent multiple builds from using the same # as it would give concurrency issues BuildAreaInfoFileUtil.ProcessInstallDirClaim(log, generatorContext.RecipePathBuilder.TargetLocation.ResolvedPath, configSDKPath, builderSettings.ForceClaimInstallArea, __g_installAreaInformationFilename) if resolvedBuildOrder is None: log.LogPrintVerbose(2, "No recipes to build") return # Filter all packages that don't have a experimental recipe resolvedBuildOrder = [entry for entry in resolvedBuildOrder if not entry.ResolvedDirectExperimentalRecipe is None] if len(resolvedBuildOrder) == 0: log.LogPrintVerbose(2, "No recipes to build") return recipePackageStateCache = RecipePackageStateCache(log) validationEngine = ValidationEngine(log, generatorContext.VariableProcessor, packageRecipeResultManager, generatorContext.ErrorHelpManager) missingPackagesInBuildOrder = __FindMissingInstallations(log, validationEngine, resolvedBuildOrder, recipePackageStateCache, generatorContext.CMakeConfig) builder = PipelineCommandBuilder(generatorContext, builderSettings.CheckBuildCommands, builderSettings.BuildThreads) recipeRecords = __CreatePipelines(log, builder, missingPackagesInBuildOrder) for recipeRecord in recipeRecords: log.LogPrint("Package location: {0}".format(recipeRecord.SourcePackage.AbsolutePath)) try: log.PushIndent() if not recipeRecord.SourcePackage.ResolvedPlatformDirectSupported: raise Exception("The package '{0}' is not supported on this platform".format(recipeRecord.SourcePackage.Name)) if not recipeRecord.Pipeline is None: log.DoPrint("Building package: {0}".format(recipeRecord.SourcePackage.Name)) if builderSettings.PreDeleteBuild: # We clear the build path to prepare for a new build IOUtil.SafeRemoveDirectoryTree(recipeRecord.Pipeline.BuildPath) for command in recipeRecord.Pipeline.CommandList: if not configIsDryRun: command.Execute() # We finished building, so lets save some information about what we did BuildInfoFileUtil.SaveBuildInformation(log, recipeRecord, recipePackageStateCache, generatorContext.CMakeConfig, __g_BuildPackageInformationFilename) if builderSettings.PostDeleteBuild: # We clear the build path if a build is successfull IOUtil.SafeRemoveDirectoryTree(recipeRecord.Pipeline.BuildPath, True) else: # Since we are trying to build this it means that the installation validation failed earlier and # we apparently have no pipelines that could remedy it, so force the install validation to occur so # we fail early as 'dependent' pipes might fail to build due to this # generatorContext.RecipeFilterManager if generatorContext.RecipeFilterManager.AllRecipesEnabled or recipeRecord.SourcePackage.Name in generatorContext.RecipeFilterManager.ContentDict: log.DoPrintWarning("Missing installation of package '{0}' and no recipe for solving it is available".format(recipeRecord.SourcePackage.Name)) else: log.LogPrintVerbose(4, "Package '{0}' recipe not enabled".format(recipeRecord.SourcePackage.Name)) validationEngine.Process(recipeRecord.SourcePackage) finally: log.PopIndent() validationEngine.Process(recipeRecord.SourcePackage) packageCount = len(recipeRecords) if packageCount > 0: log.LogPrint("Build {0} packages".format(packageCount)) else: log.LogPrintVerbose(2, "No recipe was build!")