def __init__(self, configFileName, configOverrides, defaultConfigOptions=[]): trace.config(2, "Creating ConfigStack with {0}".format(configFileName)) self._modules = CodeSurveyorModules() self._reader = configreader.ConfigReader(self.load_csmodule) self._measureRootDir = '' # Stack of config files, represented as paths and lists of ConfigEntrys self._configStack = [] # Cache of config file information # Key is path name, value is list entries that represent the config file self._configFileCache = {} # List of default config option tags passed by the application self._defaultConfigOptions = defaultConfigOptions # We either use overrides or try to read config files if configOverrides: trace.msg(1, "Ignoring config files: {0}".format(configOverrides)) self._configName = '' self._setup_config_overrides(configOverrides) else: self._configName = configFileName # Make sure the config file name does not include a path, as the point is # to look for a config file in each folder we visit if not os.path.dirname(self._configName) == '': raise utils.ConfigError(uistrings.STR_ErrorConfigFileNameHasPath) # Load the default config file to use for this job # First try in the root of the job folder; then in the surveyor folder if not self._push_file(utils.runtime_dir()): if not self._push_file(utils.surveyor_dir()): trace.msg(1, "{0} not present in default locations".format( self._configName))
def __init__(self, configOptions): # Modules overrides to tell config validation what they can measure self.measures = [] self.verbs = [] self.verbEnds = {} # We store this each measure call in case derived class wants to use self._currentPath = None # Delegate that subclasses can call to add config options self._configOptionDict = {} self._cs_init_config_options() # Process any config options for optName, optValue in configOptions: trace.config(2, "ConfigOpt: {0}:{1}".format(optName, optValue)) try: configCode, _configHelp = self._configOptionDict[optName] except KeyError, e: raise utils.CsModuleException("Invalid Config Option: {0}".format(str(e))) trace.config(3, "ConfigCode: {0}".format(configCode)) try: exec(configCode) except Exception, e: trace.traceback() raise utils.CsModuleException("Error executing Config Option: {0}".format(str(e)))
def _validate_entries(self, configEntries): ''' Are all config file entries consistent with each other, to avoid silent double counting? Throws an error exception if not. ''' trace.config(2, "Checking for duplicate config entries") # Create list of all possible measure/file combos # We ask the module to match each measure, to catch wildcard overlap fileFilters = [] possibleMeasures = [] for entry in configEntries: for fileFilter in entry.fileFilters: fileFilters.append(fileFilter) possibleMeasures.append((fileFilter, entry.measureFilter, entry.moduleName, entry.verb, entry.tags, entry.paramsRaw)) trace.config(4, fileFilters) trace.config(4, possibleMeasures) # Check that no file type would have a measure be double counted # If we have a problem, throw an exception based on the first problem item if len(fileFilters) > len(set(fileFilters)): while possibleMeasures: possibleMeasureTuple = possibleMeasures.pop() trace.config(2, "possibleMeasure: {0}".format(possibleMeasureTuple)) (fileFilter, measureFilter, modName, verb, tags, extraParams) = possibleMeasureTuple # We don't attempt the do conflict resolution on regex files extensions, # both because it doesn't make sense if fileFilter.startswith(fileext.CUSTOM_FILE_REGEX): continue # Shallow warning check for double counting by creatubg a list of entries # based on matching verb and file type warningList = [ (ff, mf, mn, v, t, ep) for ff, mf, mn, v, t, ep in possibleMeasures if v == verb and fileext.file_ext_match(ff, fileFilter) ] if warningList: trace.config(1, "WARNING - Possible double-count: {0}".format(str(warningList))) # For the deep check look at tag values and measure filter dupeList = [ (v, modName, mn, mf, fileFilter, ff, t, tags, ep, extraParams) for ff, mf, mn, v, t, ep in warningList if len(t) == len(tags) and len(t) == len(set(t) & set(tags)) and entry.module.match_measure(mf, measureFilter) ] if dupeList: trace.msg(1, "ERROR - Double-count: {0}".format(str(dupeList))) dupe = dupeList[0] raise utils.ConfigError(uistrings.STR_ErrorConfigDupeMeasures.format( dupe[0], dupe[1], dupe[2], dupe[3], dupe[4], dupe[5], dupe[6], dupe[7], dupe[8], dupe[9]))
def _validate_file(self, configEntries): if not configEntries: trace.config(2, " EMPTY") else: try: self._validate_entries(configEntries) except Exception, e: trace.traceback() raise utils.ConfigError(uistrings.STR_ErrorConfigValidate.format(str(e)))
def get_csmodule(self, csmoduleName, options=[]): ''' Return the csmodule class with the given name, if it exists ''' csmodule = None try: csmodule = self.moduleList[self._csmod_hash(csmoduleName, options)] except KeyError: trace.config(2, "Loading csmodule: {0}".format(csmoduleName)) csmodule = self._load_csmodule(csmoduleName, options) if csmodule is not None: self.moduleList[self._csmod_hash(csmoduleName, options)] = csmodule return csmodule
def read_file(self, filePath): ''' Read a Surveyor configuration file and return a list of ConfigEntrys to be stored on the configuration stack with this folder location. ''' try: trace.msg(1, "Config file: {0}".format(filePath)) configEntries = self._read_file(filePath, []) self._validate_file(configEntries) trace.config(2, "Finsihed reading config file: {0}".format(filePath)) trace.config(3, configEntries) return configEntries except Exception, e: raise utils.ConfigError(uistrings.STR_ErrorConfigFile.format(filePath, str(e)))
def load_csmodule(self, configEntry): ''' Callback for ConfigReader to load modules Module loading is delegated to our set of cached modules We concatonate and default config options from the application with any options defined in the conifig file. ''' trace.config(3, configEntry.__dict__) configEntry.module = self._modules.get_csmodule( configEntry.moduleName, self._defaultConfigOptions + configEntry.options) if configEntry.module is None: raise utils.ConfigError(uistrings.STR_ErrorFindingModule.format( configEntry.moduleName))
def load_csmodule(self, configEntry): ''' Callback for ConfigReader to load modules Module loading is delegated to our set of cached modules We concatonate and default config options from the application with any options defined in the conifig file. ''' trace.config(3, configEntry.__dict__) configEntry.module = self._modules.get_csmodule( configEntry.moduleName, self._defaultConfigOptions + configEntry.options) if configEntry.module is None: raise utils.ConfigError( uistrings.STR_ErrorFindingModule.format( configEntry.moduleName))
def add_tags_and_options(self, tagItems): # Parse the "tag" values into tags and options for item in tagItems: if item.startswith(CONFIG_DELIM_OPTION): opt = item[len(CONFIG_DELIM_OPTION):].split(CONFIG_DELIM_CHAR) # There are two types of options, with and without values if len(opt) > 1: # Everything after OPT tag is value string, we joing back together # in case the string had delim chars in it optionStr = CONFIG_DELIM_CHAR.join(opt[1:]) self.options.append((str(opt[0]), optionStr)) trace.config(2, "Option Load: {0} -> {1}".format(str(opt[0]), optionStr)) elif opt: self.options.append((str(opt[0]), None)) trace.config(2, "Option Selected: {0}".format(str(opt[0]))) else: self.tags.append(item)
def get_configuration(self, folder): ''' Returns two collections: 1) A set of all file filters active for folder 2) A dict by file filter with list of ConfigEntry objects for folder The active configuration is the contents of the config file closest to the leaf directory passed in as you look back up the parent subdirectory tree, ending with the default job config. ''' self._pop_to_active(folder) self._push_file(folder) path, fileFilters, activeConfigItems = self._active_entry() trace.config(4, "Config: {0} -- {1} possible entries".format(path, len(activeConfigItems))) return fileFilters, activeConfigItems, path
def match_measure(self, measureName, measureFilters): ''' Used to both validate config and to filter results Measure filters allow using * to match the end of the string. i.e.: size.* would match size.totallines and size.blanklines We also pass any measure that does not have a '.', which indicates it is system measure we always want to output ''' if '*' in measureFilters: return True if '.' not in measureName: return True for measureFilter in measureFilters: match = compare_filters(measureName, measureFilter) trace.config(4, "Compared {0} to {1}: {2}".format(measureName, measureFilter, match)) if match: return True return False
def _push_file(self, dirName): ''' Returns true if a config file was found in dirName and pushed on stack ''' success = False configFilePath = os.path.abspath(os.path.join(dirName, self._configName)) if not configFilePath in self._configFileCache: if os.path.isfile(configFilePath): self._configFileCache[configFilePath] = self._reader.read_file(configFilePath) if configFilePath in self._configFileCache: self._push_entries(configFilePath, self._configFileCache[configFilePath]) trace.config(1, "Config PUSH {0}: {1}".format( len(self._configFileCache[configFilePath]), configFilePath)) if len(self._configFileCache[configFilePath]) == 0: trace.config(1, "EMPTY CONFIG: {0}".format(configFilePath)) success = True; return success
def get_configuration(self, folder): ''' Returns two collections: 1) A set of all file filters active for folder 2) A dict by file filter with list of ConfigEntry objects for folder The active configuration is the contents of the config file closest to the leaf directory passed in as you look back up the parent subdirectory tree, ending with the default job config. ''' self._pop_to_active(folder) self._push_file(folder) path, fileFilters, activeConfigItems = self._active_entry() trace.config( 4, "Config: {0} -- {1} possible entries".format( path, len(activeConfigItems))) return fileFilters, activeConfigItems, path
def _pop_to_active(self, dirToCheck): ''' Removes config entries back up the folder chain, until we get to the active one. ''' configIndex = self._active_entry_index() # DO NOT EVER pop the first position, as it should be a default file while configIndex > 0: # Get active config path and remove file name configDir, _configItems, _configPath = self._configStack[ configIndex] configDir = os.path.dirname(configDir) currentDir = os.path.abspath(dirToCheck) # Is the config file equal to or "above" current position in path? # Special case the current folder '.' below because commonprefix # returns it, while dirname returns blank for empty path currentDirUnderConfigDir = False sharedPath = os.path.commonprefix([currentDir, configDir]) if not sharedPath == '': currentDir = os.path.relpath(currentDir, sharedPath) configDir = os.path.relpath(configDir, sharedPath) while True: if configDir == currentDir: currentDirUnderConfigDir = True break if '.' == currentDir: break currentDir = os.path.dirname(currentDir) if not currentDir: currentDir = '.' # If the current config file does not cover the currentDir, pop it if currentDirUnderConfigDir: break else: trace.config(1, "Config POP: {0}".format(self.active_path())) del self._configStack[configIndex] configIndex -= 1
def _pop_to_active(self, dirToCheck): ''' Removes config entries back up the folder chain, until we get to the active one. ''' configIndex = self._active_entry_index() # DO NOT EVER pop the first position, as it should be a default file while configIndex > 0: # Get active config path and remove file name configDir, _configItems, _configPath = self._configStack[configIndex] configDir = os.path.dirname(configDir) currentDir = os.path.abspath(dirToCheck) # Is the config file equal to or "above" current position in path? # Special case the current folder '.' below because commonprefix # returns it, while dirname returns blank for empty path currentDirUnderConfigDir = False sharedPath = os.path.commonprefix([currentDir, configDir]) if not sharedPath == '': currentDir = os.path.relpath(currentDir, sharedPath) configDir = os.path.relpath(configDir, sharedPath) while True: if configDir == currentDir: currentDirUnderConfigDir = True break; if '.' == currentDir: break currentDir = os.path.dirname(currentDir) if not currentDir: currentDir = '.' # If the current config file does not cover the currentDir, pop it if currentDirUnderConfigDir: break else: trace.config(1, "Config POP: {0}".format(self.active_path())) del self._configStack[configIndex] configIndex -= 1
def __init__(self, configFileName, configOverrides, defaultConfigOptions=[]): trace.config(2, "Creating ConfigStack with {0}".format(configFileName)) self._modules = CodeSurveyorModules() self._reader = configreader.ConfigReader(self.load_csmodule) self._measureRootDir = '' # Stack of config files, represented as paths and lists of ConfigEntrys self._configStack = [] # Cache of config file information # Key is path name, value is list entries that represent the config file self._configFileCache = {} # List of default config option tags passed by the application self._defaultConfigOptions = defaultConfigOptions # We either use overrides or try to read config files if configOverrides: trace.msg(1, "Ignoring config files: {0}".format(configOverrides)) self._configName = '' self._setup_config_overrides(configOverrides) else: self._configName = configFileName # Make sure the config file name does not include a path, as the point is # to look for a config file in each folder we visit if not os.path.dirname(self._configName) == '': raise utils.ConfigError( uistrings.STR_ErrorConfigFileNameHasPath) # Load the default config file to use for this job # First try in the root of the job folder; then in the surveyor folder if not self._push_file(utils.runtime_dir()): if not self._push_file(utils.surveyor_dir()): trace.msg( 1, "{0} not present in default locations".format( self._configName))
def config_items_for_file(configEntrys, fileName): ''' Return a list of config items that match the given fileName ''' neededConfigs = [] # We don't know how many config entrys could be associated with a given # file extension (files could match more than one config file filter), # so we check against every config # If there are custom RE config filters, we include them no matter what, # since we can't just match them against the file extension for configFilter in configEntrys.keys(): if fileext.file_ext_match(fileName, configFilter): for config in configEntrys[configFilter]: neededConfigs.append(config) # Make this a sorted list to ensure repeatble results in terms of # order files are processed. This doesn't normally matter, but can # be convienent and allows for single-threaded repeatability that # allows for comparison against test oracle neededConfigs.sort(key=lambda configSort: str(configSort)) trace.config(3, neededConfigs) return neededConfigs
def config_items_for_file(configEntrys, fileName): ''' Return a list of config items that match the given fileName ''' neededConfigs = [] # We don't know how many config entrys could be associated with a given # file extension (files could match more than one config file filter), # so we check against every config # If there are custom RE config filters, we include them no matter what, # since we can't just match them against the file extension for configFilter in configEntrys.keys(): if fileext.file_ext_match(fileName, configFilter): for config in configEntrys[configFilter]: neededConfigs.append(config) # Make this a sorted list to ensure repeatble results in terms of # order files are processed. This doesn't normally matter, but can # be convienent and allows for single-threaded repeatability that # allows for comparison against test oracle neededConfigs.sort(key=lambda configSort: str(configSort)) trace.config(3, neededConfigs) return neededConfigs
def _push_file(self, dirName): ''' Returns true if a config file was found in dirName and pushed on stack ''' success = False configFilePath = os.path.abspath( os.path.join(dirName, self._configName)) if not configFilePath in self._configFileCache: if os.path.isfile(configFilePath): self._configFileCache[configFilePath] = self._reader.read_file( configFilePath) if configFilePath in self._configFileCache: self._push_entries(configFilePath, self._configFileCache[configFilePath]) trace.config( 1, "Config PUSH {0}: {1}".format( len(self._configFileCache[configFilePath]), configFilePath)) if len(self._configFileCache[configFilePath]) == 0: trace.config(1, "EMPTY CONFIG: {0}".format(configFilePath)) success = True return success
else: return self._parse_help_options() # Setup the default measurement path if not provided if not self._app._jobOpt.pathsToMeasure: self._app._jobOpt.pathsToMeasure.append(utils.CURRENT_FOLDER) # Setup the default config name if not provided if not self.configOverrides and self.configCustom is None: self.configCustom = CONFIG_FILE_DEFAULT_NAME except Args.ArgsFinishedException, e: raise utils.InputException(STR_ErrorParsingEnd.format(str(e))) else: trace.config(4, vars(self._app)) def config_option_list(self): ''' Maps application modifiable config options to the name, value list format used by csmodules to process config file options ''' configOptions = [] configOptions.append(('METADATA', self._metaDataOptions)) if self._measureFilter is not None: if self._measureFilter == CMDARG_OUTPUT_FILTER_METADATA: configOptions.append(('METADATA_ONLY', None)) configOptions.append(('MEASURE_FILTER', '*')) else: configOptions.append(('MEASURE_FILTER', self._measureFilter))
# Help/invalid parameter request else: return self._parse_help_options() # Setup the default measurement path if not provided if not self._app._jobOpt.pathsToMeasure: self._app._jobOpt.pathsToMeasure.append(utils.CURRENT_FOLDER) # Setup the default config name if not provided if not self.configOverrides and self.configCustom is None: self.configCustom = CONFIG_FILE_DEFAULT_NAME except Args.ArgsFinishedException, e: raise utils.InputException(STR_ErrorParsingEnd.format(str(e))) else: trace.config(4, vars(self._app)) def config_option_list(self): ''' Maps application modifiable config options to the name, value list format used by csmodules to process config file options ''' configOptions = [] configOptions.append(('METADATA', self._metaDataOptions)) if self._measureFilter is not None: if self._measureFilter == CMDARG_OUTPUT_FILTER_METADATA: configOptions.append(('METADATA_ONLY', None)) configOptions.append(('MEASURE_FILTER', '*')) else: configOptions.append(('MEASURE_FILTER', self._measureFilter)) if self.ignoreSize > 0:
def _parse_file(self, configFile, configEntries): ''' Parse config file lines ''' configEntry = configentry.ConfigEntry('_ _ _ _') # Init to empty object to prevent PyChecker warnings constants = {} readingVerbs = False verbEndMarker = None for whiteSpaceRawline in configFile: trace.config(3, "Config line: {0}".format(whiteSpaceRawline)) rawLine = whiteSpaceRawline.strip() line = rawLine # Skip comments, blank lines if self.comment.match(line) or self.blankLine.match(line): trace.config(4, "comment/blank") continue # Skip ignore blocks (or go to end of file if no closing block) if self.ignoreStart.match(line): trace.config(4, "ignoreBlock") try: while not self.ignoreStop.match(line): line = configFile.next() trace.config(4, "Config ignore: {0}".format(line)) except Exception: trace.config(4, "Exception while seeking end of ignore block") pass continue # Includes # Attempt to load the requested file and add it's entries # to our entries, in the form INCLUDE:path: tagInfo includeMatch = self.include.match(line) if includeMatch: includePath = includeMatch.group(1) newTags = includeMatch.group(2) if not os.path.isabs(includePath): includePath = os.path.join(os.path.dirname(configFile.name), includePath) trace.config(1, "Include: {0}".format(includePath)) newEntries = self._read_file(includePath, []) existingFileFilterStrings = [entry.fileFilter for entry in configEntries] for entry in newEntries: # If an entry has already been defined with the SAME FILE FILTER STRING, # the INCLUDED ENTRY WILL BE IGNORED if entry.fileFilter in existingFileFilterStrings: continue # If 'tagInfo' is provided, it will be added to ALL entries of the file # that was included # We RELOAD THE MODULE in case new options need processed if newTags: entry.add_tags_and_options(newTags.split()) self._load_csmodule(entry) configEntries.append(entry) continue # If line closes out a verb entry store the config entry if readingVerbs and re.match(verbEndMarker, line): trace.config(4, "verbend: {0}".format(line)) readingVerbs = False configEntries.append(configEntry) continue # Handle continued lines fullLine = "" while True: contLineMatch = self.continuedLine.match(line) if contLineMatch: fullLine += contLineMatch.group(CONT_LINE_START) line = configFile.next().strip() trace.config(3, "FullLine: {0}".format(line)) else: fullLine += line break assert fullLine line = fullLine # If line defines a normal constant, store asis constantMatch = self.constant.match(line) if constantMatch: # Assign cosntant, strip spaces to support config lines that are space-delimited constants[constantMatch.group(1)] = constantMatch.group(2) trace.config(2, "Constant: {0}".format(constantMatch.group(2))) continue # If line defines a no blanks constant, strip spaces and store constantMatch = self.constant_noblanks.match(line) if constantMatch: constants[constantMatch.group(1)] = constantMatch.group(2).replace(' ', '') trace.config(2, "Noblank constant: {0}".format(constantMatch.group(2))) continue # Replace any constants used in the line line = self._replace_constants(line, constants) trace.config(4, "fullline: {0}".format(line)) # Strip any inline comments line = line.split(' #')[0] # If the line is a parameter (e.g., search terms), delegate to module # to get processed parameters and store for later usage # We keep the unprocessed raw version around for consistency checking if readingVerbs: configEntry.paramsRaw.append(rawLine) try: paramTuple = configEntry.module.add_param(line, rawLine) configEntry.paramsProcessed.append(paramTuple) trace.config(2, "LoadedParam: {0} => {1}".format( configEntry.module.__class__.__name__, paramTuple)) except Exception, e: trace.traceback() raise utils.ConfigError(uistrings.STR_ErrorConfigParam.format( str(configEntry), rawLine, str(e))) # Otherwise assume we're at the start of a config entry definition, else: try: # Load and validate the config line and its module configEntry = configentry.ConfigEntry(line, self._extraLineContent, configFile.name) self._load_csmodule(configEntry) self._validate_line(configEntry) # Check to see if there are parameter lines to read verbEndMarker = configEntry.module.verb_end_marker(configEntry.verb) if verbEndMarker is not None: readingVerbs = True # Add the completed config entry to our list if not readingVerbs: configEntries.append(configEntry) except Exception, e: trace.traceback() raise utils.ConfigError(uistrings.STR_ErrorConfigEntry.format( rawLine, str(e)))