def _validate_file(self, configEntries): if not configEntries: log.config(2, " EMPTY") else: try: self._validate_entries(configEntries) except Exception as e: log.stack() raise utils.ConfigError( uistrings.STR_ErrorConfigValidate.format(str(e)))
def get_csmodule(self, moduleName, options=[]): ''' Return the csmodule class with the given name, if it exists ''' modHash = self._csmod_hash(moduleName, options) module = self.moduleList.get(modHash) if not module: log.config(2, "Loading csmodule: {}".format(moduleName)) module = self._load_csmodule(moduleName, options) self.moduleList[modHash] = module return module
def read_file(self, filePath): ''' Read a Surveyor configuration file and return a list of ConfigEntrys to store on the configuration stack with this folder location. ''' try: log.msg(1, "Config file: {}".format(filePath)) configEntries = self._read_file(filePath, []) self._validate_file(configEntries) log.config(2, "Finsihed reading config file: {}".format(filePath)) log.config(3, configEntries) return configEntries except Exception as e: raise utils.ConfigError( uistrings.STR_ErrorConfigFile.format(filePath, str(e)))
def load_csmodule(self, configEntry): ''' Callback for ConfigReader to load modules Module loading is delegated to set of cached modules Concatonate and default config options from the application with any options defined in the conifig file. ''' log.config(3, configEntry.__dict__) configEntry.module = self._modules.get_csmodule( configEntry.moduleName, self._defaultConfigOptions + configEntry.options) if configEntry.module is None: raise utils.ConfigError( uistrings.STR_ErrorFindingModule.format( configEntry.moduleName))
def add_tags_and_options(self, tagItems): # Parse the "tag" values into tags and options for item in tagItems: if item.startswith(CONFIG_DELIM_OPTION): opt = item[len(CONFIG_DELIM_OPTION):].split(CONFIG_DELIM_CHAR) # There are two types of options, with and without values if len(opt) > 1: # Everything after OPT tag is value string, join back together # in case the string had delim chars in it optionStr = CONFIG_DELIM_CHAR.join(opt[1:]) self.options.append((str(opt[0]), optionStr)) log.config( 2, "Option Load: {} -> {}".format(str(opt[0]), optionStr)) elif opt: self.options.append((str(opt[0]), None)) log.config(2, "Option Selected: {}".format(str(opt[0]))) else: self.tags.append(item)
def get_configuration(self, folder): ''' Returns two collections: 1) A set of all file filters active for folder 2) A dict by file filter with list of ConfigEntry objects for folder The active configuration is the contents of the config file closest to the leaf directory passed in as you look back up the parent subdirectory tree, ending with the default job config. ''' self._pop_to_active(folder) self._push_file(folder) path, fileFilters, activeConfigItems = self._active_entry() log.config( 4, "Config: {} -- {} possible entries".format(path, len(activeConfigItems))) return fileFilters, activeConfigItems, path
def _pop_to_active(self, dirToCheck): ''' Removes config entries back up the folder chain, until we get to the active one. ''' configIndex = self._active_entry_index() # DO NOT EVER pop the first position, as it should be a default file while configIndex > 0: # Get active config path and remove file name configDir, _configItems, _configPath = self._configStack[ configIndex] configDir = os.path.dirname(configDir) currentDir = os.path.abspath(dirToCheck) # Is the config file equal to or "above" current position in path? # Special case the current folder '.' below because commonprefix # returns it, while dirname returns blank for empty path currentDirUnderConfigDir = False sharedPath = os.path.commonprefix([currentDir, configDir]) if not sharedPath == '': currentDir = os.path.relpath(currentDir, sharedPath) configDir = os.path.relpath(configDir, sharedPath) while True: if configDir == currentDir: currentDirUnderConfigDir = True break if '.' == currentDir: break currentDir = os.path.dirname(currentDir) if not currentDir: currentDir = '.' # If the current config file does not cover the currentDir, pop it if currentDirUnderConfigDir: break else: log.config(1, "Config POP: {}".format(self.active_path())) del self._configStack[configIndex] configIndex -= 1
def __init__(self, configFileName, configOverrides, defaultConfigOptions=[]): log.config(2, "Creating ConfigStack with {}".format(configFileName)) self._modules = CodeSurveyorModules() self._reader = configreader.ConfigReader(self.load_csmodule) self._measureRootDir = '' # Stack of config files, represented as paths and lists of ConfigEntrys self._configStack = [] # Cache of config file information # Key is path name, value is list entries that represent the config file self._configFileCache = {} # List of default config option tags passed by the application self._defaultConfigOptions = defaultConfigOptions # Either use overrides or try to read config files if configOverrides: log.msg(1, "Ignoring config files: {}".format(configOverrides)) self._configName = '' self._setup_config_overrides(configOverrides) else: self._configName = configFileName # Make sure the config file name does not include a path, as the point is # to look for a config file in each folder we visit if not os.path.dirname(self._configName) == '': raise utils.ConfigError( uistrings.STR_ErrorConfigFileNameHasPath) # Load the default config file to use for this job # First try in the root of the job folder; then in the surveyor folder if not self._push_file(runtime_dir()): if not self._push_file(surveyor_dir()): log.msg( 1, "{} not present in default locations".format( self._configName))
def config_items_for_file(configEntrys, fileName): ''' Return a list of config items that match the given fileName ''' neededConfigs = [] # Don't know how many config entrys could be associated with a given # file extension (files could match more than one config file filter), # so we check against every config # If there are custom RE config filters, we include them no matter what, # since we can't just match them against the file extension for configFilter in list(configEntrys.keys()): if fileext.file_ext_match(fileName, configFilter): for config in configEntrys[configFilter]: neededConfigs.append(config) # Make this a sorted list to ensure repeatble results in terms of # order files are processed. This doesn't normally matter, but can # be convienent and allows for single-threaded repeatability that # allows for comparison against test oracle neededConfigs.sort(key=lambda configSort: str(configSort)) log.config(3, neededConfigs) return neededConfigs
def _push_file(self, dirName): ''' Returns true if a config file was found in dirName and pushed on stack ''' success = False configFilePath = os.path.abspath( os.path.join(dirName, self._configName)) if not configFilePath in self._configFileCache: if os.path.isfile(configFilePath): self._configFileCache[configFilePath] = self._reader.read_file( configFilePath) if configFilePath in self._configFileCache: self._push_entries(configFilePath, self._configFileCache[configFilePath]) log.config( 1, "Config PUSH {}: {}".format( len(self._configFileCache[configFilePath]), configFilePath)) if len(self._configFileCache[configFilePath]) == 0: log.config(1, "EMPTY CONFIG: {}".format(configFilePath)) success = True return success
def _parse_file(self, configFile, configEntries): ''' Parse config file lines ''' configEntry = configentry.ConfigEntry( '_ _ _ _') # Init to empty object to prevent PyChecker warnings constants = {} readingVerbs = False verbEndMarker = None for whiteSpaceRawline in configFile: log.config(3, "Config line: {}".format(whiteSpaceRawline)) rawLine = whiteSpaceRawline.strip() line = rawLine # Skip comments, blank lines if self.comment.match(line) or self.blankLine.match(line): log.config(4, "comment/blank") continue # Skip ignore blocks (or go to end of file if no closing block) if self.ignoreStart.match(line): log.config(4, "ignoreBlock") try: while not self.ignoreStop.match(line): line = next(configFile) log.config(4, "Config ignore: {}".format(line)) except Exception: log.config(4, "Exception while seeking end of ignore block") pass continue # Includes # Attempt to load the requested file and add it's entries # to our entries, in the form INCLUDE:path: tagInfo includeMatch = self.include.match(line) if includeMatch: includePath = includeMatch.group(1) newTags = includeMatch.group(2) if not os.path.isabs(includePath): includePath = os.path.join( os.path.dirname(configFile.name), includePath) log.config(1, "Include: {}".format(includePath)) newEntries = self._read_file(includePath, []) existingFileFilterStrings = [ entry.fileFilter for entry in configEntries ] for entry in newEntries: # If an entry has already been defined with the SAME FILE FILTER STRING, # the INCLUDED ENTRY WILL BE IGNORED if entry.fileFilter in existingFileFilterStrings: continue # If 'tagInfo' is provided, it will be added to ALL entries of the file # that was included # RELOAD THE MODULE in case new options need processed if newTags: entry.add_tags_and_options(newTags.split()) self._load_csmodule(entry) configEntries.append(entry) continue # If line closes out a verb entry store the config entry if readingVerbs and re.match(verbEndMarker, line): log.config(4, "verbend: {}".format(line)) readingVerbs = False configEntries.append(configEntry) continue # Handle continued lines fullLine = "" while True: contLineMatch = self.continuedLine.match(line) if contLineMatch: fullLine += contLineMatch.group(CONT_LINE_START) line = next(configFile).strip() log.config(3, "FullLine: {}".format(line)) else: fullLine += line break assert fullLine line = fullLine # If line defines a normal constant, store asis constantMatch = self.constant.match(line) if constantMatch: # Assign cosntant, strip spaces to support config lines that are space-delimited constants[constantMatch.group(1)] = constantMatch.group(2) log.config(2, "Constant: {}".format(constantMatch.group(2))) continue # If line defines a no blanks constant, strip spaces and store constantMatch = self.constant_noblanks.match(line) if constantMatch: constants[constantMatch.group(1)] = constantMatch.group( 2).replace(' ', '') log.config( 2, "Noblank constant: {}".format(constantMatch.group(2))) continue # Replace any constants used in the line line = self._replace_constants(line, constants) log.config(4, "fullline: {}".format(line)) # Strip any inline comments line = line.split(' #')[0] # If the line is a parameter (e.g., search terms), delegate to module # to get processed parameters and store for later usage # Keep the unprocessed raw version around for consistency checking if readingVerbs: configEntry.paramsRaw.append(rawLine) try: paramTuple = configEntry.module.add_param(line, rawLine) configEntry.paramsProcessed.append(paramTuple) log.config( 2, "LoadedParam: {} => {}".format( configEntry.module.__class__.__name__, paramTuple)) except Exception as e: log.stack() raise utils.ConfigError( uistrings.STR_ErrorConfigParam.format( str(configEntry), rawLine, str(e))) # Otherwise assume we're at the start of a config entry definition, else: try: # Load and validate the config line and its module configEntry = configentry.ConfigEntry( line, self._extraLineContent, configFile.name) self._load_csmodule(configEntry) self._validate_line(configEntry) # Check to see if there are parameter lines to read verbEndMarker = configEntry.module.verb_end_marker( configEntry.verb) if verbEndMarker is not None: readingVerbs = True # Add the completed config entry to our list if not readingVerbs: configEntries.append(configEntry) except Exception as e: log.stack() raise utils.ConfigError( uistrings.STR_ErrorConfigEntry.format(rawLine, str(e))) # Loop to next line return configEntries
def _validate_entries(self, configEntries): ''' Are all config file entries consistent with each other, to avoid silent double counting? Throws an error exception if not. ''' log.config(2, "Checking for duplicate config entries") # Create list of all possible measure/file combos # Ask the module to match each measure, to catch wildcard overlap fileFilters = [] possibleMeasures = [] for entry in configEntries: for fileFilter in entry.fileFilters: fileFilters.append(fileFilter) possibleMeasures.append( (fileFilter, entry.measureFilter, entry.moduleName, entry.verb, entry.tags, entry.paramsRaw)) log.config(4, fileFilters) log.config(4, possibleMeasures) # Check that no file type would have a measure be double counted # If a problem, throw an exception based on the first problem item if len(fileFilters) > len(set(fileFilters)): while possibleMeasures: possibleMeasureTuple = possibleMeasures.pop() log.config(2, "possibleMeasure: {}".format(possibleMeasureTuple)) (fileFilter, measureFilter, modName, verb, tags, extraParams) = possibleMeasureTuple # Don't attempt the do conflict resolution on regex files extensions, # both because it doesn't make sense if fileFilter.startswith(fileext.CUSTOM_FILE_REGEX): continue # Shallow warning check for double counting by creatubg a list of entries # based on matching verb and file type warningList = [ (ff, mf, mn, v, t, ep) for ff, mf, mn, v, t, ep in possibleMeasures if v == verb and fileext.file_ext_match(ff, fileFilter) ] if warningList: log.config( 1, "WARNING - Possible double-count: {}".format( str(warningList))) # For the deep check look at tag values and measure filter dupeList = [ (v, modName, mn, mf, fileFilter, ff, t, tags, ep, extraParams) for ff, mf, mn, v, t, ep in warningList if len(t) == len(tags) and len(t) == len(set(t) & set(tags)) and entry.module.match_measure(mf, measureFilter) ] if dupeList: log.msg( 1, "ERROR - Double-count: {}".format(str(dupeList))) dupe = dupeList[0] raise utils.ConfigError( uistrings.STR_ErrorConfigDupeMeasures.format( dupe[0], dupe[1], dupe[2], dupe[3], dupe[4], dupe[5], dupe[6], dupe[7], dupe[8], dupe[9]))
def parse_args(self): ''' Do simple command line parsing and set the internal state of our Surveyor class based on the arguments. For any syntax we don't recognize or help is requested, return help text. Otherwise return None which indicates success. ''' try: while not self.args.finished(): self.args.move_next() # Disambiguation case for measurePath/fileFilter # A '-' may be used to replace optional arg with path/filter if self.args.is_cmd() and len(self.args.get_current()) == 1: if self.args.is_param_next(): self.args.move_next() self._parse_measurement_path() continue # Assume non-Arg is a measurePath/fileFilter definition elif not self.args.is_cmd(): self._parse_measurement_path() continue # Our processing is based on matching first character fc = self.args.get_current()[1].lower() # Debug and profiling support if fc in CMDARG_DEBUG: self._parse_debug_options() log.msg(2, "Args: {}".format(str(self.args))) elif fc in CMDARG_PROFILE: self._app._profiling = True self._app._profileCalls = self._get_next_int(optional=True, default=self._app._profileCalls) self._app._profileCalledBy = self._get_next_int(optional=True, default=self._app._profileCalledBy) self._app._profileCalled = self._get_next_int(optional=True, default=self._app._profileCalled) self._app._profileThreadFilter = self._get_next_str(optional=True, default=self._app._profileThreadFilter) self._app._profileNameFilter = self._get_next_str(optional=True, default=self._app._profileNameFilter) # Config file settings elif fc in CMDARG_CONFIG_CUSTOM: self._parse_config_options() # Delta path elif fc in CMDARG_DELTA: self._parse_delta_options() # Duplicate processing # Can have an optional integer or string after this option elif fc in CMDARG_DUPE_PROCESSING: self._app._dupeTracking = True self._metaDataOptions['DUPE'] = None dupeParam = self._get_next_param(optional=True) try: dupeParam = int(dupeParam) except Exception as e: pass self._app._dupeThreshold = dupeParam # Scan and skip options elif fc in CMDARG_SCAN_ALL: self._parse_scan_options() elif fc in CMDARG_SKIP: self._parse_skip_options() elif fc in CMDARG_INCLUDE_ONLY: self._app._jobOpt.includeFolders.extend(self._get_next_param().split(CMDLINE_SEPARATOR)) # Output elif fc in CMDARG_METADATA == fc: self._parse_metadata_options() elif fc in CMDARG_OUTPUT_FILTER: self._measureFilter = self._get_next_str() elif fc in CMDARG_OUTPUT_TYPE == fc: self._app._outType = self._get_next_str() elif fc in CMDARG_OUTPUT_FILE == fc: self._parse_output_file() elif fc in CMDARG_SUMMARY_ONLY == fc: self._app._summaryOnly = True elif fc in CMDARG_DETAILED == fc: self._app._detailed = True self._app._detailedPrintSummaryMax = self._get_next_int( optional=True, default=self._app._detailedPrintSummaryMax) elif fc in CMDARG_PROGRESS == fc: self._app._progress = True self._app._printMaxWidth = self._get_next_int( optional=True, default=self._app._printMaxWidth) elif fc in CMDARG_QUIET == fc: self._app._quiet = True # Other options elif fc in CMDARG_NUM_WORKERS: self._app._jobOpt.numWorkers = self._get_next_int(validRange=range(1,MAX_WORKERS)) elif fc in CMDARG_RECURSION: self._app._jobOpt.recursive = False elif fc in CMDARG_BREAK_ERROR: self._app._jobOpt.breakOnError = True elif fc in CMDARG_AGGREGATES: self._parse_aggregate_options() # Help/invalid parameter request else: return self._parse_help_options() # Setup the default measurement path if not provided if not self._app._jobOpt.pathsToMeasure: self._app._jobOpt.pathsToMeasure.append(utils.CURRENT_FOLDER) # Setup the default config name if not provided if not self.configOverrides and self.configCustom is None: self.configCustom = CONFIG_FILE_DEFAULT_NAME except Args.ArgsFinishedException as e: raise utils.InputException(STR_ErrorParsingEnd.format(str(e))) else: log.config(4, vars(self._app))