def __init__(self, configFileName, configOverrides, defaultConfigOptions=[]):
trace.config(2, "Creating ConfigStack with {0}".format(configFileName))
self._modules = CodeSurveyorModules()
self._reader = configreader.ConfigReader(self.load_csmodule)
self._measureRootDir = ''
# Stack of config files, represented as paths and lists of ConfigEntrys
self._configStack = []
# Cache of config file information
# Key is path name, value is list entries that represent the config file
self._configFileCache = {}
# List of default config option tags passed by the application
self._defaultConfigOptions = defaultConfigOptions
# We either use overrides or try to read config files
if configOverrides:
trace.msg(1, "Ignoring config files: {0}".format(configOverrides))
self._configName = ''
self._setup_config_overrides(configOverrides)
else:
self._configName = configFileName
# Make sure the config file name does not include a path, as the point is
# to look for a config file in each folder we visit
if not os.path.dirname(self._configName) == '':
raise utils.ConfigError(uistrings.STR_ErrorConfigFileNameHasPath)
# Load the default config file to use for this job
# First try in the root of the job folder; then in the surveyor folder
if not self._push_file(utils.runtime_dir()):
if not self._push_file(utils.surveyor_dir()):
trace.msg(1, "{0} not present in default locations".format(
self._configName))
def __init__(self, configOptions):
# Modules overrides to tell config validation what they can measure
self.measures = []
self.verbs = []
self.verbEnds = {}
# We store this each measure call in case derived class wants to use
self._currentPath = None
# Delegate that subclasses can call to add config options
self._configOptionDict = {}
self._cs_init_config_options()
# Process any config options
for optName, optValue in configOptions:
trace.config(2, "ConfigOpt: {0}:{1}".format(optName, optValue))
try:
configCode, _configHelp = self._configOptionDict[optName]
except KeyError, e:
raise utils.CsModuleException("Invalid Config Option: {0}".format(str(e)))
trace.config(3, "ConfigCode: {0}".format(configCode))
try:
exec(configCode)
except Exception, e:
trace.traceback()
raise utils.CsModuleException("Error executing Config Option: {0}".format(str(e)))
def _validate_entries(self, configEntries):
'''
Are all config file entries consistent with each other, to avoid silent
double counting? Throws an error exception if not.
'''
trace.config(2, "Checking for duplicate config entries")
# Create list of all possible measure/file combos
# We ask the module to match each measure, to catch wildcard overlap
fileFilters = []
possibleMeasures = []
for entry in configEntries:
for fileFilter in entry.fileFilters:
fileFilters.append(fileFilter)
possibleMeasures.append((fileFilter, entry.measureFilter,
entry.moduleName, entry.verb, entry.tags, entry.paramsRaw))
trace.config(4, fileFilters)
trace.config(4, possibleMeasures)
# Check that no file type would have a measure be double counted
# If we have a problem, throw an exception based on the first problem item
if len(fileFilters) > len(set(fileFilters)):
while possibleMeasures:
possibleMeasureTuple = possibleMeasures.pop()
trace.config(2, "possibleMeasure: {0}".format(possibleMeasureTuple))
(fileFilter, measureFilter, modName, verb, tags, extraParams) = possibleMeasureTuple
# We don't attempt the do conflict resolution on regex files extensions,
# both because it doesn't make sense
if fileFilter.startswith(fileext.CUSTOM_FILE_REGEX):
continue
# Shallow warning check for double counting by creatubg a list of entries
# based on matching verb and file type
warningList = [
(ff, mf, mn, v, t, ep)
for ff, mf, mn, v, t, ep in possibleMeasures if
v == verb and
fileext.file_ext_match(ff, fileFilter) ]
if warningList:
trace.config(1, "WARNING - Possible double-count: {0}".format(str(warningList)))
# For the deep check look at tag values and measure filter
dupeList = [
(v, modName, mn, mf, fileFilter, ff, t, tags, ep, extraParams)
for ff, mf, mn, v, t, ep in warningList if
len(t) == len(tags) and
len(t) == len(set(t) & set(tags)) and
entry.module.match_measure(mf, measureFilter) ]
if dupeList:
trace.msg(1, "ERROR - Double-count: {0}".format(str(dupeList)))
dupe = dupeList[0]
raise utils.ConfigError(uistrings.STR_ErrorConfigDupeMeasures.format(
dupe[0],
dupe[1], dupe[2],
dupe[3],
dupe[4], dupe[5],
dupe[6], dupe[7],
dupe[8], dupe[9]))
def _validate_file(self, configEntries):
if not configEntries:
trace.config(2, " EMPTY")
else:
try:
self._validate_entries(configEntries)
except Exception, e:
trace.traceback()
raise utils.ConfigError(uistrings.STR_ErrorConfigValidate.format(str(e)))
def get_csmodule(self, csmoduleName, options=[]):
'''
Return the csmodule class with the given name, if it exists
'''
csmodule = None
try:
csmodule = self.moduleList[self._csmod_hash(csmoduleName, options)]
except KeyError:
trace.config(2, "Loading csmodule: {0}".format(csmoduleName))
csmodule = self._load_csmodule(csmoduleName, options)
if csmodule is not None:
self.moduleList[self._csmod_hash(csmoduleName, options)] = csmodule
return csmodule
def read_file(self, filePath):
'''
Read a Surveyor configuration file and return a list of ConfigEntrys
to be stored on the configuration stack with this folder location.
'''
try:
trace.msg(1, "Config file: {0}".format(filePath))
configEntries = self._read_file(filePath, [])
self._validate_file(configEntries)
trace.config(2, "Finsihed reading config file: {0}".format(filePath))
trace.config(3, configEntries)
return configEntries
except Exception, e:
raise utils.ConfigError(uistrings.STR_ErrorConfigFile.format(filePath, str(e)))
def load_csmodule(self, configEntry):
'''
Callback for ConfigReader to load modules
Module loading is delegated to our set of cached modules
We concatonate and default config options from the application with
any options defined in the conifig file.
'''
trace.config(3, configEntry.__dict__)
configEntry.module = self._modules.get_csmodule(
configEntry.moduleName,
self._defaultConfigOptions + configEntry.options)
if configEntry.module is None:
raise utils.ConfigError(uistrings.STR_ErrorFindingModule.format(
configEntry.moduleName))
def load_csmodule(self, configEntry):
'''
Callback for ConfigReader to load modules
Module loading is delegated to our set of cached modules
We concatonate and default config options from the application with
any options defined in the conifig file.
'''
trace.config(3, configEntry.__dict__)
configEntry.module = self._modules.get_csmodule(
configEntry.moduleName,
self._defaultConfigOptions + configEntry.options)
if configEntry.module is None:
raise utils.ConfigError(
uistrings.STR_ErrorFindingModule.format(
configEntry.moduleName))
def add_tags_and_options(self, tagItems):
# Parse the "tag" values into tags and options
for item in tagItems:
if item.startswith(CONFIG_DELIM_OPTION):
opt = item[len(CONFIG_DELIM_OPTION):].split(CONFIG_DELIM_CHAR)
# There are two types of options, with and without values
if len(opt) > 1:
# Everything after OPT tag is value string, we joing back together
# in case the string had delim chars in it
optionStr = CONFIG_DELIM_CHAR.join(opt[1:])
self.options.append((str(opt[0]), optionStr))
trace.config(2, "Option Load: {0} -> {1}".format(str(opt[0]), optionStr))
elif opt:
self.options.append((str(opt[0]), None))
trace.config(2, "Option Selected: {0}".format(str(opt[0])))
else:
self.tags.append(item)
def get_configuration(self, folder):
'''
Returns two collections:
1) A set of all file filters active for folder
2) A dict by file filter with list of ConfigEntry objects for folder
The active configuration is the contents of the config file
closest to the leaf directory passed in as you look back up the
parent subdirectory tree, ending with the default job config.
'''
self._pop_to_active(folder)
self._push_file(folder)
path, fileFilters, activeConfigItems = self._active_entry()
trace.config(4, "Config: {0} -- {1} possible entries".format(path, len(activeConfigItems)))
return fileFilters, activeConfigItems, path
def match_measure(self, measureName, measureFilters):
'''
Used to both validate config and to filter results
Measure filters allow using * to match the end of the string.
i.e.: size.* would match size.totallines and size.blanklines
We also pass any measure that does not have a '.', which indicates
it is system measure we always want to output
'''
if '*' in measureFilters:
return True
if '.' not in measureName:
return True
for measureFilter in measureFilters:
match = compare_filters(measureName, measureFilter)
trace.config(4, "Compared {0} to {1}: {2}".format(measureName, measureFilter, match))
if match:
return True
return False
def _push_file(self, dirName):
'''
Returns true if a config file was found in dirName and pushed on stack
'''
success = False
configFilePath = os.path.abspath(os.path.join(dirName, self._configName))
if not configFilePath in self._configFileCache:
if os.path.isfile(configFilePath):
self._configFileCache[configFilePath] = self._reader.read_file(configFilePath)
if configFilePath in self._configFileCache:
self._push_entries(configFilePath, self._configFileCache[configFilePath])
trace.config(1, "Config PUSH {0}: {1}".format(
len(self._configFileCache[configFilePath]), configFilePath))
if len(self._configFileCache[configFilePath]) == 0:
trace.config(1, "EMPTY CONFIG: {0}".format(configFilePath))
success = True;
return success
def get_configuration(self, folder):
'''
Returns two collections:
1) A set of all file filters active for folder
2) A dict by file filter with list of ConfigEntry objects for folder
The active configuration is the contents of the config file
closest to the leaf directory passed in as you look back up the
parent subdirectory tree, ending with the default job config.
'''
self._pop_to_active(folder)
self._push_file(folder)
path, fileFilters, activeConfigItems = self._active_entry()
trace.config(
4, "Config: {0} -- {1} possible entries".format(
path, len(activeConfigItems)))
return fileFilters, activeConfigItems, path
def _pop_to_active(self, dirToCheck):
'''
Removes config entries back up the folder chain, until we get to the
active one.
'''
configIndex = self._active_entry_index()
# DO NOT EVER pop the first position, as it should be a default file
while configIndex > 0:
# Get active config path and remove file name
configDir, _configItems, _configPath = self._configStack[
configIndex]
configDir = os.path.dirname(configDir)
currentDir = os.path.abspath(dirToCheck)
# Is the config file equal to or "above" current position in path?
# Special case the current folder '.' below because commonprefix
# returns it, while dirname returns blank for empty path
currentDirUnderConfigDir = False
sharedPath = os.path.commonprefix([currentDir, configDir])
if not sharedPath == '':
currentDir = os.path.relpath(currentDir, sharedPath)
configDir = os.path.relpath(configDir, sharedPath)
while True:
if configDir == currentDir:
currentDirUnderConfigDir = True
break
if '.' == currentDir:
break
currentDir = os.path.dirname(currentDir)
if not currentDir:
currentDir = '.'
# If the current config file does not cover the currentDir, pop it
if currentDirUnderConfigDir:
break
else:
trace.config(1, "Config POP: {0}".format(self.active_path()))
del self._configStack[configIndex]
configIndex -= 1
def _pop_to_active(self, dirToCheck):
'''
Removes config entries back up the folder chain, until we get to the
active one.
'''
configIndex = self._active_entry_index()
# DO NOT EVER pop the first position, as it should be a default file
while configIndex > 0:
# Get active config path and remove file name
configDir, _configItems, _configPath = self._configStack[configIndex]
configDir = os.path.dirname(configDir)
currentDir = os.path.abspath(dirToCheck)
# Is the config file equal to or "above" current position in path?
# Special case the current folder '.' below because commonprefix
# returns it, while dirname returns blank for empty path
currentDirUnderConfigDir = False
sharedPath = os.path.commonprefix([currentDir, configDir])
if not sharedPath == '':
currentDir = os.path.relpath(currentDir, sharedPath)
configDir = os.path.relpath(configDir, sharedPath)
while True:
if configDir == currentDir:
currentDirUnderConfigDir = True
break;
if '.' == currentDir:
break
currentDir = os.path.dirname(currentDir)
if not currentDir:
currentDir = '.'
# If the current config file does not cover the currentDir, pop it
if currentDirUnderConfigDir:
break
else:
trace.config(1, "Config POP: {0}".format(self.active_path()))
del self._configStack[configIndex]
configIndex -= 1
def __init__(self,
configFileName,
configOverrides,
defaultConfigOptions=[]):
trace.config(2, "Creating ConfigStack with {0}".format(configFileName))
self._modules = CodeSurveyorModules()
self._reader = configreader.ConfigReader(self.load_csmodule)
self._measureRootDir = ''
# Stack of config files, represented as paths and lists of ConfigEntrys
self._configStack = []
# Cache of config file information
# Key is path name, value is list entries that represent the config file
self._configFileCache = {}
# List of default config option tags passed by the application
self._defaultConfigOptions = defaultConfigOptions
# We either use overrides or try to read config files
if configOverrides:
trace.msg(1, "Ignoring config files: {0}".format(configOverrides))
self._configName = ''
self._setup_config_overrides(configOverrides)
else:
self._configName = configFileName
# Make sure the config file name does not include a path, as the point is
# to look for a config file in each folder we visit
if not os.path.dirname(self._configName) == '':
raise utils.ConfigError(
uistrings.STR_ErrorConfigFileNameHasPath)
# Load the default config file to use for this job
# First try in the root of the job folder; then in the surveyor folder
if not self._push_file(utils.runtime_dir()):
if not self._push_file(utils.surveyor_dir()):
trace.msg(
1, "{0} not present in default locations".format(
self._configName))
def config_items_for_file(configEntrys, fileName):
'''
Return a list of config items that match the given fileName
'''
neededConfigs = []
# We don't know how many config entrys could be associated with a given
# file extension (files could match more than one config file filter),
# so we check against every config
# If there are custom RE config filters, we include them no matter what,
# since we can't just match them against the file extension
for configFilter in configEntrys.keys():
if fileext.file_ext_match(fileName, configFilter):
for config in configEntrys[configFilter]:
neededConfigs.append(config)
# Make this a sorted list to ensure repeatble results in terms of
# order files are processed. This doesn't normally matter, but can
# be convienent and allows for single-threaded repeatability that
# allows for comparison against test oracle
neededConfigs.sort(key=lambda configSort: str(configSort))
trace.config(3, neededConfigs)
return neededConfigs
def config_items_for_file(configEntrys, fileName):
'''
Return a list of config items that match the given fileName
'''
neededConfigs = []
# We don't know how many config entrys could be associated with a given
# file extension (files could match more than one config file filter),
# so we check against every config
# If there are custom RE config filters, we include them no matter what,
# since we can't just match them against the file extension
for configFilter in configEntrys.keys():
if fileext.file_ext_match(fileName, configFilter):
for config in configEntrys[configFilter]:
neededConfigs.append(config)
# Make this a sorted list to ensure repeatble results in terms of
# order files are processed. This doesn't normally matter, but can
# be convienent and allows for single-threaded repeatability that
# allows for comparison against test oracle
neededConfigs.sort(key=lambda configSort: str(configSort))
trace.config(3, neededConfigs)
return neededConfigs
def _push_file(self, dirName):
'''
Returns true if a config file was found in dirName and pushed on stack
'''
success = False
configFilePath = os.path.abspath(
os.path.join(dirName, self._configName))
if not configFilePath in self._configFileCache:
if os.path.isfile(configFilePath):
self._configFileCache[configFilePath] = self._reader.read_file(
configFilePath)
if configFilePath in self._configFileCache:
self._push_entries(configFilePath,
self._configFileCache[configFilePath])
trace.config(
1, "Config PUSH {0}: {1}".format(
len(self._configFileCache[configFilePath]),
configFilePath))
if len(self._configFileCache[configFilePath]) == 0:
trace.config(1, "EMPTY CONFIG: {0}".format(configFilePath))
success = True
return success
else:
return self._parse_help_options()
# Setup the default measurement path if not provided
if not self._app._jobOpt.pathsToMeasure:
self._app._jobOpt.pathsToMeasure.append(utils.CURRENT_FOLDER)
# Setup the default config name if not provided
if not self.configOverrides and self.configCustom is None:
self.configCustom = CONFIG_FILE_DEFAULT_NAME
except Args.ArgsFinishedException, e:
raise utils.InputException(STR_ErrorParsingEnd.format(str(e)))
else:
trace.config(4, vars(self._app))
def config_option_list(self):
'''
Maps application modifiable config options to the name, value list format
used by csmodules to process config file options
'''
configOptions = []
configOptions.append(('METADATA', self._metaDataOptions))
if self._measureFilter is not None:
if self._measureFilter == CMDARG_OUTPUT_FILTER_METADATA:
configOptions.append(('METADATA_ONLY', None))
configOptions.append(('MEASURE_FILTER', '*'))
else:
configOptions.append(('MEASURE_FILTER', self._measureFilter))
# Help/invalid parameter request
else:
return self._parse_help_options()
# Setup the default measurement path if not provided
if not self._app._jobOpt.pathsToMeasure:
self._app._jobOpt.pathsToMeasure.append(utils.CURRENT_FOLDER)
# Setup the default config name if not provided
if not self.configOverrides and self.configCustom is None:
self.configCustom = CONFIG_FILE_DEFAULT_NAME
except Args.ArgsFinishedException, e:
raise utils.InputException(STR_ErrorParsingEnd.format(str(e)))
else:
trace.config(4, vars(self._app))
def config_option_list(self):
'''
Maps application modifiable config options to the name, value list format
used by csmodules to process config file options
'''
configOptions = []
configOptions.append(('METADATA', self._metaDataOptions))
if self._measureFilter is not None:
if self._measureFilter == CMDARG_OUTPUT_FILTER_METADATA:
configOptions.append(('METADATA_ONLY', None))
configOptions.append(('MEASURE_FILTER', '*'))
else:
configOptions.append(('MEASURE_FILTER', self._measureFilter))
if self.ignoreSize > 0:
def _parse_file(self, configFile, configEntries):
'''
Parse config file lines
'''
configEntry = configentry.ConfigEntry('_ _ _ _') # Init to empty object to prevent PyChecker warnings
constants = {}
readingVerbs = False
verbEndMarker = None
for whiteSpaceRawline in configFile:
trace.config(3, "Config line: {0}".format(whiteSpaceRawline))
rawLine = whiteSpaceRawline.strip()
line = rawLine
# Skip comments, blank lines
if self.comment.match(line) or self.blankLine.match(line):
trace.config(4, "comment/blank")
continue
# Skip ignore blocks (or go to end of file if no closing block)
if self.ignoreStart.match(line):
trace.config(4, "ignoreBlock")
try:
while not self.ignoreStop.match(line):
line = configFile.next()
trace.config(4, "Config ignore: {0}".format(line))
except Exception:
trace.config(4, "Exception while seeking end of ignore block")
pass
continue
# Includes
# Attempt to load the requested file and add it's entries
# to our entries, in the form INCLUDE:path: tagInfo
includeMatch = self.include.match(line)
if includeMatch:
includePath = includeMatch.group(1)
newTags = includeMatch.group(2)
if not os.path.isabs(includePath):
includePath = os.path.join(os.path.dirname(configFile.name), includePath)
trace.config(1, "Include: {0}".format(includePath))
newEntries = self._read_file(includePath, [])
existingFileFilterStrings = [entry.fileFilter for entry in configEntries]
for entry in newEntries:
# If an entry has already been defined with the SAME FILE FILTER STRING,
# the INCLUDED ENTRY WILL BE IGNORED
if entry.fileFilter in existingFileFilterStrings:
continue
# If 'tagInfo' is provided, it will be added to ALL entries of the file
# that was included
# We RELOAD THE MODULE in case new options need processed
if newTags:
entry.add_tags_and_options(newTags.split())
self._load_csmodule(entry)
configEntries.append(entry)
continue
# If line closes out a verb entry store the config entry
if readingVerbs and re.match(verbEndMarker, line):
trace.config(4, "verbend: {0}".format(line))
readingVerbs = False
configEntries.append(configEntry)
continue
# Handle continued lines
fullLine = ""
while True:
contLineMatch = self.continuedLine.match(line)
if contLineMatch:
fullLine += contLineMatch.group(CONT_LINE_START)
line = configFile.next().strip()
trace.config(3, "FullLine: {0}".format(line))
else:
fullLine += line
break
assert fullLine
line = fullLine
# If line defines a normal constant, store asis
constantMatch = self.constant.match(line)
if constantMatch:
# Assign cosntant, strip spaces to support config lines that are space-delimited
constants[constantMatch.group(1)] = constantMatch.group(2)
trace.config(2, "Constant: {0}".format(constantMatch.group(2)))
continue
# If line defines a no blanks constant, strip spaces and store
constantMatch = self.constant_noblanks.match(line)
if constantMatch:
constants[constantMatch.group(1)] = constantMatch.group(2).replace(' ', '')
trace.config(2, "Noblank constant: {0}".format(constantMatch.group(2)))
continue
# Replace any constants used in the line
line = self._replace_constants(line, constants)
trace.config(4, "fullline: {0}".format(line))
# Strip any inline comments
line = line.split(' #')[0]
# If the line is a parameter (e.g., search terms), delegate to module
# to get processed parameters and store for later usage
# We keep the unprocessed raw version around for consistency checking
if readingVerbs:
configEntry.paramsRaw.append(rawLine)
try:
paramTuple = configEntry.module.add_param(line, rawLine)
configEntry.paramsProcessed.append(paramTuple)
trace.config(2, "LoadedParam: {0} => {1}".format(
configEntry.module.__class__.__name__, paramTuple))
except Exception, e:
trace.traceback()
raise utils.ConfigError(uistrings.STR_ErrorConfigParam.format(
str(configEntry), rawLine, str(e)))
# Otherwise assume we're at the start of a config entry definition,
else:
try:
# Load and validate the config line and its module
configEntry = configentry.ConfigEntry(line, self._extraLineContent, configFile.name)
self._load_csmodule(configEntry)
self._validate_line(configEntry)
# Check to see if there are parameter lines to read
verbEndMarker = configEntry.module.verb_end_marker(configEntry.verb)
if verbEndMarker is not None:
readingVerbs = True
# Add the completed config entry to our list
if not readingVerbs:
configEntries.append(configEntry)
except Exception, e:
trace.traceback()
raise utils.ConfigError(uistrings.STR_ErrorConfigEntry.format(
rawLine, str(e)))