def _validate_file(self, configEntries):
if not configEntries:
log.config(2, " EMPTY")
else:
try:
self._validate_entries(configEntries)
except Exception as e:
log.stack()
raise utils.ConfigError(
uistrings.STR_ErrorConfigValidate.format(str(e)))
def get_csmodule(self, moduleName, options=[]):
'''
Return the csmodule class with the given name, if it exists
'''
modHash = self._csmod_hash(moduleName, options)
module = self.moduleList.get(modHash)
if not module:
log.config(2, "Loading csmodule: {}".format(moduleName))
module = self._load_csmodule(moduleName, options)
self.moduleList[modHash] = module
return module
def read_file(self, filePath):
'''
Read a Surveyor configuration file and return a list of ConfigEntrys
to store on the configuration stack with this folder location.
'''
try:
log.msg(1, "Config file: {}".format(filePath))
configEntries = self._read_file(filePath, [])
self._validate_file(configEntries)
log.config(2, "Finsihed reading config file: {}".format(filePath))
log.config(3, configEntries)
return configEntries
except Exception as e:
raise utils.ConfigError(
uistrings.STR_ErrorConfigFile.format(filePath, str(e)))
def load_csmodule(self, configEntry):
'''
Callback for ConfigReader to load modules
Module loading is delegated to set of cached modules
Concatonate and default config options from the application with
any options defined in the conifig file.
'''
log.config(3, configEntry.__dict__)
configEntry.module = self._modules.get_csmodule(
configEntry.moduleName,
self._defaultConfigOptions + configEntry.options)
if configEntry.module is None:
raise utils.ConfigError(
uistrings.STR_ErrorFindingModule.format(
configEntry.moduleName))
def add_tags_and_options(self, tagItems):
# Parse the "tag" values into tags and options
for item in tagItems:
if item.startswith(CONFIG_DELIM_OPTION):
opt = item[len(CONFIG_DELIM_OPTION):].split(CONFIG_DELIM_CHAR)
# There are two types of options, with and without values
if len(opt) > 1:
# Everything after OPT tag is value string, join back together
# in case the string had delim chars in it
optionStr = CONFIG_DELIM_CHAR.join(opt[1:])
self.options.append((str(opt[0]), optionStr))
log.config(
2,
"Option Load: {} -> {}".format(str(opt[0]), optionStr))
elif opt:
self.options.append((str(opt[0]), None))
log.config(2, "Option Selected: {}".format(str(opt[0])))
else:
self.tags.append(item)
def get_configuration(self, folder):
'''
Returns two collections:
1) A set of all file filters active for folder
2) A dict by file filter with list of ConfigEntry objects for folder
The active configuration is the contents of the config file
closest to the leaf directory passed in as you look back up the
parent subdirectory tree, ending with the default job config.
'''
self._pop_to_active(folder)
self._push_file(folder)
path, fileFilters, activeConfigItems = self._active_entry()
log.config(
4,
"Config: {} -- {} possible entries".format(path,
len(activeConfigItems)))
return fileFilters, activeConfigItems, path
def _pop_to_active(self, dirToCheck):
'''
Removes config entries back up the folder chain, until we get to the
active one.
'''
configIndex = self._active_entry_index()
# DO NOT EVER pop the first position, as it should be a default file
while configIndex > 0:
# Get active config path and remove file name
configDir, _configItems, _configPath = self._configStack[
configIndex]
configDir = os.path.dirname(configDir)
currentDir = os.path.abspath(dirToCheck)
# Is the config file equal to or "above" current position in path?
# Special case the current folder '.' below because commonprefix
# returns it, while dirname returns blank for empty path
currentDirUnderConfigDir = False
sharedPath = os.path.commonprefix([currentDir, configDir])
if not sharedPath == '':
currentDir = os.path.relpath(currentDir, sharedPath)
configDir = os.path.relpath(configDir, sharedPath)
while True:
if configDir == currentDir:
currentDirUnderConfigDir = True
break
if '.' == currentDir:
break
currentDir = os.path.dirname(currentDir)
if not currentDir:
currentDir = '.'
# If the current config file does not cover the currentDir, pop it
if currentDirUnderConfigDir:
break
else:
log.config(1, "Config POP: {}".format(self.active_path()))
del self._configStack[configIndex]
configIndex -= 1
def __init__(self,
configFileName,
configOverrides,
defaultConfigOptions=[]):
log.config(2, "Creating ConfigStack with {}".format(configFileName))
self._modules = CodeSurveyorModules()
self._reader = configreader.ConfigReader(self.load_csmodule)
self._measureRootDir = ''
# Stack of config files, represented as paths and lists of ConfigEntrys
self._configStack = []
# Cache of config file information
# Key is path name, value is list entries that represent the config file
self._configFileCache = {}
# List of default config option tags passed by the application
self._defaultConfigOptions = defaultConfigOptions
# Either use overrides or try to read config files
if configOverrides:
log.msg(1, "Ignoring config files: {}".format(configOverrides))
self._configName = ''
self._setup_config_overrides(configOverrides)
else:
self._configName = configFileName
# Make sure the config file name does not include a path, as the point is
# to look for a config file in each folder we visit
if not os.path.dirname(self._configName) == '':
raise utils.ConfigError(
uistrings.STR_ErrorConfigFileNameHasPath)
# Load the default config file to use for this job
# First try in the root of the job folder; then in the surveyor folder
if not self._push_file(runtime_dir()):
if not self._push_file(surveyor_dir()):
log.msg(
1, "{} not present in default locations".format(
self._configName))
def config_items_for_file(configEntrys, fileName):
'''
Return a list of config items that match the given fileName
'''
neededConfigs = []
# Don't know how many config entrys could be associated with a given
# file extension (files could match more than one config file filter),
# so we check against every config
# If there are custom RE config filters, we include them no matter what,
# since we can't just match them against the file extension
for configFilter in list(configEntrys.keys()):
if fileext.file_ext_match(fileName, configFilter):
for config in configEntrys[configFilter]:
neededConfigs.append(config)
# Make this a sorted list to ensure repeatble results in terms of
# order files are processed. This doesn't normally matter, but can
# be convienent and allows for single-threaded repeatability that
# allows for comparison against test oracle
neededConfigs.sort(key=lambda configSort: str(configSort))
log.config(3, neededConfigs)
return neededConfigs
def _push_file(self, dirName):
'''
Returns true if a config file was found in dirName and pushed on stack
'''
success = False
configFilePath = os.path.abspath(
os.path.join(dirName, self._configName))
if not configFilePath in self._configFileCache:
if os.path.isfile(configFilePath):
self._configFileCache[configFilePath] = self._reader.read_file(
configFilePath)
if configFilePath in self._configFileCache:
self._push_entries(configFilePath,
self._configFileCache[configFilePath])
log.config(
1, "Config PUSH {}: {}".format(
len(self._configFileCache[configFilePath]),
configFilePath))
if len(self._configFileCache[configFilePath]) == 0:
log.config(1, "EMPTY CONFIG: {}".format(configFilePath))
success = True
return success
def _parse_file(self, configFile, configEntries):
'''
Parse config file lines
'''
configEntry = configentry.ConfigEntry(
'_ _ _ _') # Init to empty object to prevent PyChecker warnings
constants = {}
readingVerbs = False
verbEndMarker = None
for whiteSpaceRawline in configFile:
log.config(3, "Config line: {}".format(whiteSpaceRawline))
rawLine = whiteSpaceRawline.strip()
line = rawLine
# Skip comments, blank lines
if self.comment.match(line) or self.blankLine.match(line):
log.config(4, "comment/blank")
continue
# Skip ignore blocks (or go to end of file if no closing block)
if self.ignoreStart.match(line):
log.config(4, "ignoreBlock")
try:
while not self.ignoreStop.match(line):
line = next(configFile)
log.config(4, "Config ignore: {}".format(line))
except Exception:
log.config(4,
"Exception while seeking end of ignore block")
pass
continue
# Includes
# Attempt to load the requested file and add it's entries
# to our entries, in the form INCLUDE:path: tagInfo
includeMatch = self.include.match(line)
if includeMatch:
includePath = includeMatch.group(1)
newTags = includeMatch.group(2)
if not os.path.isabs(includePath):
includePath = os.path.join(
os.path.dirname(configFile.name), includePath)
log.config(1, "Include: {}".format(includePath))
newEntries = self._read_file(includePath, [])
existingFileFilterStrings = [
entry.fileFilter for entry in configEntries
]
for entry in newEntries:
# If an entry has already been defined with the SAME FILE FILTER STRING,
# the INCLUDED ENTRY WILL BE IGNORED
if entry.fileFilter in existingFileFilterStrings:
continue
# If 'tagInfo' is provided, it will be added to ALL entries of the file
# that was included
# RELOAD THE MODULE in case new options need processed
if newTags:
entry.add_tags_and_options(newTags.split())
self._load_csmodule(entry)
configEntries.append(entry)
continue
# If line closes out a verb entry store the config entry
if readingVerbs and re.match(verbEndMarker, line):
log.config(4, "verbend: {}".format(line))
readingVerbs = False
configEntries.append(configEntry)
continue
# Handle continued lines
fullLine = ""
while True:
contLineMatch = self.continuedLine.match(line)
if contLineMatch:
fullLine += contLineMatch.group(CONT_LINE_START)
line = next(configFile).strip()
log.config(3, "FullLine: {}".format(line))
else:
fullLine += line
break
assert fullLine
line = fullLine
# If line defines a normal constant, store asis
constantMatch = self.constant.match(line)
if constantMatch:
# Assign cosntant, strip spaces to support config lines that are space-delimited
constants[constantMatch.group(1)] = constantMatch.group(2)
log.config(2, "Constant: {}".format(constantMatch.group(2)))
continue
# If line defines a no blanks constant, strip spaces and store
constantMatch = self.constant_noblanks.match(line)
if constantMatch:
constants[constantMatch.group(1)] = constantMatch.group(
2).replace(' ', '')
log.config(
2, "Noblank constant: {}".format(constantMatch.group(2)))
continue
# Replace any constants used in the line
line = self._replace_constants(line, constants)
log.config(4, "fullline: {}".format(line))
# Strip any inline comments
line = line.split(' #')[0]
# If the line is a parameter (e.g., search terms), delegate to module
# to get processed parameters and store for later usage
# Keep the unprocessed raw version around for consistency checking
if readingVerbs:
configEntry.paramsRaw.append(rawLine)
try:
paramTuple = configEntry.module.add_param(line, rawLine)
configEntry.paramsProcessed.append(paramTuple)
log.config(
2, "LoadedParam: {} => {}".format(
configEntry.module.__class__.__name__, paramTuple))
except Exception as e:
log.stack()
raise utils.ConfigError(
uistrings.STR_ErrorConfigParam.format(
str(configEntry), rawLine, str(e)))
# Otherwise assume we're at the start of a config entry definition,
else:
try:
# Load and validate the config line and its module
configEntry = configentry.ConfigEntry(
line, self._extraLineContent, configFile.name)
self._load_csmodule(configEntry)
self._validate_line(configEntry)
# Check to see if there are parameter lines to read
verbEndMarker = configEntry.module.verb_end_marker(
configEntry.verb)
if verbEndMarker is not None:
readingVerbs = True
# Add the completed config entry to our list
if not readingVerbs:
configEntries.append(configEntry)
except Exception as e:
log.stack()
raise utils.ConfigError(
uistrings.STR_ErrorConfigEntry.format(rawLine, str(e)))
# Loop to next line
return configEntries
def _validate_entries(self, configEntries):
'''
Are all config file entries consistent with each other, to avoid silent
double counting? Throws an error exception if not.
'''
log.config(2, "Checking for duplicate config entries")
# Create list of all possible measure/file combos
# Ask the module to match each measure, to catch wildcard overlap
fileFilters = []
possibleMeasures = []
for entry in configEntries:
for fileFilter in entry.fileFilters:
fileFilters.append(fileFilter)
possibleMeasures.append(
(fileFilter, entry.measureFilter, entry.moduleName,
entry.verb, entry.tags, entry.paramsRaw))
log.config(4, fileFilters)
log.config(4, possibleMeasures)
# Check that no file type would have a measure be double counted
# If a problem, throw an exception based on the first problem item
if len(fileFilters) > len(set(fileFilters)):
while possibleMeasures:
possibleMeasureTuple = possibleMeasures.pop()
log.config(2,
"possibleMeasure: {}".format(possibleMeasureTuple))
(fileFilter, measureFilter, modName, verb, tags,
extraParams) = possibleMeasureTuple
# Don't attempt the do conflict resolution on regex files extensions,
# both because it doesn't make sense
if fileFilter.startswith(fileext.CUSTOM_FILE_REGEX):
continue
# Shallow warning check for double counting by creatubg a list of entries
# based on matching verb and file type
warningList = [
(ff, mf, mn, v, t, ep)
for ff, mf, mn, v, t, ep in possibleMeasures
if v == verb and fileext.file_ext_match(ff, fileFilter)
]
if warningList:
log.config(
1, "WARNING - Possible double-count: {}".format(
str(warningList)))
# For the deep check look at tag values and measure filter
dupeList = [
(v, modName, mn, mf, fileFilter, ff, t, tags, ep,
extraParams) for ff, mf, mn, v, t, ep in warningList
if len(t) == len(tags)
and len(t) == len(set(t) & set(tags))
and entry.module.match_measure(mf, measureFilter)
]
if dupeList:
log.msg(
1,
"ERROR - Double-count: {}".format(str(dupeList)))
dupe = dupeList[0]
raise utils.ConfigError(
uistrings.STR_ErrorConfigDupeMeasures.format(
dupe[0], dupe[1], dupe[2], dupe[3], dupe[4],
dupe[5], dupe[6], dupe[7], dupe[8], dupe[9]))
def parse_args(self):
'''
Do simple command line parsing and set the internal state of our
Surveyor class based on the arguments.
For any syntax we don't recognize or help is requested, return help text.
Otherwise return None which indicates success.
'''
try:
while not self.args.finished():
self.args.move_next()
# Disambiguation case for measurePath/fileFilter
# A '-' may be used to replace optional arg with path/filter
if self.args.is_cmd() and len(self.args.get_current()) == 1:
if self.args.is_param_next():
self.args.move_next()
self._parse_measurement_path()
continue
# Assume non-Arg is a measurePath/fileFilter definition
elif not self.args.is_cmd():
self._parse_measurement_path()
continue
# Our processing is based on matching first character
fc = self.args.get_current()[1].lower()
# Debug and profiling support
if fc in CMDARG_DEBUG:
self._parse_debug_options()
log.msg(2, "Args: {}".format(str(self.args)))
elif fc in CMDARG_PROFILE:
self._app._profiling = True
self._app._profileCalls = self._get_next_int(optional=True, default=self._app._profileCalls)
self._app._profileCalledBy = self._get_next_int(optional=True, default=self._app._profileCalledBy)
self._app._profileCalled = self._get_next_int(optional=True, default=self._app._profileCalled)
self._app._profileThreadFilter = self._get_next_str(optional=True, default=self._app._profileThreadFilter)
self._app._profileNameFilter = self._get_next_str(optional=True, default=self._app._profileNameFilter)
# Config file settings
elif fc in CMDARG_CONFIG_CUSTOM:
self._parse_config_options()
# Delta path
elif fc in CMDARG_DELTA:
self._parse_delta_options()
# Duplicate processing
# Can have an optional integer or string after this option
elif fc in CMDARG_DUPE_PROCESSING:
self._app._dupeTracking = True
self._metaDataOptions['DUPE'] = None
dupeParam = self._get_next_param(optional=True)
try:
dupeParam = int(dupeParam)
except Exception as e:
pass
self._app._dupeThreshold = dupeParam
# Scan and skip options
elif fc in CMDARG_SCAN_ALL:
self._parse_scan_options()
elif fc in CMDARG_SKIP:
self._parse_skip_options()
elif fc in CMDARG_INCLUDE_ONLY:
self._app._jobOpt.includeFolders.extend(self._get_next_param().split(CMDLINE_SEPARATOR))
# Output
elif fc in CMDARG_METADATA == fc:
self._parse_metadata_options()
elif fc in CMDARG_OUTPUT_FILTER:
self._measureFilter = self._get_next_str()
elif fc in CMDARG_OUTPUT_TYPE == fc:
self._app._outType = self._get_next_str()
elif fc in CMDARG_OUTPUT_FILE == fc:
self._parse_output_file()
elif fc in CMDARG_SUMMARY_ONLY == fc:
self._app._summaryOnly = True
elif fc in CMDARG_DETAILED == fc:
self._app._detailed = True
self._app._detailedPrintSummaryMax = self._get_next_int(
optional=True, default=self._app._detailedPrintSummaryMax)
elif fc in CMDARG_PROGRESS == fc:
self._app._progress = True
self._app._printMaxWidth = self._get_next_int(
optional=True, default=self._app._printMaxWidth)
elif fc in CMDARG_QUIET == fc:
self._app._quiet = True
# Other options
elif fc in CMDARG_NUM_WORKERS:
self._app._jobOpt.numWorkers = self._get_next_int(validRange=range(1,MAX_WORKERS))
elif fc in CMDARG_RECURSION:
self._app._jobOpt.recursive = False
elif fc in CMDARG_BREAK_ERROR:
self._app._jobOpt.breakOnError = True
elif fc in CMDARG_AGGREGATES:
self._parse_aggregate_options()
# Help/invalid parameter request
else:
return self._parse_help_options()
# Setup the default measurement path if not provided
if not self._app._jobOpt.pathsToMeasure:
self._app._jobOpt.pathsToMeasure.append(utils.CURRENT_FOLDER)
# Setup the default config name if not provided
if not self.configOverrides and self.configCustom is None:
self.configCustom = CONFIG_FILE_DEFAULT_NAME
except Args.ArgsFinishedException as e:
raise utils.InputException(STR_ErrorParsingEnd.format(str(e)))
else:
log.config(4, vars(self._app))
