def _load_csmodule(self, modName, options):
'''
Import a module given its name, and return the object.
If there are any problems finding or loading the module we return None.
If the module has python errors in it we treat as catastrophic
failure and allow caller to handle.
'''
csmoduleClassInstance = None
try:
# Load the module called modName, and then get class inside the
# module with the same name
moduleFile = __import__(self.PACKAGE_PREFIX + modName)
module = getattr(moduleFile, modName)
moduleClass = getattr(module, modName)
# Instantiate the module
csmoduleClassInstance = moduleClass(options)
# Make sure required methods are in the class; fatal error if not
for method in self.RequiredMethods:
getattr(csmoduleClassInstance, method)
except (ImportError, AttributeError):
trace.traceback()
raise utils.SurveyorException(uistrings.STR_ErrorLoadingModule.format(modName))
return csmoduleClassInstance
def __init__(self, configOptions):
# Modules overrides to tell config validation what they can measure
self.measures = []
self.verbs = []
self.verbEnds = {}
# We store this each measure call in case derived class wants to use
self._currentPath = None
# Delegate that subclasses can call to add config options
self._configOptionDict = {}
self._cs_init_config_options()
# Process any config options
for optName, optValue in configOptions:
trace.config(2, "ConfigOpt: {0}:{1}".format(optName, optValue))
try:
configCode, _configHelp = self._configOptionDict[optName]
except KeyError, e:
raise utils.CsModuleException("Invalid Config Option: {0}".format(str(e)))
trace.config(3, "ConfigCode: {0}".format(configCode))
try:
exec(configCode)
except Exception, e:
trace.traceback()
raise utils.CsModuleException("Error executing Config Option: {0}".format(str(e)))
def _survey(self, fileObject, configEntry, measurements, _analysis):
if not fileObject:
return
self.totalLines = 0
self.blankLines = 0
self.contentLines = 0
try:
measureMethod = self.filetypeMeasures.get(self._currentPath.fileExt, None)
if measureMethod is not None:
measureMethod(fileObject, measurements)
# Otherwise if it is a delta list comparison
elif isinstance(fileObject, list):
self._measure_text(fileObject, measurements)
# Otherwise if it is a text File
elif filetype.is_text_file(fileObject):
self._measure_text(fileObject, measurements)
# TBD -- what to with binary files without handlers
# Pack up our measurements
measurements[ self.LINES_TOTAL ] = self.totalLines
measurements[ self.LINES_BLANK ] = self.blankLines
measurements[ self.LINES_CONTENT ] = self.contentLines
return True
except Exception, e:
trace.traceback()
raise utils.FileMeasureError(
"Problem processing file {0} with module: {1}\n\t{1}".format(
self._currentPath.filePath, self.__class__.__name__, str(e)))
def _survey(self, fileObject, configEntry, measurements, _analysis):
if not fileObject:
return
self.totalLines = 0
self.blankLines = 0
self.contentLines = 0
try:
measureMethod = self.filetypeMeasures.get(self._currentPath.fileExt, None)
if measureMethod is not None:
measureMethod(fileObject, measurements)
# Otherwise if it is a delta list comparison
elif isinstance(fileObject, list):
self._measure_text(fileObject, measurements)
# Otherwise if it is a text File
elif filetype.is_text_file(fileObject):
self._measure_text(fileObject, measurements)
# TBD -- what to with binary files without handlers
# Pack up our measurements
measurements[ self.LINES_TOTAL ] = self.totalLines
measurements[ self.LINES_BLANK ] = self.blankLines
measurements[ self.LINES_CONTENT ] = self.contentLines
return True
except Exception, e:
trace.traceback()
raise utils.FileMeasureError(
"Problem processing file {0} with module: {1}\n\t{1}".format(
self._currentPath.filePath, self.__class__.__name__, str(e)))
def _validate_file(self, configEntries):
if not configEntries:
trace.config(2, " EMPTY")
else:
try:
self._validate_entries(configEntries)
except Exception, e:
trace.traceback()
raise utils.ConfigError(uistrings.STR_ErrorConfigValidate.format(str(e)))
def _parse_command_line(self, cmdArgs):
utils.init_surveyor_dir(cmdArgs[0])
self._args = cmdlineargs.SurveyorCmdLineArgs(cmdArgs, self)
helpText = None
try:
helpText = self._args.parse_args()
except KeyboardInterrupt:
self._keyboardInterrupt()
except Exception, e:
trace.traceback()
helpText = STR_HelpText_Usage
if len(e.args):
helpText += STR_ErrorCmdLineText.format(str(self._args.args), str(e))
else:
helpText += STR_ErrorParsingCommandLine
def run(self):
'''
Process entry point - set up debug/profile context
'''
try:
trace.set_context(self._dbgContext)
if self._profileName is not None:
import cProfile;
cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
def run(self):
trace.cc(1, "STARTING: Begining to process output queue...")
try:
if self._profileName is not None:
import cProfile;
cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name)
else:
self._run()
trace.cc(1, "FINISHED processing output queue")
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in OUTPUT THREAD")
thread.interrupt_main()
except Exception, e:
trace.cc(1, "EXCEPTION occurred while processing output queue")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback(2)
def run(self):
'''
Process entry point - set up debug/profile context
'''
try:
trace.set_context(self._dbgContext)
if self._profileName is not None:
import cProfile
cProfile.runctx('self._run()', globals(), {'self': self},
self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
def _measure_file(self, workItem):
'''
Unpack workItem and run all measures requested by the configItems
for the file
'''
( path,
deltaPath,
fileName,
configItems,
options,
numFilesInFolder
) = workItem
self._currentFilePath = os.path.join(path, fileName)
trace.file(3, "Processing: {0}".format(self._currentFilePath))
deltaFilePath = None
if deltaPath is not None:
deltaFilePath = os.path.join(deltaPath, fileName)
continueProcessing = True
try:
for configItem in configItems:
if self._check_for_stop():
break
self._open_file(configItem.module, deltaFilePath)
#
# Synchronus delegation to the measure module defined in the config file
#
configItem.module.process_file(
self._currentFilePath,
self._currentFileIterator,
configItem,
numFilesInFolder,
self.file_measured_callback)
except utils.FileMeasureError, e:
trace.traceback(2)
self._currentFileErrors.append(
uistrings.STR_ErrorMeasuringFile.format(self._currentFilePath, str(e)))
continueProcessing = not options.breakOnError
def _measure_file(self, workItem):
'''
Unpack workItem and run all measures requested by the configItems
for the file
'''
(path, deltaPath, fileName, configItems, options,
numFilesInFolder) = workItem
self._currentFilePath = os.path.join(path, fileName)
trace.file(1, "Processing: {0}".format(self._currentFilePath))
deltaFilePath = None
if deltaPath is not None:
deltaFilePath = os.path.join(deltaPath, fileName)
continueProcessing = True
try:
for configItem in configItems:
if self._check_for_stop():
break
self._open_file(configItem.module, deltaFilePath)
#
# Synchronus delegation to the measure module defined in the config file
#
configItem.module.process_file(self._currentFilePath,
self._currentFileIterator,
configItem, numFilesInFolder,
self.file_measured_callback)
except utils.FileMeasureError, e:
trace.traceback(2)
self._currentFileErrors.append(
uistrings.STR_ErrorMeasuringFile.format(
self._currentFilePath, str(e)))
continueProcessing = not options.breakOnError
def _survey_lines(self, linesToSurvey, params, measurements, analysis):
'''
Analyze file line by line. linesToSurvey is an iterable set of lines.
Processing is driven by the regular expressions in member variables.
The order of processing each line is:
- Preprocess line string
- Detect machine vs. human code
- Detect blank lines
- Detect single and multi-line comments
- Capture line measures
- Peform line processing (searches, routines, etc.)
'''
# Setup dictionary for measures and searches we'll do
self._survey_start(params)
# If no lines to process, we may still want to output empty measures
if linesToSurvey is None:
linesToSurvey = []
# Track whether we are inside a multi-line comment - we ignore nesting
scanningMultiLine = False
# If we have a line seperator, apply it
for bufferLine in linesToSurvey:
self.counts['RawLines'][self._activeBlock] += 1
if self._traceLevel: trace.file(4, "Raw: {0}".format(bufferLine))
# Allow specializations to special-case certain lines
if self._alternate_line_processing(bufferLine):
continue
lines = [bufferLine]
if self.addLineSep is not None:
lines = bufferLine.split(self.addLineSep)
#
# Read through the file lines and process them one at a time
# This is the main processing loop for all csmodules derived from NBNC
#
try:
for rawLine in lines:
self.counts['TotalLines'][self._activeBlock] += 1
# Allow for clean up of artifacts or other pre-processing
line = self._preprocess_line(rawLine)
# Detect true blank lines
if self.reTrueBlankLine.match(line):
self.counts['TrueBlankLines'][self._activeBlock] += 1
self._trace_line(line, "T")
continue
# Block Detection
if len(self.blockDetectors) > 1:
if self._detect_block_change(line, analysis):
scanningMultiLine = False # Don't allow multi-line comment to span blocks
# Determine comment state
# This is done before blank lines to make sure we consider multi-line
# comment syntax that will be counted as "blank", e.g., /* on it's own line
onCommentLine, scanningMultiLine = self._detect_line_comment(
line, scanningMultiLine)
# Detect blank lines
if self._detect_blank_line(line):
continue
# Measure and analyze -- overriden in derived classes
self._measure_line(line, onCommentLine)
self._analyze_line(line, analysis, onCommentLine)
except Exception, e:
trace.traceback()
raise utils.FileMeasureError(
"Problem processing line: {0} with module: {1}\n{2}".
format(str(sum(self.counts['RawLines'])),
self.__class__.__name__, str(e)))
if self._profileName is not None:
import cProfile
cProfile.runctx('self._run()', globals(), {'self': self},
self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in job worker loop")
except Exception, e:
trace.cc(1, "EXCEPTION occurred in job worker loop")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
finally:
# We know the input and out queues are empty or that we're bailing
# on them, so we cancel_join_thread (don't wait for them to clear)
self._inputQueue.close()
self._inputQueue.cancel_join_thread()
self._outputQueue.close()
self._outputQueue.cancel_join_thread()
# We may have put items on the control queue, so we join_thread to
# make sure what we've put in the pipe is flushed
self._controlQueue.close()
self._controlQueue.join_thread()
trace.cc(1, "TERMINATING")
def _run(self):
'''
def _exception(self, e):
trace.cc(1, "EXCEPTION -- EXITING JOB...")
self._send_workers_command('EXIT')
self._send_output_command('EXIT')
trace.traceback()
raise e
def _survey_lines(self, linesToSurvey, params, measurements, analysis):
'''
Analyze file line by line. linesToSurvey is an iterable set of lines.
Processing is driven by the regular expressions in member variables.
The order of processing each line is:
- Preprocess line string
- Detect machine vs. human code
- Detect blank lines
- Detect single and multi-line comments
- Capture line measures
- Peform line processing (searches, routines, etc.)
'''
# Setup dictionary for measures and searches we'll do
self._survey_start(params)
# If no lines to process, we may still want to output empty measures
if linesToSurvey is None:
linesToSurvey = []
# Track whether we are inside a multi-line comment - we ignore nesting
scanningMultiLine = False
# Loop through the raw lines we were passed
for bufferLine in linesToSurvey:
self.counts['RawLines'][self._activeBlock] += 1
if self._traceLevel: trace.file(4, "Raw: {0}".format(bufferLine))
# Allow specializations to skip and/or special-case certain lines
if self._alternate_line_processing(bufferLine):
continue
# If we have a line seperator, apply it
lines = [bufferLine]
if self.addLineSep is not None:
lines = bufferLine.split(self.addLineSep)
#
# Read through the lines to measure and process them one at a time
# This is the main measure loop for csmodules derived from NBNC
#
try:
for rawLine in lines:
self.counts['TotalLines'][self._activeBlock] += 1
# Allow for clean up of artifacts or other pre-processing
line = self._preprocess_line(rawLine)
# Detect true blank lines
if self.reTrueBlankLine.match(line):
self.counts['TrueBlankLines'][self._activeBlock] += 1
self._trace_line(line, "T")
continue
# Block Detection
if len(self.blockDetectors) > 1:
if self._detect_block_change(line, analysis):
scanningMultiLine = False # Don't allow multi-line comment to span blocks
# Determine comment state
# This is done before blank lines to make sure we consider multi-line
# comment syntax that will be counted as "blank", e.g., /* on it's own line
onCommentLine, scanningMultiLine = self._detect_line_comment(line, scanningMultiLine)
# Detect "blank" lines with no useful info
if self._detect_blank_line(line):
continue
# Measure and analyze -- overriden in derived classes
self._measure_line(line, onCommentLine)
self._analyze_line(line, analysis, onCommentLine)
except Exception, e:
trace.traceback()
raise utils.FileMeasureError(
"Problem processing line: {0} with module: {1}\n{2}".format(
str(sum(self.counts['RawLines'])), self.__class__.__name__, str(e)))
if self._profileName is not None:
import cProfile;
cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in job worker loop")
except Exception, e:
trace.cc(1, "EXCEPTION occurred in job worker loop")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
finally:
# We know the input and out queues are empty or that we're bailing
# on them, so we cancel_join_thread (don't wait for them to clear)
self._inputQueue.close()
self._inputQueue.cancel_join_thread()
self._outputQueue.close()
self._outputQueue.cancel_join_thread()
# We may have put items on the control queue, so we join_thread to
# make sure what we've put in the pipe is flushed
self._controlQueue.close()
self._controlQueue.join_thread()
trace.cc(1, "TERMINATING")
def _run(self):
def _parse_file(self, configFile, configEntries):
'''
Parse config file lines
'''
configEntry = configentry.ConfigEntry('_ _ _ _') # Init to empty object to prevent PyChecker warnings
constants = {}
readingVerbs = False
verbEndMarker = None
for whiteSpaceRawline in configFile:
trace.config(3, "Config line: {0}".format(whiteSpaceRawline))
rawLine = whiteSpaceRawline.strip()
line = rawLine
# Skip comments, blank lines
if self.comment.match(line) or self.blankLine.match(line):
trace.config(4, "comment/blank")
continue
# Skip ignore blocks (or go to end of file if no closing block)
if self.ignoreStart.match(line):
trace.config(4, "ignoreBlock")
try:
while not self.ignoreStop.match(line):
line = configFile.next()
trace.config(4, "Config ignore: {0}".format(line))
except Exception:
trace.config(4, "Exception while seeking end of ignore block")
pass
continue
# Includes
# Attempt to load the requested file and add it's entries
# to our entries, in the form INCLUDE:path: tagInfo
includeMatch = self.include.match(line)
if includeMatch:
includePath = includeMatch.group(1)
newTags = includeMatch.group(2)
if not os.path.isabs(includePath):
includePath = os.path.join(os.path.dirname(configFile.name), includePath)
trace.config(1, "Include: {0}".format(includePath))
newEntries = self._read_file(includePath, [])
existingFileFilterStrings = [entry.fileFilter for entry in configEntries]
for entry in newEntries:
# If an entry has already been defined with the SAME FILE FILTER STRING,
# the INCLUDED ENTRY WILL BE IGNORED
if entry.fileFilter in existingFileFilterStrings:
continue
# If 'tagInfo' is provided, it will be added to ALL entries of the file
# that was included
# We RELOAD THE MODULE in case new options need processed
if newTags:
entry.add_tags_and_options(newTags.split())
self._load_csmodule(entry)
configEntries.append(entry)
continue
# If line closes out a verb entry store the config entry
if readingVerbs and re.match(verbEndMarker, line):
trace.config(4, "verbend: {0}".format(line))
readingVerbs = False
configEntries.append(configEntry)
continue
# Handle continued lines
fullLine = ""
while True:
contLineMatch = self.continuedLine.match(line)
if contLineMatch:
fullLine += contLineMatch.group(CONT_LINE_START)
line = configFile.next().strip()
trace.config(3, "FullLine: {0}".format(line))
else:
fullLine += line
break
assert fullLine
line = fullLine
# If line defines a normal constant, store asis
constantMatch = self.constant.match(line)
if constantMatch:
# Assign cosntant, strip spaces to support config lines that are space-delimited
constants[constantMatch.group(1)] = constantMatch.group(2)
trace.config(2, "Constant: {0}".format(constantMatch.group(2)))
continue
# If line defines a no blanks constant, strip spaces and store
constantMatch = self.constant_noblanks.match(line)
if constantMatch:
constants[constantMatch.group(1)] = constantMatch.group(2).replace(' ', '')
trace.config(2, "Noblank constant: {0}".format(constantMatch.group(2)))
continue
# Replace any constants used in the line
line = self._replace_constants(line, constants)
trace.config(4, "fullline: {0}".format(line))
# Strip any inline comments
line = line.split(' #')[0]
# If the line is a parameter (e.g., search terms), delegate to module
# to get processed parameters and store for later usage
# We keep the unprocessed raw version around for consistency checking
if readingVerbs:
configEntry.paramsRaw.append(rawLine)
try:
paramTuple = configEntry.module.add_param(line, rawLine)
configEntry.paramsProcessed.append(paramTuple)
trace.config(2, "LoadedParam: {0} => {1}".format(
configEntry.module.__class__.__name__, paramTuple))
except Exception, e:
trace.traceback()
raise utils.ConfigError(uistrings.STR_ErrorConfigParam.format(
str(configEntry), rawLine, str(e)))
# Otherwise assume we're at the start of a config entry definition,
else:
try:
# Load and validate the config line and its module
configEntry = configentry.ConfigEntry(line, self._extraLineContent, configFile.name)
self._load_csmodule(configEntry)
self._validate_line(configEntry)
# Check to see if there are parameter lines to read
verbEndMarker = configEntry.module.verb_end_marker(configEntry.verb)
if verbEndMarker is not None:
readingVerbs = True
# Add the completed config entry to our list
if not readingVerbs:
configEntries.append(configEntry)
except Exception, e:
trace.traceback()
raise utils.ConfigError(uistrings.STR_ErrorConfigEntry.format(
rawLine, str(e)))