def _validate_file(self, configEntries):
if not configEntries:
log.config(2, " EMPTY")
else:
try:
self._validate_entries(configEntries)
except Exception as e:
log.stack()
raise utils.ConfigError(
uistrings.STR_ErrorConfigValidate.format(str(e)))
def run(self):
log.cc(1, "STARTING: Begining to process output queue...")
try:
if self._profileName is not None:
import cProfile
cProfile.runctx('self._run()', globals(), {'self': self},
self._profileName + self.name)
else:
self._run()
log.cc(1, "FINISHED processing output queue")
except KeyboardInterrupt:
log.cc(1, "Ctrl-c occurred in OUTPUT THREAD")
_thread.interrupt_main()
except Exception as e:
log.msg(1, "EXCEPTION processing output queue: " + str(e))
log.stack()
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
finally:
log.cc(1, "TERMINATING")
def _parse_command_line(self, cmdArgs):
init_surveyor_dir(cmdArgs[0])
self._args = cmdlineargs.SurveyorCmdLineArgs(cmdArgs, self)
helpText = None
try:
helpText = self._args.parse_args()
except KeyboardInterrupt:
self._keyboardInterrupt()
except Exception as e:
log.stack()
helpText = STR_HelpText_Usage
if len(e.args):
helpText += STR_ErrorCmdLineText.format(str(self._args.args), str(e))
else:
helpText += STR_ErrorParsingCommandLine
finally:
if helpText is None:
return True
else:
self._display_help(helpText)
return False
def _put_files_in_queue(self, path, deltaPath, filesAndConfigs):
'''
Package files from the path into workItems that are grouped
into workPackages and placed into the task queue for jobworkers.
Packages are broken up if files number or total size exceeds
thresholds to help evenly distribute load across cores
'''
if not filesAndConfigs:
return
for fileName, configEntrys in filesAndConfigs:
# Expensive to check file size here, but worth it for pracelling widely
# varying file sizes out to cores for CPU intensive jobs.
# Profiling shows it is not worth caching this
try:
fileSize = utils.get_file_size(os.path.join(path, fileName))
except Exception as e:
# It is possible (at least in Windows) for a fileName to exist
# in the file system but be invalid for Windows calls. This is
# the first place the file is accessed through the file system;
# if it blows up don't want the job to fall apart, and this is
# an unusual case, so don't bother with a pathway back to the main
# application; just swallow it and provide debug
log.msg(1, str(e))
log.stack()
continue
log.cc(3, "WorkItem: {}, {}".format(fileSize, fileName))
self.numFilesToProcess += 1
workItem = (path, deltaPath, fileName, configEntrys, self._options,
len(filesAndConfigs))
self._workPackage.add(workItem, fileSize)
if self._workPackage.ready_to_send() or (self._filesSinceLastSend >
MAX_FILES_BEFORE_SEND):
self._send_current_package()
if not self._check_command():
break
def _survey_lines(self, linesToSurvey, params, measurements, analysis):
'''
Analyze file line by line. linesToSurvey is an iterable set of lines.
Processing is driven by the regular expressions in member variables.
The order of processing each line is:
- Preprocess line string
- Detect machine vs. human code
- Detect blank lines
- Detect single and multi-line comments
- Capture line measures
- Peform line processing (searches, routines, etc.)
'''
# Setup dictionary for measures and searches we'll do
self._survey_start(params)
# If no lines to process, may still want to output empty measures
if linesToSurvey is None:
linesToSurvey = []
# Track whether inside a multi-line comment - ignore nesting
scanningMultiLine = False
for bufferLine in linesToSurvey:
# Handle option of reading out binary files
bufferLine = utils.safe_string(bufferLine)
self.counts['RawLines'][self._activeBlock] += 1
if self._logLevel: log.file(4, "Raw: {}".format(bufferLine))
try:
# Allow specializations to skip and/or special-case certain lines
if self._alternate_line_processing(bufferLine):
continue
# If line seperator, apply it
lines = [bufferLine]
if self.addLineSep is not None:
lines = bufferLine.split(self.addLineSep)
#
# Read through the lines to measure and process them one at a time
# This is the main measure loop for csmodules derived from NBNC
#
for rawLine in lines:
self.counts['TotalLines'][self._activeBlock] += 1
# Allow for clean up of artifacts or other pre-processing
line = self._preprocess_line(rawLine)
# Detect true blank lines
if self.reTrueBlankLine.match(line):
self.counts['TrueBlankLines'][self._activeBlock] += 1
self._log_line(line, "T")
continue
# Block Detection
if len(self.blockDetectors) > 1:
if self._detect_block_change(line, analysis):
scanningMultiLine = False # Don't allow multi-line comment to span blocks
# Determine comment state
# This is done before blank lines to consider multi-line
# comment syntax that will be counted as "blank", e.g., /* on it's own line
onCommentLine, scanningMultiLine = self._detect_line_comment(line, scanningMultiLine)
# Detect "blank" lines with no useful info
if self._detect_blank_line(line):
continue
# Measure and analyze -- overriden in derived classes
self._measure_line(line, onCommentLine)
self._analyze_line(line, analysis, onCommentLine)
except Exception as e:
log.stack()
if self.stopOnError:
raise utils.FileMeasureError(
"Problem processing line: {} with module: {}\n{}".format(
str(sum(self.counts['RawLines'])), self.__class__.__name__, str(e)))
# Package results
self._survey_end(measurements, analysis)
def _parse_file(self, configFile, configEntries):
'''
Parse config file lines
'''
configEntry = configentry.ConfigEntry(
'_ _ _ _') # Init to empty object to prevent PyChecker warnings
constants = {}
readingVerbs = False
verbEndMarker = None
for whiteSpaceRawline in configFile:
log.config(3, "Config line: {}".format(whiteSpaceRawline))
rawLine = whiteSpaceRawline.strip()
line = rawLine
# Skip comments, blank lines
if self.comment.match(line) or self.blankLine.match(line):
log.config(4, "comment/blank")
continue
# Skip ignore blocks (or go to end of file if no closing block)
if self.ignoreStart.match(line):
log.config(4, "ignoreBlock")
try:
while not self.ignoreStop.match(line):
line = next(configFile)
log.config(4, "Config ignore: {}".format(line))
except Exception:
log.config(4,
"Exception while seeking end of ignore block")
pass
continue
# Includes
# Attempt to load the requested file and add it's entries
# to our entries, in the form INCLUDE:path: tagInfo
includeMatch = self.include.match(line)
if includeMatch:
includePath = includeMatch.group(1)
newTags = includeMatch.group(2)
if not os.path.isabs(includePath):
includePath = os.path.join(
os.path.dirname(configFile.name), includePath)
log.config(1, "Include: {}".format(includePath))
newEntries = self._read_file(includePath, [])
existingFileFilterStrings = [
entry.fileFilter for entry in configEntries
]
for entry in newEntries:
# If an entry has already been defined with the SAME FILE FILTER STRING,
# the INCLUDED ENTRY WILL BE IGNORED
if entry.fileFilter in existingFileFilterStrings:
continue
# If 'tagInfo' is provided, it will be added to ALL entries of the file
# that was included
# RELOAD THE MODULE in case new options need processed
if newTags:
entry.add_tags_and_options(newTags.split())
self._load_csmodule(entry)
configEntries.append(entry)
continue
# If line closes out a verb entry store the config entry
if readingVerbs and re.match(verbEndMarker, line):
log.config(4, "verbend: {}".format(line))
readingVerbs = False
configEntries.append(configEntry)
continue
# Handle continued lines
fullLine = ""
while True:
contLineMatch = self.continuedLine.match(line)
if contLineMatch:
fullLine += contLineMatch.group(CONT_LINE_START)
line = next(configFile).strip()
log.config(3, "FullLine: {}".format(line))
else:
fullLine += line
break
assert fullLine
line = fullLine
# If line defines a normal constant, store asis
constantMatch = self.constant.match(line)
if constantMatch:
# Assign cosntant, strip spaces to support config lines that are space-delimited
constants[constantMatch.group(1)] = constantMatch.group(2)
log.config(2, "Constant: {}".format(constantMatch.group(2)))
continue
# If line defines a no blanks constant, strip spaces and store
constantMatch = self.constant_noblanks.match(line)
if constantMatch:
constants[constantMatch.group(1)] = constantMatch.group(
2).replace(' ', '')
log.config(
2, "Noblank constant: {}".format(constantMatch.group(2)))
continue
# Replace any constants used in the line
line = self._replace_constants(line, constants)
log.config(4, "fullline: {}".format(line))
# Strip any inline comments
line = line.split(' #')[0]
# If the line is a parameter (e.g., search terms), delegate to module
# to get processed parameters and store for later usage
# Keep the unprocessed raw version around for consistency checking
if readingVerbs:
configEntry.paramsRaw.append(rawLine)
try:
paramTuple = configEntry.module.add_param(line, rawLine)
configEntry.paramsProcessed.append(paramTuple)
log.config(
2, "LoadedParam: {} => {}".format(
configEntry.module.__class__.__name__, paramTuple))
except Exception as e:
log.stack()
raise utils.ConfigError(
uistrings.STR_ErrorConfigParam.format(
str(configEntry), rawLine, str(e)))
# Otherwise assume we're at the start of a config entry definition,
else:
try:
# Load and validate the config line and its module
configEntry = configentry.ConfigEntry(
line, self._extraLineContent, configFile.name)
self._load_csmodule(configEntry)
self._validate_line(configEntry)
# Check to see if there are parameter lines to read
verbEndMarker = configEntry.module.verb_end_marker(
configEntry.verb)
if verbEndMarker is not None:
readingVerbs = True
# Add the completed config entry to our list
if not readingVerbs:
configEntries.append(configEntry)
except Exception as e:
log.stack()
raise utils.ConfigError(
uistrings.STR_ErrorConfigEntry.format(rawLine, str(e)))
# Loop to next line
return configEntries