def _file_match(fileName, fileFilter):
'''
Performs the match check of filename to filter
In the case of blank detection, look for no extension
Otherwise use regex comparison using cached version of either the
re from fnmatch.translate or custom RE string provided in filter
'''
if BLANK_FILE_EXT == fileFilter:
root, ext = os.path.splitext(fileName)
filterMatch = ('' == ext and not root.startswith('.'))
else:
filterRe = None
try:
filterRe = _FilterCache[fileFilter]
except KeyError:
if fileFilter.startswith(CUSTOM_FILE_REGEX):
filterRe = re.compile(fileFilter.replace(CUSTOM_FILE_REGEX, ''), RE_OPTIONS)
else:
filterRe = re.compile(fnmatch.translate(fileFilter), RE_OPTIONS)
_FilterCache[fileFilter] = filterRe
filterMatch = filterRe.match(fileName)
if trace.level() and filterMatch is None:
trace.file(3, "FilterExtFilter: %s, no match: %s" % (filterRe.pattern[:10], fileName))
return filterMatch is not None
def is_text_file(fileObject):
textChars = string.letters + string.digits + string.punctuation + string.whitespace
bytesToCheck = 128 # Big enough window to grab, but small for speed
startPoint = 4 # Skip start of file, for hidden text codes
minWindowSize = 32 # Get a big enough min window to be feasible
nonTextThreshold = 0.2 # Have some tolerance to avoid false positives
# Grab the first bytes of the file, STRIPPING NULLS (for unicode text files)
fileBytes = utils.strip_null_chars(
utils.get_file_start(fileObject, bytesToCheck))
# Special case for PDF that looks like text but isn't
if is_pdf_file(fileObject):
isBelowThreshold = False
trace.file(
3, " IsTextFile({0}): {1} ==> PDF File detected".format(
isBelowThreshold, os.path.basename(fileObject.name)))
else:
isBelowThreshold = utils.check_bytes_below_threshold(
fileBytes, textChars, minWindowSize, startPoint, nonTextThreshold)
trace.file(
3, " IsTextFile({0}): {1} ==> {2}".format(
isBelowThreshold, os.path.basename(fileObject.name),
fileBytes))
return isBelowThreshold
def _run(self):
# We keep processing queue until the job signals it is done and
# the queue is empty, or we receive an abort command
while self._continue_processing():
try:
if self._workDone and self._outQueue.empty():
break
filesOutput = self._outQueue.get_nowait()
except Empty:
trace.cc(3, "EMPTY OUTPUT")
time.sleep(OUTPUT_EMPTY_WAIT)
else:
self.taskPackagesReceived += 1
trace.cc(2, "GOT {0} measures".format(len(filesOutput)))
# We get a set of output for multiple files with each
# outputQueue item. Each file has a set of output
# and potential errors that we pack to the app
for filePath, outputList, errorList in filesOutput:
# Synchronus callback to applicaiton
# Output writing and screen update occurs in this call
self._file_measure_callback(filePath, outputList, errorList)
if errorList:
trace.file(1, "ERROR measuring: {0}".format(filePath))
self._controlQueue.put_nowait(('JOB', 'ERROR', filePath))
def _valid_folder(self, folderName):
'''
Is this folder one we should process?
'''
if not self._skipFolders and not self._includeFolders:
return True
validFolder = True
# First verfiy this folder is not to be skipped
if self._skipFolders:
_root, currentFolder = os.path.split(folderName)
for folderPattern in self._skipFolders:
if fnmatch.fnmatch(currentFolder, folderPattern):
trace.file(1, "Skiping folder: %s" % folderName)
validFolder = False
break
# Next verify if it is on the include list
if validFolder and self._includeFolders:
includeMatch = False
for folderPattern in self._includeFolders:
if fnmatch.fnmatch(folderName, folderPattern):
includeMatch = True
break
if not includeMatch:
trace.file(1, "Excluding folder: %s" % folderName)
validFolder = False
return validFolder
def is_noncode_ext(filePath):
rv = False
if is_compressed_ext(filePath) or _has_ext(filePath,
NonCodeFileExtensions):
rv = True
trace.file(3, " NonCodeExt: {0}".format(os.path.basename(filePath)))
return rv
def file_measured_callback(self, filePath, outputList, errorList):
'''
Job output thread callback to provide file measurements.
A list of output and potential errors is provided for each file.
Called ONCE for each file in the job; if there were multiple
config entries for the file, outputList will have multiple items.
'''
self._numFilesProcessed += 1
self._errorList.extend(errorList)
fileTime = 0
fileMeasured = False
for measures, analysisResults in outputList:
trace.file(2, "Callback: {0} -- {1}".format(filePath, measures))
if measures.items():
# Zero out dupe measures in place
if self._dupeTracking:
self._filter_dupes(filePath, measures, analysisResults)
# Send results to metrics writer
fileMeasured = True
self._numMeasures += max(1, len(analysisResults))
if not self._summaryOnly:
self._writer.write_items(measures, analysisResults)
# Capture summary metrics and aggregates
self._stash_summary_metrics(filePath, measures, analysisResults)
self._stash_aggregates(filePath, analysisResults)
fileTime += utils.safe_dict_get_float(measures, basemodule.METADATA_TIMING)
self._numFilesMeasured += (1 if fileMeasured else 0)
self._display_file_progress(filePath, fileTime)
self._display_feedback()
def is_noncode_file(fileObject):
maxWindowSize = 30
fileStart = utils.get_file_start(fileObject, maxWindowSize)
phraseFound = utils.check_start_phrases(fileStart, NonCodeFileStart)
trace.file(3, " NonCodeFileStart({0}): {1} ==> {2}".format(
phraseFound, fileStart, os.path.basename(fileObject.name)))
return phraseFound is not None
def _valid_folder(self, folderName):
'''
Is this folder one we should process?
'''
if not self._skipFolders and not self._includeFolders:
return True
validFolder = True
# First verfiy this folder is not to be skipped
if self._skipFolders:
_root, currentFolder = os.path.split(folderName)
for folderPattern in self._skipFolders:
if fnmatch.fnmatch(currentFolder, folderPattern):
trace.file(1, "Skiping folder: {0}".format(folderName))
validFolder = False
break
# Next verify if it is on the include list
if validFolder and self._includeFolders:
includeMatch = False
for folderPattern in self._includeFolders:
if fnmatch.fnmatch(folderName, folderPattern):
includeMatch = True
break
if not includeMatch:
trace.file(1, "Excluding folder: {0}".format(folderName))
validFolder = False
return validFolder
def _open_file(self, filename):
MeasureWriter._open_file(self, filename)
filePath = os.path.join(self._outDir, filename)
outFile = file(filePath, 'w')
doc = minidom.Document()
outFile.write(doc.toprettyxml())
trace.file(2, "Opened XML Output File: {0}".format(filePath))
return outFile
def is_noncode_file(fileObject):
maxWindowSize = 30
fileStart = utils.get_file_start(fileObject, maxWindowSize)
phraseFound = utils.check_start_phrases(fileStart, NonCodeFileStart)
trace.file(
3, " NonCodeFileStart({0}): {1} ==> {2}".format(
phraseFound, fileStart, os.path.basename(fileObject.name)))
return phraseFound is not None
def _open_file(self, filename):
MeasureWriter._open_file(self, filename)
filePath = os.path.join(self._outDir, filename)
outFile = file(filePath, 'w')
doc = minidom.Document()
outFile.write(doc.toprettyxml())
trace.file(2, "Opened XML Output File: {0}".format(filePath))
return outFile
def _open_file(self, fileName):
MeasureWriter._open_file(self, fileName)
filePath = os.path.join(self._outDir, fileName)
self._rawFiles[fileName] = file(filePath, 'wb')
outWriter = csv.writer(
self._rawFiles[fileName], delimiter=self._delimiter, quoting=csv.QUOTE_NONNUMERIC)
trace.file(2, "Opened Delimited Output File: {0}".format(filePath))
return outWriter
def _open_file(self, fileName):
MeasureWriter._open_file(self, fileName)
filePath = os.path.join(self._outDir, fileName)
self._rawFiles[fileName] = file(filePath, 'wb')
outWriter = csv.writer(self._rawFiles[fileName],
delimiter=self._delimiter,
quoting=csv.QUOTE_NONNUMERIC)
trace.file(2, "Opened Delimited Output File: {0}".format(filePath))
return outWriter
def file_measured_callback(self, filePath, measures, analysisResults):
'''
Callback from the masurement module
We store up a list of tuples with the work output for a given file
'''
assert filePath == self._currentFilePath, "Measure callback out of sync"
trace.cc(3, "_file_measured_callback: {0}".format(filePath))
trace.file(3, " measures: {0}".format(measures))
trace.file(3, " analysis: {0}".format(analysisResults))
self._currentFileOutput.append((measures, analysisResults))
def file_measured_callback(self, filePath, measures, analysisResults):
'''
Callback from the masurement module
We store up a list of tuples with the work output for a given file
'''
assert filePath == self._currentFilePath, "Measure callback out of sync"
trace.cc(3, "_file_measured_callback: {0}".format(filePath))
trace.file(3, " measures: {0}".format(measures))
trace.file(3, " analysis: {0}".format(analysisResults))
self._currentFileOutput.append((measures, analysisResults))
def _remove_skip_dirs(self, root, dirs):
'''
Decide what children dirs should be skipped
Filter out dirs in place (vs a copy), so os.walk will skip
'''
dirsToRemove = []
for folderPattern in self._skipFolders:
dirsToRemove += fnmatch.filter(dirs, folderPattern)
dirsToRemove = set(dirsToRemove)
for folder in dirsToRemove:
trace.file(1, "Skiping over: %s\\%s" % (root, folder))
dirs.remove(folder)
def _remove_skip_dirs(self, root, dirs):
'''
Decide what children dirs should be skipped
Filter out dirs in place (vs a copy), so os.walk will skip
'''
dirsToRemove = []
for folderPattern in self._skipFolders:
dirsToRemove += fnmatch.filter(dirs, folderPattern)
dirsToRemove = set(dirsToRemove)
for folder in dirsToRemove:
trace.file(1, "Skiping over: {0}\\{1}".format(root, folder))
dirs.remove(folder)
def _has_ext(filePath, extensions):
'''
Does file have an extension, stripping off any numeric only extensions
'''
fileExt = None
while True:
(base, extension) = os.path.splitext(filePath)
fileExt = str(extension).strip('.')
if fileExt.isdigit():
filePath = base
else:
break
trace.file(3, " File Extension: {0}".format(fileExt))
return fileExt.lower() in extensions
def _get_files_to_process(self, folderName, fileNames, fileFilters, configPath):
'''
Filter the list of files based on command line options and active
config file filters
'''
# if fileFilters is empty it means an empty config file, so skip all files
if not fileFilters:
return []
# Optimize the most common matching of extensions by creating cache of
# simple '*.xxx' extensions from config filters for each config file
filterExts = []
try:
filterExts = self._configFilterCache[configPath]
except KeyError:
filterSplits = [os.path.splitext(fileFilter) for fileFilter in fileFilters if
os.path.splitext(fileFilter)[0] == '*']
filterExts = [ext for _root, ext in filterSplits]
self._configFilterCache[configPath] = filterExts
# Select files based on matching filters
filesToProcess = []
for fileName in fileNames:
# Filter file list by command-line postive filter, if provided
if fileext.file_matches_filters(fileName, self._fileExtFilters):
# Optimize most common case of direct match of file extension, then
# fall back to doing a full filter match on config file filter
_root, fileExt = os.path.splitext(fileName)
fileFilter = None
if fileExt in filterExts:
fileFilter = '*' + fileExt
else:
fileFilter = fileext.file_matches_filters(fileName, fileFilters)
if fileFilter is not None:
filesToProcess.append((fileName, fileFilter))
# Remove files that should be skipped
if self._skipFiles:
filesToProcess = [(fileName, fileFilter) for fileName, fileFilter in filesToProcess if
not fileext.file_matches_filters(fileName, self._skipFiles)]
# Debug tracing of files that were not measured
if trace.level():
filesSkipped = set(fileNames) - set([f for f, _filter in filesToProcess])
if filesSkipped:
trace.file(2, "SkippingFiles: %s" % filesSkipped)
return filesToProcess
def _measure_text(self, fileObject, measurements):
'''
Default handler For text based files, go through each file line
'''
if self._traceLevel: trace.file(4, "Document: {0}".format(fileObject))
for rawLine in fileObject:
self.totalLines += 1
line = utils.strip_null_chars(rawLine)
# Detect blank lines
if self.reBlankLine.match(line):
self.blankLines += 1
continue
# Content line
self.contentLines += 1
def is_text_file(fileObject):
textChars = string.letters + string.digits + string.punctuation + string.whitespace
bytesToCheck = 128 # Big enough window to grab, but small for speed
startPoint = 4 # Skip start of file, for hidden text codes
minWindowSize = 32 # Get a big enough min window to be feasible
nonTextThreshold = 0.2 # Have some tolerance to avoid false positives
# Grab the first bytes of the file, STRIPPING NULLS (for unicode text files)
fileBytes = utils.strip_null_chars(utils.get_file_start(fileObject, bytesToCheck))
isBelowThreshold = utils.check_bytes_below_threshold(
fileBytes, textChars, minWindowSize, startPoint, nonTextThreshold)
trace.file(3," IsTextFile({0}): {1} ==> {2}".format(
isBelowThreshold, os.path.basename(fileObject.name), fileBytes))
return isBelowThreshold
def walk(self, pathToMeasure):
'''
Walk folders while filtering sending updates via callback
We may be asked to terminate in our callback
'''
self._configStack.set_measure_root(pathToMeasure)
for folderName, childFolders, fileNames in os.walk(pathToMeasure,
topdown=True):
trace.file(2, "Scanning: {0}".format(folderName))
numUnfilteredFiles = len(fileNames)
filesAndConfigs = []
if fileNames and self._valid_folder(folderName):
# Get the current set of active config filters
fileFilters, activeConfigs, configPath = self._configStack.get_configuration(
folderName)
# Filter out files by options and config items
filesToProcess = self._get_files_to_process(
folderName, fileNames, fileFilters, configPath)
# Create list of tuples with fileName and configEntrys for each file
for fileName, fileFilter in filesToProcess:
configEntrys = self._get_configs_for_file(
fileName, fileFilter, activeConfigs, configPath)
filesAndConfigs.append((fileName, configEntrys))
# For delta measure create a fully qualified delta path name
# Note when we split on path to measure, it will start with seperator
deltaFolder = None
if self._deltaPath is not None:
deltaFolder = self._deltaPath + folderName[len(pathToMeasure):]
# Call back to job with files and configs
continueProcessing = self._add_files_to_job(
folderName, deltaFolder, filesAndConfigs, numUnfilteredFiles)
if not continueProcessing or not self._expandSubdirs:
break
# Remove any folders, and sort remaining to ensure consistent walk
# order across file systems (for our testing if nothing else)
self._remove_skip_dirs(folderName, childFolders)
childFolders.sort()
def walk(self, pathToMeasure):
'''
Walk folders while filtering sending updates via callback
We may be asked to terminate in our callback
'''
self._configStack.set_measure_root(pathToMeasure)
for folderName, childFolders, fileNames in os.walk(pathToMeasure, topdown=True):
trace.file(2, "Scanning: {0}".format(folderName))
numUnfilteredFiles = len(fileNames)
filesAndConfigs = []
if fileNames and self._valid_folder(folderName):
# Get the current set of active config filters
fileFilters, activeConfigs, configPath = self._configStack.get_configuration(folderName)
# Filter out files by options and config items
filesToProcess = self._get_files_to_process(folderName, fileNames, fileFilters, configPath)
# Create list of tuples with fileName and configEntrys for each file
for fileName, fileFilter in filesToProcess:
configEntrys = self._get_configs_for_file(fileName, fileFilter, activeConfigs, configPath)
filesAndConfigs.append((fileName, configEntrys))
# For delta measure create a fully qualified delta path name
# Note when we split on path to measure, it will start with seperator
deltaFolder = None
if self._deltaPath is not None:
deltaFolder = self._deltaPath + folderName[len(pathToMeasure):]
# Call back to job with files and configs
continueProcessing = self._add_files_to_job(
folderName,
deltaFolder,
filesAndConfigs,
numUnfilteredFiles)
if not continueProcessing or not self._expandSubdirs:
break
# Remove any folders, and sort remaining to ensure consistent walk
# order across file systems (for our testing if nothing else)
self._remove_skip_dirs(folderName, childFolders)
childFolders.sort()
def _measure_file(self, workItem):
'''
Unpack workItem and run all measures requested by the configItems
for the file
'''
( path,
deltaPath,
fileName,
configItems,
options,
numFilesInFolder
) = workItem
self._currentFilePath = os.path.join(path, fileName)
trace.file(3, "Processing: {0}".format(self._currentFilePath))
deltaFilePath = None
if deltaPath is not None:
deltaFilePath = os.path.join(deltaPath, fileName)
continueProcessing = True
try:
for configItem in configItems:
if self._check_for_stop():
break
self._open_file(configItem.module, deltaFilePath)
#
# Synchronus delegation to the measure module defined in the config file
#
configItem.module.process_file(
self._currentFilePath,
self._currentFileIterator,
configItem,
numFilesInFolder,
self.file_measured_callback)
except utils.FileMeasureError, e:
trace.traceback(2)
self._currentFileErrors.append(
uistrings.STR_ErrorMeasuringFile.format(self._currentFilePath, str(e)))
continueProcessing = not options.breakOnError
def _stash_aggregates(self, filePath, analysisResults):
'''
As we receive results for files, if we have requests to aggregate
results, store away aggregate information.
The aggreate functionality is based on names of items generated
by specific csmodules; we consider it a fatal error if what is
requested for aggregation and what is present in analysisResults
are out of sync
'''
# For each set of aggregates we go through results and add
# them to the appropriate aggregate set
for aggKey, aggNames in self._aggregateNames.iteritems():
aggregateDict = self._aggregates.setdefault(aggKey, {})
trace.file(2, "Aggregating {0} items in {1}".format(len(analysisResults), aggKey))
for result in analysisResults:
# aggKey has the name for the value from results that we
# will be keying the aggreate dictionary on
try:
newKey = result[aggKey]
except KeyError, e:
raise utils.InputException(STR_AggregateKeyError.format(str(e)))
else:
aggregate = aggregateDict.setdefault(newKey, {'aggregate.count':0})
# Sepcific names can be provided to aggregate, or can do all
namesToAggregate = aggNames
if isinstance(aggNames, basestring):
if aggNames == 'all':
namesToAggregate = result.keys()
# Take each value from the result and aggregate according to type
for itemName in namesToAggregate:
self._aggregate_update(itemName, result[itemName], aggregate)
# Count the item
aggregate['aggregate.count'] += 1
# Update the aggregate
aggregateDict[newKey] = aggregate
# The dictionary for this aggKey has been updated, so stash it
self._aggregates[aggKey] = aggregateDict
def _get_delta_lines(self, filePath, deltaFilePath):
'''
Return a line buffer that represents additional lines relative to the
delta path. We are not doing a full diff, only taking into account new
files, and lines in existing files that are new/modified.
'''
self._deltaFilePath = deltaFilePath
deltaLines = None
# If no correpsonding file exists in delta, we do a normal file open
if not os.path.exists(deltaFilePath):
trace.file(1, "Delta file doesn't exist for: {0}".format(deltaFilePath))
deltaLines = self._open_file(filePath)
# We only do a diff if there is an identical file name that has been modified
elif not filecmp.cmp(deltaFilePath, filePath):
fileToMeasure = self._open_file(filePath)
if fileToMeasure is not None:
measureFileLines = fileToMeasure.readlines()
fileToMeasure.close()
deltaFileLines = None
with open(deltaFilePath, 'rU') as deltaFile:
deltaFileLines = deltaFile.readlines()
diffLines = difflib.unified_diff(deltaFileLines, measureFileLines)
if diffLines:
deltaLines = []
for line in diffLines:
if line.startswith('+') or (self._deltaIncludeDeleted and line.startswith('-')):
deltaLines.append(line[2:])
trace.file(1, "{0} delta lines with: {1}".format(len(deltaLines), deltaFilePath))
else:
trace.file(1, "Delta skip: {0} == {1}".format(filePath, deltaFilePath))
return deltaLines
def _is_file_survey_dupe(self, filePath, measures):
'''
Simple mechanism to identify duplicate and near-dupicate code by tracking
a dictionary of files we see as measures. There are two modes:
1) File Size: Build a dictionary in memory based on a hash of fileName
and config info. In the hash buckets we store a dict of file sizes for
the first of each size we see that is not within the dupe threshold.
If we see a file size within the threshold of one of our existing
hashed sizes, we treat it as a dupe and increment count for reporting.
2) NBNC CRC: We use the nbnc.crc measure to identify duplicates
Note that we ASSUME the necessary file metadata will be present in the
measures dicitonary, as basemodule.py puts it there for the Dupe option.
'''
firstDupeFilePath = None
# 1) File name and Size check
if isinstance(self._dupeThreshold, int):
fileSize = int(measures[basemodule.METADATA_FILESIZE])
dupeKey = (measures[basemodule.METADATA_FULLNAME] +
measures[basemodule.METADATA_CONFIG].replace(' ', ''))
if dupeKey in self._dupeFileSurveys:
for dupeFileSize, (fileCount, firstFilePath) in self._dupeFileSurveys[dupeKey].iteritems():
if (dupeFileSize - self._dupeThreshold) <= fileSize and (
fileSize <= (dupeFileSize + self._dupeThreshold)):
firstDupeFilePath = firstFilePath
self._dupeFileSurveys[dupeKey][dupeFileSize] = (fileCount + 1, firstFilePath)
trace.msg(1, "Dupe {0} by {1} of {2} bytes: {3}".format(
fileCount, fileSize - dupeFileSize, fileSize, filePath))
break
else:
self._dupeFileSurveys[dupeKey] = {}
if firstDupeFilePath is None:
self._dupeFileSurveys[dupeKey][fileSize] = (1, filePath)
trace.file(2, "Added {0} -- {1} to dupe dictionary".format(dupeKey, fileSize))
# 2) Code CRC check
# Our relying on the nbnc.crc is brittle, because it is both a code and runtime
# dependency on the Code csmodule being used. And there are valid scenarios
# where nbnc.crc may not be present (e.g., skipping dupe file). Thus if the
# measure isn't present, we fail silently
else:
fileCrc = None
try:
fileCrc = measures['nbnc.crc']
except:
trace.file(2, "CRC Dupe - nbnc.crc missing: {0}".format(filePath))
if fileCrc in self._dupeFileSurveys:
fileCount, firstDupeFilePath = self._dupeFileSurveys[fileCrc]
self._dupeFileSurveys[fileCrc] = (fileCount + 1, firstDupeFilePath)
trace.msg(1, "Dupe {0}: {1} DUPE_OF {2}".format(fileCount, filePath, firstDupeFilePath))
elif fileCrc is not None:
self._dupeFileSurveys[fileCrc] = (1, filePath)
trace.file(2, "Added {0} -- {1} to dupe dictionary".format(filePath, fileCrc))
return firstDupeFilePath
def _measure_file(self, workItem):
'''
Unpack workItem and run all measures requested by the configItems
for the file
'''
(path, deltaPath, fileName, configItems, options,
numFilesInFolder) = workItem
self._currentFilePath = os.path.join(path, fileName)
trace.file(1, "Processing: {0}".format(self._currentFilePath))
deltaFilePath = None
if deltaPath is not None:
deltaFilePath = os.path.join(deltaPath, fileName)
continueProcessing = True
try:
for configItem in configItems:
if self._check_for_stop():
break
self._open_file(configItem.module, deltaFilePath)
#
# Synchronus delegation to the measure module defined in the config file
#
configItem.module.process_file(self._currentFilePath,
self._currentFileIterator,
configItem, numFilesInFolder,
self.file_measured_callback)
except utils.FileMeasureError, e:
trace.traceback(2)
self._currentFileErrors.append(
uistrings.STR_ErrorMeasuringFile.format(
self._currentFilePath, str(e)))
continueProcessing = not options.breakOnError
def _open_file(self, filePath, oldFileHandle=None):
'''
Return the requested fileObject if criteria are met
'''
tryToOpen = True
newFileHandle = None
# Check for extensions
if self._ignoreNonCode and (filetype.is_noncode_ext(filePath)):
trace.file(1, "Skipping, non-code ext: {0}".format(filePath))
tryToOpen = False
# Check for size threshold
elif self._sizeThreshold > 0:
fileSize = utils.get_file_size(filePath)
if self._sizeThreshold < fileSize:
trace.file(1, "Skipping, size {0}: {1}".format(fileSize, filePath))
tryToOpen = False
if tryToOpen:
# Open the file if it hasn't been opened, otherwise reset it
if not oldFileHandle:
# Use a universal open with line buffering to support binary files and
# reduce the cost of open on larger files
newFileHandle = open(filePath, 'rU', 1)
else:
newFileHandle = oldFileHandle
newFileHandle.seek(0) # Reset the file
# Do tests that look at start of the file
keepFileOpen = False
if self._ignoreNonCode and filetype.is_noncode_file(newFileHandle):
trace.file(1, "Skipping, non-code start: {0}".format(filePath))
elif self._ignoreBinary and not filetype.is_text_file(newFileHandle):
trace.file(1, "Skipping, binary char: {0}".format(filePath))
else:
keepFileOpen = True
if not keepFileOpen:
# If we were NOT passed an existing file handle, close what we opened
if not oldFileHandle and newFileHandle:
newFileHandle.close()
newFileHandle = None
return newFileHandle
def process_file(self, filePath, fileLines,
configEntry,
numSameFiles,
file_measured_callback):
'''
Inherited modules use the default implementation of process_file
to handle calling _survey and packaging results, including any
file metadata
'''
utils.timing_set('FILE_MEASURE_TIME')
trace.file(2, "process_file: {0} {1}".format(self.__class__.__name__, filePath))
trace.file(3, " config: {0}".format(str(configEntry)))
# Stash path for error handling in derived classes
self._currentPath = utils.SurveyorPathParser(filePath)
# Does the config measure filter need to be overridden?
if self._measureFilter is not None:
configEntry.new_measure_filter(self._measureFilter)
# Measurements (whole file metrics) will be stored in a dictionary
# Pack measurement data with file metadata
measurements = {}
self._pack_metadata_into_measures(configEntry, numSameFiles, measurements)
# Analysis items (per line items) are a list of dictionaries
analysis = []
#
# Delegate the survey work to specializations
#
measureResults = {}
analysisResults = []
if self._survey(fileLines, configEntry, measurements, analysis):
# Pack measurements that match our measure filter
for measureName, measure in measurements.iteritems():
if self.match_measure(measureName, configEntry.measureFilters):
measureResults[measureName] = measure
# Pack analysis items into a list of dictionaries for return to app
# We only send analysis items that match filter
for analysisItem in analysis:
analysisRow = {}
for itemName, itemValue in analysisItem.iteritems():
if self.match_measure(itemName, configEntry.measureFilters):
analysisRow[itemName] = itemValue
if analysisRow:
analysisResults.append(analysisRow)
# If this is a delta comparison and there are no lines, it means the
# delta file is an exact dupe
if not fileLines and self._deltaFilePath:
measureResults[METADATA_DUPE_PATH] = self._deltaFilePath
# Add timing info
if self.match_measure(METADATA_TIMING, configEntry.measureFilters):
measureResults[METADATA_TIMING] = "{0:.4f}".format(utils.timing_get('FILE_MEASURE_TIME'))
self._currentPath = None
self._deltaFilePath = None
# Send data back to the caller (jobworker.Worker in default framework)
file_measured_callback(filePath, measureResults, analysisResults)
def _survey_lines(self, linesToSurvey, params, measurements, analysis):
'''
Analyze file line by line. linesToSurvey is an iterable set of lines.
Processing is driven by the regular expressions in member variables.
The order of processing each line is:
- Preprocess line string
- Detect machine vs. human code
- Detect blank lines
- Detect single and multi-line comments
- Capture line measures
- Peform line processing (searches, routines, etc.)
'''
# Setup dictionary for measures and searches we'll do
self._survey_start(params)
# If no lines to process, we may still want to output empty measures
if linesToSurvey is None:
linesToSurvey = []
# Track whether we are inside a multi-line comment - we ignore nesting
scanningMultiLine = False
# If we have a line seperator, apply it
for bufferLine in linesToSurvey:
self.counts['RawLines'][self._activeBlock] += 1
if self._traceLevel: trace.file(4, "Raw: {0}".format(bufferLine))
# Allow specializations to special-case certain lines
if self._alternate_line_processing(bufferLine):
continue
lines = [bufferLine]
if self.addLineSep is not None:
lines = bufferLine.split(self.addLineSep)
#
# Read through the file lines and process them one at a time
# This is the main processing loop for all csmodules derived from NBNC
#
try:
for rawLine in lines:
self.counts['TotalLines'][self._activeBlock] += 1
# Allow for clean up of artifacts or other pre-processing
line = self._preprocess_line(rawLine)
# Detect true blank lines
if self.reTrueBlankLine.match(line):
self.counts['TrueBlankLines'][self._activeBlock] += 1
self._trace_line(line, "T")
continue
# Block Detection
if len(self.blockDetectors) > 1:
if self._detect_block_change(line, analysis):
scanningMultiLine = False # Don't allow multi-line comment to span blocks
# Determine comment state
# This is done before blank lines to make sure we consider multi-line
# comment syntax that will be counted as "blank", e.g., /* on it's own line
onCommentLine, scanningMultiLine = self._detect_line_comment(
line, scanningMultiLine)
# Detect blank lines
if self._detect_blank_line(line):
continue
# Measure and analyze -- overriden in derived classes
self._measure_line(line, onCommentLine)
self._analyze_line(line, analysis, onCommentLine)
except Exception, e:
trace.traceback()
raise utils.FileMeasureError(
"Problem processing line: {0} with module: {1}\n{2}".
format(str(sum(self.counts['RawLines'])),
self.__class__.__name__, str(e)))
def _survey_lines(self, linesToSurvey, params, measurements, analysis):
'''
Analyze file line by line. linesToSurvey is an iterable set of lines.
Processing is driven by the regular expressions in member variables.
The order of processing each line is:
- Preprocess line string
- Detect machine vs. human code
- Detect blank lines
- Detect single and multi-line comments
- Capture line measures
- Peform line processing (searches, routines, etc.)
'''
# Setup dictionary for measures and searches we'll do
self._survey_start(params)
# If no lines to process, we may still want to output empty measures
if linesToSurvey is None:
linesToSurvey = []
# Track whether we are inside a multi-line comment - we ignore nesting
scanningMultiLine = False
# Loop through the raw lines we were passed
for bufferLine in linesToSurvey:
self.counts['RawLines'][self._activeBlock] += 1
if self._traceLevel: trace.file(4, "Raw: {0}".format(bufferLine))
# Allow specializations to skip and/or special-case certain lines
if self._alternate_line_processing(bufferLine):
continue
# If we have a line seperator, apply it
lines = [bufferLine]
if self.addLineSep is not None:
lines = bufferLine.split(self.addLineSep)
#
# Read through the lines to measure and process them one at a time
# This is the main measure loop for csmodules derived from NBNC
#
try:
for rawLine in lines:
self.counts['TotalLines'][self._activeBlock] += 1
# Allow for clean up of artifacts or other pre-processing
line = self._preprocess_line(rawLine)
# Detect true blank lines
if self.reTrueBlankLine.match(line):
self.counts['TrueBlankLines'][self._activeBlock] += 1
self._trace_line(line, "T")
continue
# Block Detection
if len(self.blockDetectors) > 1:
if self._detect_block_change(line, analysis):
scanningMultiLine = False # Don't allow multi-line comment to span blocks
# Determine comment state
# This is done before blank lines to make sure we consider multi-line
# comment syntax that will be counted as "blank", e.g., /* on it's own line
onCommentLine, scanningMultiLine = self._detect_line_comment(line, scanningMultiLine)
# Detect "blank" lines with no useful info
if self._detect_blank_line(line):
continue
# Measure and analyze -- overriden in derived classes
self._measure_line(line, onCommentLine)
self._analyze_line(line, analysis, onCommentLine)
except Exception, e:
trace.traceback()
raise utils.FileMeasureError(
"Problem processing line: {0} with module: {1}\n{2}".format(
str(sum(self.counts['RawLines'])), self.__class__.__name__, str(e)))
def is_noncode_ext(filePath):
rv = False
if is_compressed_ext(filePath) or _has_ext(filePath, NonCodeFileExtensions):
rv = True
trace.file(3, " NonCodeExt: {0}".format(os.path.basename(filePath)))
return rv