def onMenuItemOpenActivate(self, widget):
if self.lastImage == None:
return # Can't load until we have an image
filename = self.chooseMarkerFile()
if filename != None:
markerBuffer = MarkerBuffer.loadMarkerBuffer(filename)
if markerBuffer != None:
blockWidth = self.lastImage.width / (markerBuffer.shape[1] + 1)
blockHeight = self.lastImage.height / (markerBuffer.shape[0] +
1)
if blockHeight < self.adjBlockHeight.get_upper() \
and blockWidth < self.adjBlockWidth.get_upper():
self.adjBlockWidth.set_value(blockWidth)
self.adjBlockHeight.set_value(blockHeight)
self.markerBuffer = markerBuffer
# Successful so remember filename
self.filename = filename
else:
print "Error: Invalid block width or height, please fix with Glade"
def evaluateClassifier( markerFilename, bagFilenames,
rocGraphFilename = None, accuracyGraphFilename = None,
gaussianStdDev = 0.0 ):
'''Takes a list of bag files in as data for the classifier and evaluates
the classifer against a ground truth marker file. Returns the average AUC
and optimal threshold for the classifier.'''
# Load in marker buffer for evaluating classifier
markerBuffer = MarkerBuffer.loadMarkerBuffer( markerFilename )
if markerBuffer == None:
print "Error: Unable to load marker buffer -", markerFilename
# Process each bag file in turn
dataList = []
for bagFilename in bagFilenames:
print "Reading in bag file -", bagFilename
inputSequence = InputSequence( bagFilename )
if ADD_DISTRACTORS:
distractors = [
Distractor( radius=24, startPos=( 25, 35 ), endPos=( 100, 100 ), frequency=2.0 ),
Distractor( radius=24, startPos=( 200, 200 ), endPos=( 150, 50 ), frequency=0.25 ),
Distractor( radius=24, startPos=( 188, 130 ), endPos=( 168, 258 ), frequency=0.6 ),
Distractor( radius=24, startPos=( 63, 94 ), endPos=( 170, 81 ), frequency=1.5 ),
Distractor( radius=24, startPos=( 40, 287 ), endPos=( 50, 197 ), frequency=3.0 ) ]
inputSequence.addDistractorObjects( distractors )
inputSequence.calculateOpticalFlow(
OPTICAL_FLOW_BLOCK_WIDTH, OPTICAL_FLOW_BLOCK_HEIGHT,
OPTICAL_FLOW_RANGE_WIDTH, OPTICAL_FLOW_RANGE_HEIGHT,
OPTICAL_FLOW_METHOD )
#print "Resampling data"
regularisedInputSequence = RegularisedInputSequence(
inputSequence, SAMPLES_PER_SECOND )
if gaussianStdDev > 0.0:
regularisedInputSequence.smoothOpticalFlow( gaussianStdDev )
#print "Performing cross correlation"
crossCorrelatedSequence = CrossCorrelatedSequence(
regularisedInputSequence,
MAX_CORRELATION_LAG, COMBINATION_METHOD )
#print "Building ROC Curve"
rocCurve = GripperDetectorROCCurve( crossCorrelatedSequence, markerBuffer )
print "__AUC =", rocCurve.areaUnderCurve
dataList.append( WorkingData( bagFilename, inputSequence,
regularisedInputSequence, crossCorrelatedSequence, rocCurve ) )
# Average results
if len( dataList ) > 1:
curveList = [ data.rocCurve for data in dataList ]
averageROCCurve, varianceROCCurve = ROCCurve.averageROCCurves( curveList )
else:
averageROCCurve = dataList[ 0 ].rocCurve
varianceROCCurve = None
# Plot ROC Curve
figureROC = Figure( figsize=(8,6), dpi=72 )
canvasROC = FigureCanvas( figureROC )
axisROC = figureROC.add_subplot( 111 )
axisROC.plot( averageROCCurve.falsePositiveRates, averageROCCurve.truePositiveRates )
axisROC.set_xlabel( 'False Positive Rate' )
axisROC.set_ylabel( 'True Positive Rate' )
# Add error bars
if varianceROCCurve != None:
diffBetweenErrorBars = 0.025 #1.0/(NUM_ERROR_BARS)
lastFPRValue = averageROCCurve.falsePositiveRates[ 0 ]
errorFPR = []
errorTPR = []
errorErrFPR = []
errorErrTPR = []
for i in range( len( averageROCCurve.falsePositiveRates ) ):
curFPRValue = averageROCCurve.falsePositiveRates[ i ]
if abs( curFPRValue - lastFPRValue ) >= diffBetweenErrorBars:
lastFPRValue = curFPRValue
errorFPR.append( averageROCCurve.falsePositiveRates[ i ] )
errorTPR.append( averageROCCurve.truePositiveRates[ i ] )
errorErrFPR.append( math.sqrt( varianceROCCurve.falsePositiveRates[ i ] )*2.571 )
errorErrTPR.append( math.sqrt( varianceROCCurve.truePositiveRates[ i ] )*2.571 )
axisROC.errorbar( errorFPR, errorTPR,
yerr=errorErrTPR, linestyle='None' )
#xerr=errorErrFPR, yerr=errorErrTPR, linestyle='None' )
#print errorFPR
#print errorTPR
#print lastFPRValue, averageROCCurve.falsePositiveRates[ -1 ]
axisROC.set_xlim( 0.0, 1.0 )
axisROC.set_ylim( 0.0, 1.0 )
# Plot accuracy
figureAccuracy = Figure( figsize=(8,6), dpi=72 )
canvasAccuracy = FigureCanvas( figureAccuracy )
axisAccuracy = figureAccuracy.add_subplot( 111 )
thresholds = averageROCCurve.calculateThresholds()
axisAccuracy.plot( thresholds, averageROCCurve.accuracy )
#for data in dataList:
# axisAccuracy.plot( thresholds, data.rocCurve.accuracy )
# Add error bars
if varianceROCCurve != None:
diffBetweenErrorBars = 1.0/(NUM_ERROR_BARS)
lastThresholdValue = thresholds[ 0 ]
errorThreshold = []
errorAccuracy = []
errorErrAccuracy = []
for i in range( len( thresholds ) ):
curThresholdValue = thresholds[ i ]
if abs( curThresholdValue - lastThresholdValue ) >= diffBetweenErrorBars:
lastThresholdValue = curThresholdValue
errorThreshold.append( curThresholdValue )
errorAccuracy.append( averageROCCurve.accuracy[ i ] )
errorErrAccuracy.append( math.sqrt( varianceROCCurve.accuracy[ i ] )*2.571 )
axisAccuracy.errorbar( errorThreshold, errorAccuracy, yerr=errorErrAccuracy, linestyle='None' )
# Mark the threshold with the maximum accuracy
maxAccuracyThreshold = thresholds[ np.argsort( averageROCCurve.accuracy )[ -1 ] ]
axisAccuracy.axvline( x=float( maxAccuracyThreshold ), color='red' )
axisAccuracy.set_xlabel( 'Threshold' )
axisAccuracy.set_ylabel( 'Accuracy' )
# Save the graphs
if rocGraphFilename != None:
figureROC.savefig( rocGraphFilename )
if accuracyGraphFilename != None:
figureAccuracy.savefig( accuracyGraphFilename )
return averageROCCurve.areaUnderCurve, maxAccuracyThreshold
def saveData(self, filename):
MarkerBuffer.saveMarkerBuffer(self.markerBuffer, filename)
def __init__( self, bagFilename ):
self.scriptPath = os.path.dirname( __file__ )
self.cameraImagePixBuf = None
self.bagFilename = bagFilename
self.lastImageGray = None
# Read in sequence
t1 = time.time()
self.inputSequence = InputSequence( bagFilename )
distractors = [
Distractor( radius=24, startPos=( 25, 35 ), endPos=( 100, 100 ), frequency=2.0 ),
Distractor( radius=24, startPos=( 200, 200 ), endPos=( 150, 50 ), frequency=0.25 ),
Distractor( radius=24, startPos=( 188, 130 ), endPos=( 168, 258 ), frequency=0.6 ),
Distractor( radius=24, startPos=( 63, 94 ), endPos=( 170, 81 ), frequency=1.5 ),
Distractor( radius=24, startPos=( 40, 287 ), endPos=( 50, 197 ), frequency=3.0 ) ]
#self.inputSequence.addDistractorObjects( distractors )
self.inputSequence.calculateOpticalFlow(
self.OPTICAL_FLOW_BLOCK_WIDTH, self.OPTICAL_FLOW_BLOCK_HEIGHT,
self.OPTICAL_FLOW_RANGE_WIDTH, self.OPTICAL_FLOW_RANGE_HEIGHT,
self.OPTICAL_FLOW_METHOD )
t2 = time.time()
print 'Processing sequence took %0.3f ms' % ((t2-t1)*1000.0)
# Resample sequence
t1 = time.time()
self.regularisedInputSequence = RegularisedInputSequence(
self.inputSequence, self.SAMPLES_PER_SECOND )
t2 = time.time()
print 'Resampling took %0.3f ms' % ((t2-t1)*1000.0)
t1 = time.time()
self.crossCorrelatedSequence = CrossCorrelatedSequence(
self.regularisedInputSequence,
self.MAX_CORRELATION_LAG, self.COMBINATION_METHOD )
t2 = time.time()
print 'Correlation took %0.3f ms' % ((t2-t1)*1000.0)
# Detect the input signal based on the correlation in the x and y axis
self.inputSignalDetectedArray = \
self.crossCorrelatedSequence.detectInputSequence( self.CORRELATION_THRESHOLD )
# Build a histogram for the gripper
self.gripperHistogram = cv.CreateHist(
[ 256/8, 256/8, 256/8 ], cv.CV_HIST_ARRAY, [ (0,255), (0,255), (0,255) ], 1 )
firstImage = self.inputSequence.cameraImages[ 0 ]
imageRGB = cv.CreateImageHeader( ( firstImage.shape[ 1 ], firstImage.shape[ 0 ] ), cv.IPL_DEPTH_8U, 3 )
cv.SetData( imageRGB, firstImage.data, firstImage.shape[ 1 ]*3 )
r_plane = cv.CreateMat( imageRGB.height, imageRGB.width, cv.CV_8UC1 )
g_plane = cv.CreateMat( imageRGB.height, imageRGB.width, cv.CV_8UC1 )
b_plane = cv.CreateMat( imageRGB.height, imageRGB.width, cv.CV_8UC1 )
cv.Split( imageRGB, r_plane, g_plane, b_plane, None )
planes = [ r_plane, g_plane, b_plane ]
maskArray = np.zeros(shape=( imageRGB.height, imageRGB.width ), dtype=np.uint8 )
for rowIdx in range( self.inputSignalDetectedArray.shape[ 0 ] ):
for colIdx in range( self.inputSignalDetectedArray.shape[ 1 ] ):
if self.inputSignalDetectedArray[ rowIdx, colIdx ]:
rowStartIdx = rowIdx*self.OPTICAL_FLOW_BLOCK_HEIGHT
rowEndIdx = rowStartIdx + self.OPTICAL_FLOW_BLOCK_HEIGHT
colStartIdx = colIdx*self.OPTICAL_FLOW_BLOCK_WIDTH
colEndIdx = colStartIdx + self.OPTICAL_FLOW_BLOCK_WIDTH
maskArray[ rowStartIdx:rowEndIdx, colStartIdx:colEndIdx ] = 255
cv.CalcHist( [ cv.GetImage( i ) for i in planes ],
self.gripperHistogram, 0, mask=cv.fromarray( maskArray ) )
markerBuffer = MarkerBuffer.loadMarkerBuffer( self.scriptPath + self.GROUND_TRUTH_FILENAME )
if markerBuffer == None:
raise Exception( "Unable to load marker buffer" )
self.rocCurve = GripperDetectorROCCurve( self.crossCorrelatedSequence, markerBuffer )
# Create the matplotlib graph
self.figure = Figure( figsize=(8,6), dpi=72 )
self.axisX = self.figure.add_subplot( 311 )
self.axisY = self.figure.add_subplot( 312 )
self.axisROC = self.figure.add_subplot( 313 )
self.canvas = None # Wait for GUI to be created before creating canvas
self.navToolbar = None
# Setup the GUI
builder = gtk.Builder()
builder.add_from_file( self.scriptPath + "/GUI/OpticalFlowExplorer.glade" )
self.dwgCameraImage = builder.get_object( "dwgCameraImage" )
self.window = builder.get_object( "winMain" )
self.vboxMain = builder.get_object( "vboxMain" )
self.hboxWorkArea = builder.get_object( "hboxWorkArea" )
self.adjTestPointX = builder.get_object( "adjTestPointX" )
self.adjTestPointY = builder.get_object( "adjTestPointY" )
self.adjTestPointX.set_upper( self.MAX_TEST_POINT_X )
self.adjTestPointY.set_upper( self.MAX_TEST_POINT_Y )
self.sequenceControls = builder.get_object( "sequenceControls" )
self.sequenceControls.setNumFrames( len( self.inputSequence.cameraImages ) )
self.sequenceControls.setOnFrameIdxChangedCallback( self.onSequenceControlsFrameIdxChanged )
self.setFrameIdx( 0 )
self.processOpticalFlowData()
builder.connect_signals( self )
updateLoop = self.update()
gobject.idle_add( updateLoop.next )
self.window.show()
self.window.maximize()
def __init__(self, bagFilename):
self.scriptPath = os.path.dirname(__file__)
self.cameraImagePixBuf = None
self.bagFilename = bagFilename
self.lastImageGray = None
# Read in sequence
t1 = time.time()
self.inputSequence = InputSequence(bagFilename)
distractors = [
Distractor(radius=24,
startPos=(25, 35),
endPos=(100, 100),
frequency=2.0),
Distractor(radius=24,
startPos=(200, 200),
endPos=(150, 50),
frequency=0.25),
Distractor(radius=24,
startPos=(188, 130),
endPos=(168, 258),
frequency=0.6),
Distractor(radius=24,
startPos=(63, 94),
endPos=(170, 81),
frequency=1.5),
Distractor(radius=24,
startPos=(40, 287),
endPos=(50, 197),
frequency=3.0)
]
#self.inputSequence.addDistractorObjects( distractors )
self.inputSequence.calculateOpticalFlow(self.OPTICAL_FLOW_BLOCK_WIDTH,
self.OPTICAL_FLOW_BLOCK_HEIGHT,
self.OPTICAL_FLOW_RANGE_WIDTH,
self.OPTICAL_FLOW_RANGE_HEIGHT,
self.OPTICAL_FLOW_METHOD)
t2 = time.time()
print 'Processing sequence took %0.3f ms' % ((t2 - t1) * 1000.0)
# Resample sequence
t1 = time.time()
self.regularisedInputSequence = RegularisedInputSequence(
self.inputSequence, self.SAMPLES_PER_SECOND)
t2 = time.time()
print 'Resampling took %0.3f ms' % ((t2 - t1) * 1000.0)
t1 = time.time()
self.crossCorrelatedSequence = CrossCorrelatedSequence(
self.regularisedInputSequence, self.MAX_CORRELATION_LAG,
self.COMBINATION_METHOD)
t2 = time.time()
print 'Correlation took %0.3f ms' % ((t2 - t1) * 1000.0)
# Detect the input signal based on the correlation in the x and y axis
self.inputSignalDetectedArray = \
self.crossCorrelatedSequence.detectInputSequence( self.CORRELATION_THRESHOLD )
# Build a histogram for the gripper
self.gripperHistogram = cv.CreateHist([256 / 8, 256 / 8, 256 / 8],
cv.CV_HIST_ARRAY, [(0, 255),
(0, 255),
(0, 255)], 1)
firstImage = self.inputSequence.cameraImages[0]
imageRGB = cv.CreateImageHeader(
(firstImage.shape[1], firstImage.shape[0]), cv.IPL_DEPTH_8U, 3)
cv.SetData(imageRGB, firstImage.data, firstImage.shape[1] * 3)
r_plane = cv.CreateMat(imageRGB.height, imageRGB.width, cv.CV_8UC1)
g_plane = cv.CreateMat(imageRGB.height, imageRGB.width, cv.CV_8UC1)
b_plane = cv.CreateMat(imageRGB.height, imageRGB.width, cv.CV_8UC1)
cv.Split(imageRGB, r_plane, g_plane, b_plane, None)
planes = [r_plane, g_plane, b_plane]
maskArray = np.zeros(shape=(imageRGB.height, imageRGB.width),
dtype=np.uint8)
for rowIdx in range(self.inputSignalDetectedArray.shape[0]):
for colIdx in range(self.inputSignalDetectedArray.shape[1]):
if self.inputSignalDetectedArray[rowIdx, colIdx]:
rowStartIdx = rowIdx * self.OPTICAL_FLOW_BLOCK_HEIGHT
rowEndIdx = rowStartIdx + self.OPTICAL_FLOW_BLOCK_HEIGHT
colStartIdx = colIdx * self.OPTICAL_FLOW_BLOCK_WIDTH
colEndIdx = colStartIdx + self.OPTICAL_FLOW_BLOCK_WIDTH
maskArray[rowStartIdx:rowEndIdx,
colStartIdx:colEndIdx] = 255
cv.CalcHist([cv.GetImage(i) for i in planes],
self.gripperHistogram,
0,
mask=cv.fromarray(maskArray))
markerBuffer = MarkerBuffer.loadMarkerBuffer(
self.scriptPath + self.GROUND_TRUTH_FILENAME)
if markerBuffer == None:
raise Exception("Unable to load marker buffer")
self.rocCurve = GripperDetectorROCCurve(self.crossCorrelatedSequence,
markerBuffer)
# Create the matplotlib graph
self.figure = Figure(figsize=(8, 6), dpi=72)
self.axisX = self.figure.add_subplot(311)
self.axisY = self.figure.add_subplot(312)
self.axisROC = self.figure.add_subplot(313)
self.canvas = None # Wait for GUI to be created before creating canvas
self.navToolbar = None
# Setup the GUI
builder = gtk.Builder()
builder.add_from_file(self.scriptPath +
"/GUI/OpticalFlowExplorer.glade")
self.dwgCameraImage = builder.get_object("dwgCameraImage")
self.window = builder.get_object("winMain")
self.vboxMain = builder.get_object("vboxMain")
self.hboxWorkArea = builder.get_object("hboxWorkArea")
self.adjTestPointX = builder.get_object("adjTestPointX")
self.adjTestPointY = builder.get_object("adjTestPointY")
self.adjTestPointX.set_upper(self.MAX_TEST_POINT_X)
self.adjTestPointY.set_upper(self.MAX_TEST_POINT_Y)
self.sequenceControls = builder.get_object("sequenceControls")
self.sequenceControls.setNumFrames(len(
self.inputSequence.cameraImages))
self.sequenceControls.setOnFrameIdxChangedCallback(
self.onSequenceControlsFrameIdxChanged)
self.setFrameIdx(0)
self.processOpticalFlowData()
builder.connect_signals(self)
updateLoop = self.update()
gobject.idle_add(updateLoop.next)
self.window.show()
self.window.maximize()
def evaluateClassifier(markerFilename,
bagFilenames,
rocGraphFilename=None,
accuracyGraphFilename=None,
gaussianStdDev=0.0):
'''Takes a list of bag files in as data for the classifier and evaluates
the classifer against a ground truth marker file. Returns the average AUC
and optimal threshold for the classifier.'''
# Load in marker buffer for evaluating classifier
markerBuffer = MarkerBuffer.loadMarkerBuffer(markerFilename)
if markerBuffer == None:
print "Error: Unable to load marker buffer -", markerFilename
# Process each bag file in turn
dataList = []
for bagFilename in bagFilenames:
print "Reading in bag file -", bagFilename
inputSequence = InputSequence(bagFilename)
if ADD_DISTRACTORS:
distractors = [
Distractor(radius=24,
startPos=(25, 35),
endPos=(100, 100),
frequency=2.0),
Distractor(radius=24,
startPos=(200, 200),
endPos=(150, 50),
frequency=0.25),
Distractor(radius=24,
startPos=(188, 130),
endPos=(168, 258),
frequency=0.6),
Distractor(radius=24,
startPos=(63, 94),
endPos=(170, 81),
frequency=1.5),
Distractor(radius=24,
startPos=(40, 287),
endPos=(50, 197),
frequency=3.0)
]
inputSequence.addDistractorObjects(distractors)
inputSequence.calculateOpticalFlow(OPTICAL_FLOW_BLOCK_WIDTH,
OPTICAL_FLOW_BLOCK_HEIGHT,
OPTICAL_FLOW_RANGE_WIDTH,
OPTICAL_FLOW_RANGE_HEIGHT,
OPTICAL_FLOW_METHOD)
#print "Resampling data"
regularisedInputSequence = RegularisedInputSequence(
inputSequence, SAMPLES_PER_SECOND)
if gaussianStdDev > 0.0:
regularisedInputSequence.smoothOpticalFlow(gaussianStdDev)
#print "Performing cross correlation"
crossCorrelatedSequence = CrossCorrelatedSequence(
regularisedInputSequence, MAX_CORRELATION_LAG, COMBINATION_METHOD)
#print "Building ROC Curve"
rocCurve = GripperDetectorROCCurve(crossCorrelatedSequence,
markerBuffer)
print "__AUC =", rocCurve.areaUnderCurve
dataList.append(
WorkingData(bagFilename, inputSequence, regularisedInputSequence,
crossCorrelatedSequence, rocCurve))
# Average results
if len(dataList) > 1:
curveList = [data.rocCurve for data in dataList]
averageROCCurve, varianceROCCurve = ROCCurve.averageROCCurves(
curveList)
else:
averageROCCurve = dataList[0].rocCurve
varianceROCCurve = None
# Plot ROC Curve
figureROC = Figure(figsize=(8, 6), dpi=72)
canvasROC = FigureCanvas(figureROC)
axisROC = figureROC.add_subplot(111)
axisROC.plot(averageROCCurve.falsePositiveRates,
averageROCCurve.truePositiveRates)
axisROC.set_xlabel('False Positive Rate')
axisROC.set_ylabel('True Positive Rate')
# Add error bars
if varianceROCCurve != None:
diffBetweenErrorBars = 0.025 #1.0/(NUM_ERROR_BARS)
lastFPRValue = averageROCCurve.falsePositiveRates[0]
errorFPR = []
errorTPR = []
errorErrFPR = []
errorErrTPR = []
for i in range(len(averageROCCurve.falsePositiveRates)):
curFPRValue = averageROCCurve.falsePositiveRates[i]
if abs(curFPRValue - lastFPRValue) >= diffBetweenErrorBars:
lastFPRValue = curFPRValue
errorFPR.append(averageROCCurve.falsePositiveRates[i])
errorTPR.append(averageROCCurve.truePositiveRates[i])
errorErrFPR.append(
math.sqrt(varianceROCCurve.falsePositiveRates[i]) * 2.571)
errorErrTPR.append(
math.sqrt(varianceROCCurve.truePositiveRates[i]) * 2.571)
axisROC.errorbar(errorFPR,
errorTPR,
yerr=errorErrTPR,
linestyle='None')
#xerr=errorErrFPR, yerr=errorErrTPR, linestyle='None' )
#print errorFPR
#print errorTPR
#print lastFPRValue, averageROCCurve.falsePositiveRates[ -1 ]
axisROC.set_xlim(0.0, 1.0)
axisROC.set_ylim(0.0, 1.0)
# Plot accuracy
figureAccuracy = Figure(figsize=(8, 6), dpi=72)
canvasAccuracy = FigureCanvas(figureAccuracy)
axisAccuracy = figureAccuracy.add_subplot(111)
thresholds = averageROCCurve.calculateThresholds()
axisAccuracy.plot(thresholds, averageROCCurve.accuracy)
#for data in dataList:
# axisAccuracy.plot( thresholds, data.rocCurve.accuracy )
# Add error bars
if varianceROCCurve != None:
diffBetweenErrorBars = 1.0 / (NUM_ERROR_BARS)
lastThresholdValue = thresholds[0]
errorThreshold = []
errorAccuracy = []
errorErrAccuracy = []
for i in range(len(thresholds)):
curThresholdValue = thresholds[i]
if abs(curThresholdValue -
lastThresholdValue) >= diffBetweenErrorBars:
lastThresholdValue = curThresholdValue
errorThreshold.append(curThresholdValue)
errorAccuracy.append(averageROCCurve.accuracy[i])
errorErrAccuracy.append(
math.sqrt(varianceROCCurve.accuracy[i]) * 2.571)
axisAccuracy.errorbar(errorThreshold,
errorAccuracy,
yerr=errorErrAccuracy,
linestyle='None')
# Mark the threshold with the maximum accuracy
maxAccuracyThreshold = thresholds[np.argsort(averageROCCurve.accuracy)[-1]]
axisAccuracy.axvline(x=float(maxAccuracyThreshold), color='red')
axisAccuracy.set_xlabel('Threshold')
axisAccuracy.set_ylabel('Accuracy')
# Save the graphs
if rocGraphFilename != None:
figureROC.savefig(rocGraphFilename)
if accuracyGraphFilename != None:
figureAccuracy.savefig(accuracyGraphFilename)
return averageROCCurve.areaUnderCurve, maxAccuracyThreshold
