def evaluate(self,
n=1000,
successAndPrecisionPlotName='',
histogramPlot=''):
'''
Evaluate the dataset
:return: accuracy and precision
'''
completeMetricDict = dict()
plotMetricsDict = dict()
for run, experimentName in zip(self.listOfExperiments,
self.experimentNames):
fullP = 0
fullS = 0
ll = 0
experimentDict = dict()
for expName, experiment in run.data.iteritems():
x_p_var = np.zeros((n, 1))
y_p_var = np.zeros((n, 1))
x_s_var = np.zeros((n, 1))
y_s_var = np.zeros((n, 1))
averageP = 0
averageS = 0
l = 0
for videoData in experiment.data:
gt = [
x for x in self.dataset.data if x[0] == videoData[0]
][0]
for expRunIndex in range(0, len(videoData[1])):
(x_pr, y_pr, x_s, y_s) = Evaluator.evaluateSingleVideo(
videoData, gt, experimentNumber=expRunIndex, n=n)
x_p_var = x_p_var + x_pr
y_p_var = y_p_var + y_pr
x_s_var = x_s_var + x_s
y_s_var = y_s_var + y_s
l = l + 1
(p1, s1) = Evaluator.getIntegralValues(
x_pr, y_pr, x_s, y_s)
averageP = averageP + p1
averageS = averageS + s1
if expName != 'default':
fullP = fullP + p1
fullS = fullS + s1
ll = ll + 1
x_p_var = x_p_var / (float(l))
y_p_var = y_p_var / (float(l))
x_s_var = x_s_var / (float(l))
y_s_var = y_s_var / (float(l))
averageP = averageP / (float(l))
averageS = averageS / (float(l))
experimentDict[expName] = (x_p_var, y_p_var, x_s_var, y_s_var,
averageP, averageS)
fullP = fullP / (float(ll))
fullS = fullS / (float(ll))
completeMetricDict[experimentName] = (fullP, fullS)
plotMetricsDict[experimentName] = experimentDict
print experimentName
print "===================="
print "Precision: ", fullP
print "Success: ", fullS
for key, value in experimentDict.iteritems():
print key
print value[4], value[5]
print "===================="
# runEvaluation is
self.createPlot(plotMetricsDict,
completeMetricDict,
savefilename=successAndPrecisionPlotName)
self.createHistogramPlot(plotMetricsDict,
completeMetricDict,
savefilename=histogramPlot)
def evaluate(self, n=1000, successAndPrecisionPlotName='', histogramPlot=''):
'''
Evaluate the dataset
:return: accuracy and precision
'''
completeMetricDict=dict()
plotMetricsDict=dict()
for run,experimentName in zip(self.listOfExperiments,self.experimentNames):
fullP=0
fullS=0
ll=0
experimentDict=dict()
for expName, experiment in run.data.iteritems():
x_p_var=np.zeros((n,1))
y_p_var= np.zeros((n, 1))
x_s_var= np.zeros((n, 1))
y_s_var= np.zeros((n, 1))
averageP=0
averageS=0
l=0
for videoData in experiment.data:
gt = [x for x in self.dataset.data if x[0] == videoData[0]][0]
for expRunIndex in range(0, len(videoData[1])):
(x_pr, y_pr, x_s, y_s) = Evaluator.evaluateSingleVideo(videoData, gt, experimentNumber=expRunIndex,
n=n)
x_p_var= x_p_var+x_pr
y_p_var= y_p_var+y_pr
x_s_var=x_s_var+x_s
y_s_var=y_s_var+y_s
l=l+1
(p1,s1)=Evaluator.getIntegralValues(x_pr,y_pr,x_s,y_s)
averageP=averageP+p1
averageS=averageS+s1
if expName!='default':
fullP= fullP+p1
fullS=fullS+s1;
ll=ll+1
x_p_var=x_p_var/(float(l))
y_p_var = y_p_var / (float(l))
x_s_var = x_s_var / (float(l))
y_s_var = y_s_var / (float(l))
averageP= averageP/(float(l))
averageS=averageS/(float(l))
experimentDict[expName]=(x_p_var,y_p_var,x_s_var,y_s_var,averageP,averageS)
fullP= fullP/(float(ll))
fullS = fullS / (float(ll))
completeMetricDict[experimentName]=(fullP,fullS)
plotMetricsDict[experimentName]=experimentDict
print experimentName
print "===================="
print "Precision: ",fullP
print "Success: ",fullS
for key,value in experimentDict.iteritems():
print key
print value[4],value[5]
print "===================="
# runEvaluation is
self.createPlot(plotMetricsDict,completeMetricDict,savefilename=successAndPrecisionPlotName)
self.createHistogramPlot(plotMetricsDict, completeMetricDict, savefilename=histogramPlot)
def calculateMetricsAndSave(self, savePath, n=1000):
completeMetricDict = dict()
plotMetricsDict = dict()
for run, experimentName in zip(self.listOfExperiments,
self.experimentNames):
fullP = 0
fullS = 0
ll = 0
experimentDict = dict()
for expName, experiment in run.data.iteritems():
# if there is only one experiment
if len(experiment.data) == 0:
continue
x_p_var = np.zeros(n)
y_p_var = np.zeros(n)
x_s_var = np.zeros(n)
y_s_var = np.zeros(n)
averageP = 0
averageS = 0
l = 0
for videoData in experiment.data:
gt = [
x for x in self.dataset.data if x[0] == videoData[0]
][0]
for expRunIndex in range(0, len(videoData[1])):
(x_pr, y_pr, x_s, y_s) = Evaluator.evaluateSingleVideo(
videoData, gt, experimentNumber=expRunIndex, n=n)
x_p_var = np.add(x_p_var, x_pr)
y_p_var = np.add(y_p_var, y_pr)
x_s_var = np.add(x_s_var, x_s)
y_s_var = np.add(y_s_var, y_s)
l = l + 1
(p1, s1) = Evaluator.getIntegralValues(
x_pr, y_pr, x_s, y_s)
averageP = averageP + p1
averageS = averageS + s1
if expName != 'default':
fullP = fullP + p1
fullS = fullS + s1
ll = ll + 1
x_p_var = x_p_var / (float(l))
y_p_var = y_p_var / (float(l))
x_s_var = x_s_var / (float(l))
y_s_var = y_s_var / (float(l))
averageP = averageP / (float(l))
averageS = averageS / (float(l))
experimentDict[expName] = (x_p_var, y_p_var, x_s_var, y_s_var,
averageP, averageS)
if ll == 0:
fullP = 0
fullS = 0
else:
fullP = fullP / (float(ll))
fullS = fullS / (float(ll))
completeMetricDict[experimentName] = (fullP, fullS)
plotMetricsDict[experimentName] = experimentDict
print experimentName
print "===================="
print "Precision: ", fullP
print "Success: ", fullS
for key, value in experimentDict.iteritems():
print key
print value[4], value[5]
print "===================="
e = Evaluated(plotMetricsDict[experimentName],
completeMetricDict[experimentName], experimentName)
e.save(savePath + "/" + experimentName + ".p")
def calculateMetricsAndSave(self,savePath,n=1000):
completeMetricDict = dict()
plotMetricsDict = dict()
for run, experimentName in zip(self.listOfExperiments, self.experimentNames):
fullP = 0
fullS = 0
ll = 0
experimentDict = dict()
for expName, experiment in run.data.iteritems():
x_p_var = np.zeros(n)
y_p_var = np.zeros(n)
x_s_var = np.zeros(n)
y_s_var = np.zeros(n)
averageP = 0
averageS = 0
l = 0
for videoData in experiment.data:
gt = [x for x in self.dataset.data if x[0] == videoData[0]][0]
for expRunIndex in range(0, len(videoData[1])):
(x_pr, y_pr, x_s, y_s) = Evaluator.evaluateSingleVideo(videoData, gt,
experimentNumber=expRunIndex,
n=n)
x_p_var =np.add(x_p_var, x_pr)
y_p_var = np.add(y_p_var,y_pr)
x_s_var = np.add(x_s_var,x_s)
y_s_var = np.add(y_s_var,y_s)
l = l + 1
(p1, s1) = Evaluator.getIntegralValues(x_pr, y_pr, x_s, y_s)
averageP = averageP + p1
averageS = averageS + s1
if expName != 'default':
fullP = fullP + p1
fullS = fullS + s1;
ll = ll + 1
x_p_var = x_p_var / (float(l))
y_p_var = y_p_var / (float(l))
x_s_var = x_s_var / (float(l))
y_s_var = y_s_var / (float(l))
averageP = averageP / (float(l))
averageS = averageS / (float(l))
experimentDict[expName] = (x_p_var, y_p_var, x_s_var, y_s_var, averageP, averageS)
fullP = fullP / (float(ll))
fullS = fullS / (float(ll))
completeMetricDict[experimentName] = (fullP, fullS)
plotMetricsDict[experimentName] = experimentDict
print experimentName
print "===================="
print "Precision: ", fullP
print "Success: ", fullS
for key, value in experimentDict.iteritems():
print key
print value[4], value[5]
print "===================="
e=Evaluated(plotMetricsDict[experimentName],completeMetricDict[experimentName],experimentName)
e.save(savePath+"/"+experimentName+".p")
