def test_1tp_IoU(self):
'''
Check different IoU_thresholds.
1 tp. AP should be 1. IoU with GT = 0.5.
'''
c = self.conn.cursor()
# Shifted by dx=10 (given width=30)
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",0,40+10,20,30,10,"name",1.0)')
c_gt = self.conn_gt.cursor()
# IoU_thresh=0.49.
self.args.IoU_thresh = 0.49
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 1., msg='pascal, IoU_thresh=0.49.')
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 1., places=4, msg='sklearn, IoU_thr=0.49.')
# IoU_thresh=0.51.
self.args.IoU_thresh = 0.51
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 0., msg='pascal, IoU_thresh=0.51.')
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 0., places=4, msg='sklearn, IoU_thr=0.51.')
def test_precRecallIsTriangle(self):
'''
Precision-recall plot is a triangle made with self.N different thresholds.
At each threshold one fn becomes a tp, and one fp is added.
'''
c_gt = self.conn_gt.cursor()
c = self.conn.cursor()
for i in range(self.N):
imagefile = 'image%d' % i
c.execute('INSERT INTO images(imagefile) VALUES (?)',
(imagefile, ))
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES (?,?,40,20,40,20,"name",?)',
(imagefile, 2 * i, 1. - i / self.N))
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES (?,?,140,20,40,20,"name",?)',
(imagefile, 2 * i + 1, 1. - i / self.N))
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertAlmostEqual(aps, 0.5, places=1, msg='pascal')
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 0.5, places=1, msg='sklearn')
def test_2tp_2fp_atSeveralThresh(self):
'''
1 tp, @thres=0.2.
1 tp, 2 fp @thres=0.1;
prec(0.5)=1, prec(1)=0.5.
Sklearn: AP should be 1 (don't know why)
Pascal: AP should be 0.75.
'''
c = self.conn.cursor()
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",0,40,20,40,20,"name",0.2)')
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image1",1,40,20,40,20,"name",0.1)')
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",2,140,20,40,20,"name",0.1)')
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",3,140,20,40,20,"name",0.1)')
c_gt = self.conn_gt.cursor()
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 1.0, msg='pascal') # Not sure why 1.0.
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 0.75, places=4, msg='sklearn')
def test_0gt_0det(self):
''' 0 gt, 0 det. Pascal AP should be 0. Sklearn AP should be NaN. '''
c = self.conn.cursor()
c_gt = self.conn_gt.cursor()
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 0.)
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertTrue(np.isnan(aps))
def test_1fn(self):
''' pascal, 1 fn. AP should be 0. '''
c = self.conn.cursor()
c_gt = self.conn_gt.cursor()
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 0.)
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 0., places=4)
def test_1tp(self):
''' 1 tp. AP should be 1. '''
c = self.conn.cursor()
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",0,40,20,30,10,"name",1.0)')
c_gt = self.conn_gt.cursor()
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 1.)
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertEqual(aps, 1.)
def test_1tp_1fn(self):
''' 1 tp, 1 fn. prec(0.5)=1, prec(1)=0. AP should be 0.5. '''
c = self.conn.cursor()
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",0,40,20,40,20,"name",0.5)')
c_gt = self.conn_gt.cursor()
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 0.5)
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 0.5, places=4)
def test_0gt_1fp(self):
''' 0 gt, 1 fp. AP should be NaN. '''
c = self.conn.cursor()
# Add 1 FP object.
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",0,40,20,40,30,"name",1.0)')
c_gt = self.conn_gt.cursor()
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertTrue(np.isnan(aps))
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertTrue(np.isnan(aps))
def test_1tp_whereObjectsGt(self):
''' 1 tp, using where_gt_objects to keep only 1 GT. AP should be 1. '''
c = self.conn.cursor()
c.execute(
'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) '
'VALUES ("image0",0,40,20,40,20,"name",0.5)')
c_gt = self.conn_gt.cursor()
self.args.where_object_gt = 'imagefile="image0"'
aps = dbEvaluate._evaluateDetectionForClassPascal(
c, c_gt, 'name', self.args)
self.assertEqual(aps, 1)
aps = dbEvaluate._evaluateDetectionForClassSklearn(
c, c_gt, 'name', self.args, sklearn)
self.assertAlmostEqual(aps, 1, places=4)