def test_1tp_IoU(self): ''' Check different IoU_thresholds. 1 tp. AP should be 1. IoU with GT = 0.5. ''' c = self.conn.cursor() # Shifted by dx=10 (given width=30) c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",0,40+10,20,30,10,"name",1.0)') c_gt = self.conn_gt.cursor() # IoU_thresh=0.49. self.args.IoU_thresh = 0.49 aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 1., msg='pascal, IoU_thresh=0.49.') aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 1., places=4, msg='sklearn, IoU_thr=0.49.') # IoU_thresh=0.51. self.args.IoU_thresh = 0.51 aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 0., msg='pascal, IoU_thresh=0.51.') aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 0., places=4, msg='sklearn, IoU_thr=0.51.')
def test_precRecallIsTriangle(self): ''' Precision-recall plot is a triangle made with self.N different thresholds. At each threshold one fn becomes a tp, and one fp is added. ''' c_gt = self.conn_gt.cursor() c = self.conn.cursor() for i in range(self.N): imagefile = 'image%d' % i c.execute('INSERT INTO images(imagefile) VALUES (?)', (imagefile, )) c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES (?,?,40,20,40,20,"name",?)', (imagefile, 2 * i, 1. - i / self.N)) c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES (?,?,140,20,40,20,"name",?)', (imagefile, 2 * i + 1, 1. - i / self.N)) aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertAlmostEqual(aps, 0.5, places=1, msg='pascal') aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 0.5, places=1, msg='sklearn')
def test_2tp_2fp_atSeveralThresh(self): ''' 1 tp, @thres=0.2. 1 tp, 2 fp @thres=0.1; prec(0.5)=1, prec(1)=0.5. Sklearn: AP should be 1 (don't know why) Pascal: AP should be 0.75. ''' c = self.conn.cursor() c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",0,40,20,40,20,"name",0.2)') c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image1",1,40,20,40,20,"name",0.1)') c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",2,140,20,40,20,"name",0.1)') c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",3,140,20,40,20,"name",0.1)') c_gt = self.conn_gt.cursor() aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 1.0, msg='pascal') # Not sure why 1.0. aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 0.75, places=4, msg='sklearn')
def test_0gt_0det(self): ''' 0 gt, 0 det. Pascal AP should be 0. Sklearn AP should be NaN. ''' c = self.conn.cursor() c_gt = self.conn_gt.cursor() aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 0.) aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertTrue(np.isnan(aps))
def test_1fn(self): ''' pascal, 1 fn. AP should be 0. ''' c = self.conn.cursor() c_gt = self.conn_gt.cursor() aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 0.) aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 0., places=4)
def test_1tp(self): ''' 1 tp. AP should be 1. ''' c = self.conn.cursor() c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",0,40,20,30,10,"name",1.0)') c_gt = self.conn_gt.cursor() aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 1.) aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertEqual(aps, 1.)
def test_1tp_1fn(self): ''' 1 tp, 1 fn. prec(0.5)=1, prec(1)=0. AP should be 0.5. ''' c = self.conn.cursor() c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",0,40,20,40,20,"name",0.5)') c_gt = self.conn_gt.cursor() aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 0.5) aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 0.5, places=4)
def test_0gt_1fp(self): ''' 0 gt, 1 fp. AP should be NaN. ''' c = self.conn.cursor() # Add 1 FP object. c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",0,40,20,40,30,"name",1.0)') c_gt = self.conn_gt.cursor() aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertTrue(np.isnan(aps)) aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertTrue(np.isnan(aps))
def test_1tp_whereObjectsGt(self): ''' 1 tp, using where_gt_objects to keep only 1 GT. AP should be 1. ''' c = self.conn.cursor() c.execute( 'INSERT INTO objects(imagefile,objectid,x1,y1,width,height,name,score) ' 'VALUES ("image0",0,40,20,40,20,"name",0.5)') c_gt = self.conn_gt.cursor() self.args.where_object_gt = 'imagefile="image0"' aps = dbEvaluate._evaluateDetectionForClassPascal( c, c_gt, 'name', self.args) self.assertEqual(aps, 1) aps = dbEvaluate._evaluateDetectionForClassSklearn( c, c_gt, 'name', self.args, sklearn) self.assertAlmostEqual(aps, 1, places=4)