def test__accuracy(self): predicts1 = [1, 0, 1, 1] labels1 = [0, 1, 1, 1] predicts2 = [[0, 0], [0, 0]] labels2 = [[0, 1], [1, 1]] predicts3 = [[[0, 1], [0, 0], [0, 1]], [[0, 1], [0, 1], [0, 1]]] labels3 = [[[0, 1], [0, 1], [1, 0]], [[1, 0], [1, 0], [1, 0]]] predicts4 = [[0.2, 0.8], [0.1, 0.9], [0.3, 0.7], [0.4, 0.6]] #1,1,1,1 labels4 = [0, 1, 0, 0] metrics = METRICS('pytorch') acc = metrics['Accuracy']() acc.update(predicts1, labels1) acc_result = acc.result() self.assertEqual(acc_result, 0.5) acc.reset() acc.update(predicts2, labels2) self.assertEqual(acc.result(), 0.25) acc.reset() acc.update(predicts3, labels3) self.assertEqual(acc.result(), 0.25) acc.reset() acc.update(predicts4, labels4) self.assertEqual(acc.result(), 0.25) metrics = METRICS('mxnet') acc = metrics['Accuracy']() acc.update(predicts1, labels1) acc_result = acc.result() self.assertEqual(acc_result, 0.5) acc.reset() acc.update(predicts2, labels2) self.assertEqual(acc.result(), 0.25) acc.reset() acc.update(predicts3, labels3) self.assertEqual(acc.result(), 0.25) acc.reset() acc.update(predicts4, labels4) self.assertEqual(acc.result(), 0.25) metrics = METRICS('onnxrt_qlinearops') acc = metrics['Accuracy']() acc.update(predicts1, labels1) acc_result = acc.result() self.assertEqual(acc_result, 0.5) acc.reset() acc.update(predicts2, labels2) self.assertEqual(acc.result(), 0.25) acc.reset() acc.update(predicts3, labels3) self.assertEqual(acc.result(), 0.25) acc.reset() acc.update(predicts4, labels4) self.assertEqual(acc.result(), 0.25)
class Accuracy(object): def __init__(self): self.metric = METRICS('tensorflow')['Accuracy']() # it's ugly that the label is in the iterator def update(self, preds, label): logits, labels = preds self.metric.update(logits, labels) def reset(self): self.metric.reset() def result(self): return self.metric.result()
def test_tensorflow_F1(self): metrics = METRICS('tensorflow') F1 = metrics['F1']() preds = [1, 1, 1, 1] labels = [0, 1, 1, 0] F1.update(preds, labels) self.assertEqual(F1.result(), 0.5)
def test_mxnet_F1(self): metrics = METRICS('mxnet') F1 = metrics['F1']() preds = [0, 1, 1, 1, 1, 0] labels = [0, 1, 1, 1] F1.update(preds, labels) self.assertEqual(F1.result(), 0.8)
def test_mxnet_accuracy(self): metrics = METRICS('mxnet') acc = metrics['Accuracy']() predicts = [1, 0, 1, 1] labels = [0, 1, 1, 1] acc.update(predicts, labels) acc_result = acc.result() self.assertEqual(acc_result, 0.5)
def test_mae(self): predicts1 = [1, 0, 0, 1] labels1 = [0, 1, 0, 0] predicts2 = [1, 1, 1, 1] labels2 = [1, 1, 1, 0] metrics = METRICS('tensorflow') mae = metrics['MAE']() mae.update(predicts1, labels1) mae_result = mae.result() self.assertEqual(mae_result, 0.75) mae.update(0, 1) mae_result = mae.result() self.assertEqual(mae_result, 0.8) mae.reset() mae.update(predicts2, labels2) mae_result = mae.result() self.assertEqual(mae_result, 0.25) metrics = METRICS('pytorch') mae = metrics['MAE']() mae.update(predicts1, labels1) mae_result = mae.result() self.assertEqual(mae_result, 0.75) mae.update(predicts2, labels2) mae_result = mae.result() self.assertEqual(mae_result, 0.5) metrics = METRICS('mxnet') mae = metrics['MAE']() mae.update(predicts1, labels1) mae_result = mae.result() self.assertEqual(mae_result, 0.75) mae.update(predicts2, labels2) mae_result = mae.result() self.assertEqual(mae_result, 0.5) metrics = METRICS('onnxrt_qlinearops') mae = metrics['MAE']() mae.update(predicts1, labels1) mae_result = mae.result() self.assertEqual(mae_result, 0.75) mae.update(predicts2, labels2) mae_result = mae.result() self.assertEqual(mae_result, 0.5)
def test_pytorch_F1(self): metrics = METRICS('pytorch') F1 = metrics['F1']() F1.reset() preds = [1, 1] labels = [2, 1, 1] F1.update(preds, labels) self.assertEqual(F1.result(), 0.8)
def test_rmse(self): predicts1 = [1, 0, 0, 1] labels1 = [1, 0, 0, 0] predicts2 = [1, 1, 1, 1] labels2 = [1, 0, 0, 0] metrics = METRICS('tensorflow') rmse = metrics['RMSE']() rmse.update(predicts1, labels1) rmse_result = rmse.result() self.assertEqual(rmse_result, 0.5) rmse.reset() rmse.update(predicts2, labels2) rmse_result = rmse.result() self.assertAlmostEqual(rmse_result, np.sqrt(0.75)) metrics = METRICS('pytorch') rmse = metrics['RMSE']() rmse.update(predicts1, labels1) rmse_result = rmse.result() self.assertEqual(rmse_result, 0.5) rmse.update(predicts2, labels2) rmse_result = rmse.result() self.assertAlmostEqual(rmse_result, np.sqrt(0.5)) metrics = METRICS('mxnet') rmse = metrics['RMSE']() rmse.update(predicts1, labels1) rmse_result = rmse.result() self.assertEqual(rmse_result, 0.5) rmse.update(predicts2, labels2) rmse_result = rmse.result() self.assertAlmostEqual(rmse_result, np.sqrt(0.5)) metrics = METRICS('onnxrt_qlinearops') rmse = metrics['RMSE']() rmse.update(predicts1, labels1) rmse_result = rmse.result() self.assertEqual(rmse_result, 0.5) rmse.update(predicts2, labels2) rmse_result = rmse.result() self.assertAlmostEqual(rmse_result, np.sqrt(0.5))
def test_mse(self): predicts1 = [1, 0, 0, 1] labels1 = [0, 1, 0, 0] predicts2 = [1, 1, 1, 1] labels2 = [0, 1, 1, 0] metrics = METRICS('onnxrt_qlinearops') mse = metrics['MSE'](compare_label=False) mse.update(predicts1, labels1) mse_result = mse.result() self.assertEqual(mse_result, 0.75) mse.update(predicts2, labels2) mse_result = mse.result() self.assertEqual(mse_result, 0.625) metrics = METRICS('tensorflow') mse = metrics['MSE'](compare_label=False) mse.update(predicts1, labels1) mse_result = mse.result() self.assertEqual(mse_result, 0.75) mse.update(predicts2, labels2) mse_result = mse.result() self.assertEqual(mse_result, 0.625) metrics = METRICS('mxnet') mse = metrics['MSE']() mse.update(predicts1, labels1) mse_result = mse.result() self.assertEqual(mse_result, 0.75) mse.update(predicts2, labels2) mse_result = mse.result() self.assertEqual(mse_result, 0.625) metrics = METRICS('pytorch') mse = metrics['MSE']() mse.update(predicts1, labels1) mse_result = mse.result() self.assertEqual(mse_result, 0.75) mse.update(predicts2, labels2) mse_result = mse.result() self.assertEqual(mse_result, 0.625)
def test_loss(self): metrics = METRICS('onnxrt_qlinearops') loss = metrics['Loss']() predicts = [1, 0, 0, 1] labels = [0, 1, 0, 0] loss.update(predicts, labels) loss_result = loss.result() self.assertEqual(loss_result, 0.5) predicts = [1, 1, 0, 1] labels = [0, 1, 0, 0] loss.update(predicts, labels) loss_result = loss.result() self.assertEqual(loss_result, 0.625)
def testmIOU(self): metrics = METRICS('tensorflow') miou = metrics['mIOU']() preds = np.array([0, 0, 1, 1]) labels = np.array([0, 1, 0, 1]) miou.update(preds, labels) self.assertAlmostEqual(miou.result(), 0.33333334) miou.reset() preds = np.array([0, 0, 1, 1]) labels = np.array([0, 1, 1, 1]) miou.update(preds, labels) self.assertAlmostEqual(miou.result(), 0.58333333)
def testBLEU(self): metrics = METRICS('tensorflow') bleu = metrics['BLEU']() preds = ['Gutach: Mehr Sicherheit für Fußgänger'] labels = ('Gutach: Noch mehr Sicherheit für Fußgänger',) bleu.update(preds, labels) self.assertAlmostEqual(bleu.result(), 51.1507809) bleu.reset() preds = ['Dies wurde auch von Peter Arnold vom Offenburg District Office bestätigt.'] labels = ('Dies bestätigt auch Peter Arnold vom Landratsamt Offenburg.',) bleu.update(preds, labels) self.assertAlmostEqual(bleu.result(), 16.108992695) with self.assertRaises(ValueError): bleu.update(['a','b'], ('c',))
def test_onnx_topk(self): metrics = METRICS('onnxrt_qlinearops') top1 = metrics['topk']() top1.reset() self.assertEqual(top1.result(), 0) self.assertEqual(top1.result(), 0) top2 = metrics['topk'](k=2) top3 = metrics['topk'](k=3) predicts = [[0, 0.2, 0.9, 0.3], [0, 0.9, 0.8, 0]] single_predict = [0, 0.2, 0.9, 0.3] labels = [[0, 1, 0, 0], [0, 0, 1, 0]] sparse_labels = [2, 2] single_label = 2 # test functionality of one-hot label top1.update(predicts, labels) top2.update(predicts, labels) top3.update(predicts, labels) self.assertEqual(top1.result(), 0.0) self.assertEqual(top2.result(), 0.5) self.assertEqual(top3.result(), 1) # test functionality of sparse label top1.update(predicts, sparse_labels) top2.update(predicts, sparse_labels) top3.update(predicts, sparse_labels) self.assertEqual(top1.result(), 0.25) self.assertEqual(top2.result(), 0.75) self.assertEqual(top3.result(), 1) # test functionality of single label top1.update(single_predict, single_label) top2.update(single_predict, single_label) top3.update(single_predict, single_label) self.assertEqual(top1.result(), 0.4) self.assertEqual(top2.result(), 0.8) self.assertEqual(top3.result(), 1)
def test_tensorflow_COCOmAP(self): import json import os metrics = METRICS('tensorflow') fake_dict = { 'info': {}, 'licenses':{}, 'images':[{ 'file_name': '000000397133.jpg', 'height': 100, 'width': 100, 'id': 397133 }], 'annotations':[{ 'category_id': 18, 'id': 1768, 'iscrowd': 0, 'image_id': 397133, 'bbox': [473.07, 395.93, 38.65, 28.67] }], 'categories':[{ 'supercategory': 'animal', 'id': 18, 'name': 'dog' }] } fake_json = json.dumps(fake_dict) with open('anno.json', 'w') as f: f.write(fake_json) mAP = metrics['COCOmAP']('anno.json') self.assertEqual(mAP.category_map_reverse['dog'], 1) detection = [ np.array([[5]]), np.array([[5]]), np.array([[[0.16117382, 0.59801614, 0.81511605, 0.7858219 ], [0.5589304 , 0. , 0.98301625, 0.520178 ], [0.62706745, 0.35748824, 0.6892729 , 0.41513762], [0.40032804, 0.01218696, 0.6924763 , 0.30341768], [0.62706745, 0.35748824, 0.6892729 , 0.41513762]]]), np.array([[0.9267181 , 0.8510787 , 0.60418576, 0.35155892, 0.31158054]]), np.array([[ 1., 67., 51., 79., 47.]]) ] ground_truth = [ np.array([[[0.5633255 , 0.34003124, 0.69857144, 0.4009531 ], [0.4763466 , 0.7769531 , 0.54334897, 0.9675937 ]]]), np.array([['a', 'b']]), np.array([[]]), np.array([b'000000397133.jpg']) ] self.assertRaises(ValueError, mAP.update, detection, ground_truth) os.remove('anno.json') mAP = metrics['COCOmAP']() detection = [ np.array([[[0.16117382, 0.59801614, 0.81511605, 0.7858219 ], [0.5589304 , 0. , 0.98301625, 0.520178 ], [0.62706745, 0.35748824, 0.6892729 , 0.41513762], [0.40032804, 0.01218696, 0.6924763 , 0.30341768], [0.62706745, 0.35748824, 0.6892729 , 0.41513762]]]), np.array([[0.9267181 , 0.8510787 , 0.60418576, 0.35155892, 0.31158054]]), np.array([[ 1., 67., 51., 79., 47.]]) ] detection_2 = [ np.array([[8]]), np.array([[[0.82776225, 0.5865939 , 0.8927653 , 0.6302338 ], [0.8375764 , 0.6424138 , 0.9055594 , 0.6921875 ], [0.57902956, 0.39394334, 0.8342961 , 0.5577197 ], [0.7949219 , 0.6513021 , 0.8472295 , 0.68427753], [0.809729 , 0.5947042 , 0.8539927 , 0.62916476], [0.7258591 , 0.08907133, 1. , 0.86224866], [0.43100086, 0.37782395, 0.8384069 , 0.5616918 ], [0.32005906, 0.84334356, 1. , 1. ]]]), np.array([[0.86698544, 0.7562499 , 0.66414887, 0.64498234,\ 0.63083494,0.46618757, 0.3914739 , 0.3094324 ]]), np.array([[55., 55., 79., 55., 55., 67., 79., 82.]]) ] ground_truth = [ np.array([[[0.5633255 , 0.34003124, 0.69857144, 0.4009531 ], [0.56262296, 0.0015625 , 1. , 0.5431719 ], [0.16374707, 0.60728127, 0.813911 , 0.77823436], [0.5841452 , 0.21182813, 0.65156907, 0.24670312], [0.8056206 , 0.048875 , 0.90124124, 0.1553125 ], [0.6729742 , 0.09317187, 0.7696956 , 0.21203125], [0.3848478 , 0.002125 , 0.61522245, 0.303 ], [0.61548007, 0. , 0.7015925 , 0.097125 ], [0.6381967 , 0.1865625 , 0.7184075 , 0.22534375], [0.6274239 , 0.22104688, 0.71140516, 0.27134374], [0.39566743, 0.24370313, 0.43578455, 0.284375 ], [0.2673302 , 0.245625 , 0.3043794 , 0.27353126], [0.7137705 , 0.15429688, 0.726815 , 0.17114063], [0.6003747 , 0.25942189, 0.6438876 , 0.27320313], [0.68845433, 0.13501562, 0.714637 , 0.17245312], [0.69358313, 0.10959375, 0.7043091 , 0.12409375], [0.493911 , 0. , 0.72571427, 0.299 ], [0.69576114, 0.15107812, 0.70714283, 0.16332813], [0.4763466 , 0.7769531 , 0.54334897, 0.9675937 ]]]), np.array([[]]), np.array([[44, 67, 1, 49, 51, 51, 79, 1, 47, 47, 51, 51,\ 56, 50, 56, 56, 79, 57, 81]]), np.array([b'000000397133.jpg']) ] ground_truth_2 = [ np.array([[[0.51508695, 0.2911648 , 0.5903478 , 0.31360796], [0.9358696 , 0.07528409, 0.99891305, 0.25 ], [0.8242174 , 0.3309659 , 0.93508697, 0.47301137], [0.77413046, 0.22599432, 0.9858696 , 0.8179261 ], [0.32582608, 0.8575 , 0.98426086, 0.9984659 ], [0.77795655, 0.6268466 , 0.89930433, 0.73434657], [0.5396087 , 0.39053977, 0.8483913 , 0.5615057 ], [0.58473915, 0.75661933, 0.5998261 , 0.83579546], [0.80391306, 0.6129829 , 0.8733478 , 0.66201705], [0.8737391 , 0.6579546 , 0.943 , 0.7053693 ], [0.775 , 0.6549716 , 0.8227391 , 0.6882955 ], [0.8130869 , 0.58292615, 0.90526086, 0.62551135], [0.7844348 , 0.68735796, 0.98182607, 0.83329546], [0.872 , 0.6190057 , 0.9306522 , 0.6591761 ]]]), np.array([[]]), np.array([[64, 62, 62, 67, 82, 52, 79, 81, 55, 55, 55, 55, 62, 55]]), np.array([b'000000037777.jpg']) ] mAP.update(detection, ground_truth) mAP.update(detection, ground_truth) self.assertEqual(format(mAP.result(), '.5f'), '0.14149') mAP.update(detection_2, ground_truth_2) self.assertEqual(format(mAP.result(), '.5f'), '0.13366') mAP.reset() mAP.update(detection, ground_truth) self.assertEqual(format(mAP.result(), '.5f'), '0.14149') ground_truth_1 = [ np.array([[[0.51508695, 0.2911648 , 0.5903478 , 0.31360796], [0.872 , 0.6190057 , 0.9306522 , 0.6591761 ]]]), np.array([[]]), np.array([[[64, 62]]]), np.array([b'000000037777.jpg']) ] self.assertRaises(ValueError, mAP.update, detection, ground_truth_1) ground_truth_2 = [ np.array([[[0.51508695, 0.2911648 , 0.5903478 , 0.31360796], [0.872 , 0.6190057 , 0.9306522 , 0.6591761 ]]]), np.array([[]]), np.array([[64]]), np.array([b'000000037700.jpg']) ] self.assertRaises(ValueError, mAP.update, detection, ground_truth_2) detection_1 = [ np.array([[[0.16117382, 0.59801614, 0.81511605, 0.7858219 ], [0.5589304 , 0. , 0.98301625, 0.520178 ]]]), np.array([[0.9267181 , 0.8510787 , 0.60418576, 0.35155892, 0.31158054]]), np.array([[ 1., 67., 51., 79., 47.]]) ] ground_truth_1 = [ np.array([[[0.51508695, 0.2911648 , 0.5903478 , 0.31360796], [0.872 , 0.6190057 , 0.9306522 , 0.6591761 ]]]), np.array([[]]), np.array([[64, 62]]), np.array([b'000000011.jpg']) ] self.assertRaises(ValueError, mAP.update, detection_1, ground_truth_1) ground_truth_2 = [ np.array([[[0.51508695, 0.2911648 , 0.5903478 , 0.31360796], [0.872 , 0.6190057 , 0.9306522 , 0.6591761 ]]]), np.array([[]]), np.array([[64, 62]]), np.array([b'000000012.jpg']) ] detection_2 = [ np.array([[[0.16117382, 0.59801614, 0.81511605, 0.7858219 ], [0.5589304 , 0. , 0.98301625, 0.520178 ]]]), np.array([[0.9267181 , 0.8510787]]), np.array([[ 1., 67., 51., 79., 47.]]) ] self.assertRaises(ValueError, mAP.update, detection_2, ground_truth_2)
def __init__(self): self.metric = METRICS('tensorflow')['Accuracy']()