def test_save_to_json(self):
name = 'test'
outdir = self.get_temp_dir()
m = metrics.Metrics(name, outdir=outdir)
# Create test list to save
L = []
for i in range(10):
L.append(dict(
name=i,
value=i,
feature='test',
stat_type='output'
))
m.save_to_json(L)
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
outfilename = os.path.join(outdir, name + '_' + todays_date + '.json')
# Check that file exists
self.assertEqual(os.path.isfile(outfilename), True)
# Check that it can be opened
with open(outfilename) as json_file:
data = json.load(json_file)
# Check data types from loaded data
self.assertIsInstance(data, dict)
self.assertItemsEqual(list(data.keys()), ['metrics', 'metadata'])
self.assertIsInstance(data['metrics'], list)
self.assertIsInstance(data['metadata'], dict)
def test_confusion_matrix(self):
y_true = _generate_stack_4d()
y_pred = _generate_stack_4d()
m = metrics.Metrics('test')
cm = m.calc_pixel_confusion_matrix(y_true, y_pred)
self.assertEqual(cm.shape[0], y_true.shape[-1])
def test_df_to_dict(self):
m = metrics.Metrics('test')
df = _generate_df()
L = m.pixel_df_to_dict(df)
# Check output types
self.assertNotEqual(len(L), 0)
self.assertIsInstance(L, list)
self.assertIsInstance(L[0], dict)
def test_metric_object_stats(self):
y_true = label(_generate_stack_3d())
y_pred = label(_generate_stack_3d())
m = metrics.Metrics('test')
before = len(m.output)
m.calc_object_stats(y_true, y_pred)
# Check data added to output
self.assertNotEqual(before, len(m.output))
def test_all_pixel_stats(self):
m = metrics.Metrics('test')
before = len(m.output)
y_true = _generate_stack_4d()
y_pred = _generate_stack_4d()
m.all_pixel_stats(y_true, y_pred)
# Check that items were added to output
self.assertNotEqual(before, len(m.output))
# Check mismatch error
self.assertRaises(ValueError, m.all_pixel_stats, np.ones(
(10, 10, 10, 1)), np.ones((5, 5, 5, 1)))
def test_run_all(self):
y_true_lbl = label(_generate_stack_3d())
y_pred_lbl = label(_generate_stack_3d())
y_true_unlbl = _generate_stack_4d()
y_pred_unlbl = _generate_stack_4d()
name = 'test'
outdir = self.get_temp_dir()
m = metrics.Metrics(name, outdir=outdir)
before = len(m.output)
m.run_all(y_true_lbl, y_pred_lbl, y_true_unlbl, y_pred_unlbl)
# Assert that data was added to output
self.assertNotEqual(len(m.output), before)
# Check output file
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
outname = os.path.join(outdir, name + '_' + todays_date + '.json')
self.assertEqual(os.path.isfile(outname), True)
def test_Metrics_init(self):
m = metrics.Metrics('test')
self.assertEqual(hasattr(m, 'output'), True)
i = np.random.randint(0, high=1440)
print(i)
y_true = label(y_test[i:i + 1, :, :, 0].astype('int'))
y_pred = label(predict_lbl[i:i + 1])
fig, ax = plt.subplots(1, 3, figsize=(10, 8))
ax[0].imshow(predict[i, :, :, 1])
ax[0].set_title('Prediction')
# Repeat labeling to get number assignments in range for this particular frame
ax[1].imshow(y_pred[0], cmap='jet')
ax[1].set_title('Labeled Prediction')
ax[2].imshow(y_true[0], cmap='jet')
ax[2].set_title('Labeled Truth')
m = metrics.Metrics('singleton', seg=True)
m.calc_object_stats(y_true, y_pred)
#reload(metrics)
#%%time
m = metrics.Metrics('fgbg', seg=True)
y_true_lbl = y_test[:, :, :, 0].astype('int')
y_pred_lbl = predict_lbl
m.calc_object_stats(y_true_lbl, y_pred_lbl)
#Note: pixel based statistics are also available from `m.all_pixel_stats`, but it requires that the ground truth annotation be transformed to match the features output by the model prediction.
#Additionally, the final metric report can be saved using `m.save_to_json(m.output)` after object and/or pixel statistics have been generated.
#Currently this dataset has small enough frame sizes that they can be looped over in `m.calc_object_stats` without too much of a problem. For datasets with larger frame sizes,