def test_metric(self):
a = np.array([1, 2, 3], dtype=np.uint64)
b = np.array([4, 5, 6], dtype=np.uint64)
m = evaluate.rand_voi(a, b)
assert m['rand_split'] == 1.0
assert m['rand_merge'] == 1.0
assert m['voi_split'] == 0.0
assert m['voi_merge'] == 0.0
a = np.array([1, 1, 2, 2], dtype=np.uint64)
b = np.array([2, 2, 2, 2], dtype=np.uint64)
m = evaluate.rand_voi(a, b)
assert m['rand_split'] == 1.0
assert m['rand_merge'] == 0.5
assert m['voi_split'] == 0.0
assert m['voi_merge'] == 1.0
m = evaluate.rand_voi(b, a)
assert m['rand_split'] == 0.5
assert m['rand_merge'] == 1.0
assert m['voi_split'] == 1.0
assert m['voi_merge'] == 0.0
a = np.array([1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4], dtype=np.uint64)
b = np.array([3, 3, 3, 3, 4, 5, 6, 6, 5, 5, 5], dtype=np.uint64)
m = evaluate.rand_voi(a, b)
self.assertAlmostEqual(m['rand_split'], 0.6363636363636364)
self.assertAlmostEqual(m['rand_merge'], 0.5675675675675675)
self.assertAlmostEqual(m['voi_split'], 0.6140806820148608)
self.assertAlmostEqual(m['voi_merge'], 0.7272727272727271)
m = evaluate.rand_voi(b, a, return_cluster_scores=True)
self.assertAlmostEqual(m['rand_split'], 0.5675675675675675)
self.assertAlmostEqual(m['rand_merge'], 0.6363636363636364)
self.assertAlmostEqual(m['voi_split'], 0.7272727272727271)
self.assertAlmostEqual(m['voi_merge'], 0.6140806820148608)
self.assertAlmostEqual(sum(m['voi_split_i'].values()), m['voi_split'])
self.assertAlmostEqual(sum(m['voi_merge_j'].values()), m['voi_merge'])
def evaluate(self, output_array_identifier, evaluation_array):
output_array = ZarrArray.open_from_array_identifier(
output_array_identifier)
evaluation_data = evaluation_array[evaluation_array.roi].astype(
np.uint64)
output_data = output_array[output_array.roi].astype(np.uint64)
results = rand_voi(evaluation_data, output_data)
return InstanceEvaluationScores(voi_merge=results["voi_merge"],
voi_split=results["voi_split"])
def evaluate_threshold(
experiment,
setup,
iteration,
db_host,
scores_db_name,
edges_collection,
segment_ids,
gt,
threshold,
configuration,
volume_size):
# open score DB
client = MongoClient(db_host)
database = client[scores_db_name]
score_collection = database['scores']
# get VOI and RAND
logging.info("Calculating VOI scores for threshold %f...", threshold)
logging.info(type(segment_ids))
rand_voi_report = rand_voi(
gt,
segment_ids,
return_cluster_scores=False)
metrics = rand_voi_report.copy()
for k in {'voi_split_i', 'voi_merge_j'}:
del metrics[k]
logging.info("Storing VOI values for threshold %f in DB" % threshold)
metrics['threshold'] = threshold
metrics['experiment'] = experiment
metrics['setup'] = setup
metrics['iteration'] = iteration
metrics['network'] = configuration
metrics['volume_size'] = volume_size
metrics['merge_function'] = edges_collection.strip('edges_')
logging.info(metrics)
score_collection.replace_one(
filter={
'network': metrics['network'],
'volume_size': metrics['volume_size'],
'merge_function': metrics['merge_function'],
'threshold': metrics['threshold']
},
replacement=metrics,
upsert=True)
def compute_rand_voi(self,
site_component_ids,
site_segment_ids,
return_cluster_scores=False):
logging.info("Computing RAND and VOI...")
rand_voi_report = rand_voi(np.array([[site_component_ids]]),
np.array([[site_segment_ids]]),
return_cluster_scores=return_cluster_scores)
logging.info(f"VOI split: {rand_voi_report['voi_split']}")
logging.info(f"VOI merge: {rand_voi_report['voi_merge']}")
return rand_voi_report
def evaluate_threshold(experiment, setup, iteration, db_host, scores_db_name,
edges_collection, segment_ids, gt, threshold, method,
run_type):
#open score DB
client = MongoClient(db_host)
database = client[scores_db_name]
score_collection = database['scores']
#get VOI and RAND
logging.info(f"Calculating VOI scores for threshold {threshold}...")
logging.info(type(segment_ids))
rand_voi_report = rand_voi(gt, segment_ids, return_cluster_scores=False)
metrics = rand_voi_report.copy()
for k in {'voi_split_i', 'voi_merge_j'}:
del metrics[k]
logging.info(f"Storing VOI values for threshold {threshold} in DB")
metrics['threshold'] = threshold
metrics['experiment'] = experiment
metrics['setup'] = setup
metrics['iteration'] = iteration
metrics['merge_function'] = edges_collection.strip('edges_')
if method is not None:
metrics['method'] = method
if run_type is not None:
metrics['run_type'] = run_type
logging.info(f"VOI split: {metrics['voi_split']}")
logging.info(f"VOI merge: {metrics['voi_merge']}")
logging.info(metrics)
score_collection.replace_one(filter={
'method': metrics['method'],
'run_type': metrics['run_type'],
'merge_function': metrics['merge_function'],
'threshold': metrics['threshold']
},
replacement=metrics,
upsert=True)
def test_inputs(self):
with self.assertRaises(AssertionError):
a = np.array([1, 2, 3], dtype=np.uint64)
b = np.array([[4, 5, 6]], dtype=np.uint64)
evaluate.rand_voi(a, b)
with self.assertRaises(AssertionError):
a = np.array([1, 2, 3], dtype=np.uint64)
b = np.array([4, 5], dtype=np.uint64)
evaluate.rand_voi(a, b)
with self.assertRaises(ValueError):
a = np.array([1, 2, 3], dtype=np.uint32)
b = np.array([4, 5, 6], dtype=np.uint64)
evaluate.rand_voi(a, b)
def evaluate_affs(pred_labels, gt_labels, return_results=False):
results = rand_voi(gt_labels.data, pred_labels.data)
results["voi_sum"] = results["voi_split"] + results["voi_merge"]
scores = {"sample": results, "average": results}
if return_results:
results = {
"pred_labels":
gp.Array(
pred_labels.data.astype(np.uint64),
gp.ArraySpec(roi=pred_labels.spec.roi,
voxel_size=pred_labels.spec.voxel_size)),
"gt_labels":
gp.Array(
gt_labels.data.astype(np.uint64),
gp.ArraySpec(roi=gt_labels.spec.roi,
voxel_size=gt_labels.spec.voxel_size)),
}
return scores, results
return scores
print('compute connected components')
#connected components
gt_comp, gt_num_comp = ndimage.measurements.label(gt_bin)
yhat_comp, yhat_num_comp = ndimage.measurements.label(yhat_bin)
gt_lum_comp, gt_lum_num_comp = ndimage.measurements.label(gt_lum_bin)
yhat_lum_comp, yhat_lum_num_comp = ndimage.measurements.label(yhat_lum_bin)
if (gt_comp.dtype != np.dtype('uint64')):
gt_comp = gt_comp.astype('uint64')
if (yhat_comp.dtype != np.dtype('uint64')):
yhat_comp = yhat_comp.astype('uint64')
print('compute vi')
#vi
rand_voi_arrays = rand_voi(gt_comp, yhat_comp, False)
rand_voi_control = rand_voi(gt_comp, gt_comp, False)
print('compute iou over all')
#IOU over all
iou_score = np.sum(np.logical_and(gt_bin, yhat_bin)) / np.sum(
np.logical_or(gt_bin, yhat_bin)).item()
iou_control = np.sum(np.logical_and(gt_bin, gt_bin)) / np.sum(
np.logical_or(gt_bin, gt_bin)).item()
print('compute cleft matching')
#cleft matching
#signed distance transformation on gt_bin to create gt_sdt
sdt = ndimage.morphology.distance_transform_edt(gt_bin,sampling=(4,4,4)) -\
ndimage.morphology.distance_transform_edt(np.logical_not(gt_bin),sampling=(4,4,4))
stdt = np.tanh(sdt / 50.)
