def scorer(pred,label,vxlspacing): """ :param pred: :param label: :param voxelspacing: :return: """ volscores = {} volscores['dice'] = metric.dc(pred,label) volscores['jaccard'] = metric.binary.jc(pred,label) volscores['voe'] = 1. - volscores['jaccard'] volscores['rvd'] = metric.ravd(label,pred) if np.count_nonzero(pred) ==0 or np.count_nonzero(label)==0: volscores['assd'] = 0 volscores['msd'] = 0 else: evalsurf = Surface(pred,label,physical_voxel_spacing = vxlspacing,mask_offset = [0.,0.,0.], reference_offset = [0.,0.,0.]) volscores['assd'] = evalsurf.get_average_symmetric_surface_distance() volscores['msd'] = metric.hd(label,pred,voxelspacing=vxlspacing) logging.info("\tDice " + str(volscores['dice'])) logging.info("\tJaccard " + str(volscores['jaccard'])) logging.info("\tVOE " + str(volscores['voe'])) logging.info("\tRVD " + str(volscores['rvd'])) logging.info("\tASSD " + str(volscores['assd'])) logging.info("\tMSD " + str(volscores['msd'])) return volscores
def get_scores(pred, label, vxlspacing): """ pred: HxWxZ (x, y, z) of boolean label: HxWxZ (e.g. (512,512,75)) vxlspacing: 3-tuple of float (spacing) """ volscores = {} volscores['dice'] = metric.dc(pred, label) try: jaccard = metric.binary.jc(pred, label) except ZeroDivisionError: jaccard = 0.0 volscores['jaccard'] = jaccard volscores['voe'] = 1. - volscores['jaccard'] try: rvd = metric.ravd(label, pred) except: rvd = None volscores['rvd'] = rvd if np.count_nonzero(pred) == 0 or np.count_nonzero(label) == 0: volscores['assd'] = 0 volscores['msd'] = 0 else: evalsurf = Surface(pred, label, physical_voxel_spacing=vxlspacing, mask_offset=[0., 0., 0.], reference_offset=[0., 0., 0.]) volscores['assd'] = evalsurf.get_average_symmetric_surface_distance() volscores['msd'] = metric.hd(label, pred, voxelspacing=vxlspacing) return volscores
def compute_segmentation_scores(prediction_mask, reference_mask): """ Calculates metrics scores from numpy arrays and returns an dict. Assumes that each object in the input mask has an integer label that defines object correspondence between prediction_mask and reference_mask. :param prediction_mask: numpy.array, int :param reference_mask: numpy.array, int :param voxel_spacing: list with x,y and z spacing :return: dict with dice, jaccard, voe, rvd, assd, rmsd, and msd """ scores = {'dice': [], 'jaccard': [], 'voe': [], 'rvd': []} for i, obj_id in enumerate(np.unique(prediction_mask)): if obj_id == 0: continue # 0 is background, not an object; skip # Limit processing to the bounding box containing both the prediction # and reference objects. target_mask = (reference_mask == obj_id) + (prediction_mask == obj_id) bounding_box = ndimage.find_objects(target_mask)[0] p = (prediction_mask == obj_id)[bounding_box] r = (reference_mask == obj_id)[bounding_box] if np.any(p) and np.any(r): dice = metric.dc(p, r) jaccard = dice / (2. - dice) scores['dice'].append(dice) scores['jaccard'].append(jaccard) scores['voe'].append(1. - jaccard) scores['rvd'].append(metric.ravd(r, p)) return scores
def compute_segmentation_scores(prediction_mask, reference_mask, voxel_spacing): """ Calculates metrics scores from numpy arrays and returns an dict. Assumes that each object in the input mask has an integer label that defines object correspondence between prediction_mask and reference_mask. :param prediction_mask: numpy.array, int :param reference_mask: numpy.array, int :param voxel_spacing: list with x,y and z spacing :return: dict with dice, jaccard, voe, rvd, assd, rmsd, and msd """ scores = { 'dice': [], 'jaccard': [], 'voe': [], 'rvd': [], 'assd': [], 'rmsd': [], 'msd': [] } p = (prediction_mask > 0) r = (reference_mask > 0) if np.any(p) and np.any(r): dice = metric.dc(p, r) jaccard = dice / (2. - dice) scores['dice'].append(dice) scores['jaccard'].append(jaccard) scores['voe'].append(1. - jaccard) scores['rvd'].append(metric.ravd(r, p)) evalsurf = Surface(p, r, physical_voxel_spacing=voxel_spacing, mask_offset=[0., 0., 0.], reference_offset=[0., 0., 0.]) assd = evalsurf.get_average_symmetric_surface_distance() rmsd = evalsurf.get_root_mean_square_symmetric_surface_distance() msd = evalsurf.get_maximum_symmetric_surface_distance() scores['assd'].append(assd) scores['rmsd'].append(rmsd) scores['msd'].append(msd) else: # There are no objects in the prediction, in the reference, or both scores['dice'].append(0) scores['jaccard'].append(0) scores['voe'].append(1.) # Surface distance (and volume difference) metrics between the two # masks are meaningless when any one of the masks is empty. Assign # maximum penalty. The average score for these metrics, over all # objects, will thus also not be finite as it also loses meaning. scores['rvd'].append(LARGE) scores['assd'].append(LARGE) scores['rmsd'].append(LARGE) scores['msd'].append(LARGE) return scores
def calculate_metrics(mask1, mask2): true_positives = metric.obj_tpr(mask1, mask2) false_positives = metric.obj_fpr(mask1, mask2) dc = metric.dc(mask1, mask2) hd = metric.hd(mask1, mask2) precision = metric.precision(mask1, mask2) recall = metric.recall(mask1, mask2) ravd = metric.ravd(mask1, mask2) assd = metric.assd(mask1, mask2) asd = metric.asd(mask1, mask2) return true_positives, false_positives, dc, hd, precision, recall, ravd, assd, asd
def total_score(pred_path,gt_path, file_names, metric_name): print(pred_path) gt_to_pred = {k:k for k in [0,1]} list_labels = sorted(gt_to_pred.keys()) score = dict() thre = 400 score['names'] = [] score['lesion'] = [] for name in file_names: ground_truth = gt_path.format(name) ground_truth = os.path.expanduser(ground_truth) image_gt = nibabel.load(ground_truth) image_gt= nibabel.funcs.as_closest_canonical(image_gt).get_data() image_gt = image_gt.reshape(image_gt.shape[:3]) pred = pred_path.format(name) pred = os.path.expanduser(pred) image_pred = nibabel.load(pred) affine = image_pred.affine voxel = [affine[0,0],affine[1,1],affine[2,2]] image_pred = image_pred.get_data() image_pred = image_pred.reshape(image_pred.shape[:3]) score['names'].append(name) if metric_name=='assd': score['lesion'].append(metric.assd(image_gt,image_pred,voxel)) elif metric_name=='rve': score['lesion'].append(metric.ravd(image_gt,image_pred)) else: score['lesion'].append(metric.dc(image_gt,image_pred)) print('Sample size: {}'.format(len(list(score.values())[0]))) for label in score.keys(): if label != 'names': print('Label: {}, {} mean: {}'.format(label, metric_name, round(np.mean(score[label]),2))) print('Label: {}, {} std: {}'.format(label, metric_name, round(np.std(score[label]),2))) return score
def get_scores(pred,label,vxlspacing): volscores = {} volscores['dice'] = metric.dc(pred,label) volscores['jaccard'] = metric.binary.jc(pred,label) volscores['voe'] = 1. - volscores['jaccard'] volscores['rvd'] = metric.ravd(label,pred) if np.count_nonzero(pred) ==0 or np.count_nonzero(label)==0: volscores['assd'] = 0 volscores['msd'] = 0 else: evalsurf = Surface(pred,label,physical_voxel_spacing = vxlspacing,mask_offset = [0.,0.,0.], reference_offset = [0.,0.,0.]) volscores['assd'] = evalsurf.get_average_symmetric_surface_distance() volscores['msd'] = metric.hd(label,pred,voxelspacing=vxlspacing) return volscores
def calculate_validation_metrics(probas_pred, image_gt, class_labels=None, num_classes=5): classes = np.arange(probas_pred.shape[-1]) # determine valid classes (those that actually appear in image_gt). Some images may miss some classes classes = [c for c in classes if np.sum(image_gt == c) != 0] image_pred = probas_pred.argmax(-1) assert image_gt.shape == image_pred.shape accuracy = np.sum(image_gt == image_pred) / float(image_pred.size) class_metrics = {} y_true = convert_seg_flat_to_binary_label_indicator_array( image_gt.ravel(), num_classes).astype(int)[:, classes] y_pred = probas_pred.transpose(3, 0, 1, 2).reshape(num_classes, -1).transpose(1, 0)[:, classes] scores = roc_auc_score(y_true, y_pred, None) for i, c in enumerate(classes): true_positives = metric.obj_tpr(image_gt == c, image_pred == c) false_positives = metric.obj_fpr(image_gt == c, image_pred == c) dc = metric.dc(image_gt == c, image_pred == c) hd = metric.hd(image_gt == c, image_pred == c) precision = metric.precision(image_gt == c, image_pred == c) recall = metric.recall(image_gt == c, image_pred == c) ravd = metric.ravd(image_gt == c, image_pred == c) assd = metric.assd(image_gt == c, image_pred == c) asd = metric.asd(image_gt == c, image_pred == c) label = c if class_labels is not None and c in class_labels.keys(): label = class_labels[c] class_metrics[label] = { 'true_positives': true_positives, 'false_positives': false_positives, 'DICE\t\t': dc, 'Hausdorff dist': hd, 'precision\t': precision, 'recall\t\t': recall, 'rel abs vol diff': ravd, 'avg surf dist symm': assd, 'avg surf dist\t': asd, 'roc_auc\t\t': scores[i] } return accuracy, class_metrics
def calculate_metrics(mask1, mask2): try: true_positives = metric.obj_tpr(mask1, mask2) if mask2.sum() != 0: false_positives = metric.obj_fpr(mask1, mask2) else: false_positives = 0 if mask1.sum() == 0 or mask2.sum() == 0: hd = 999 assd = 999 asd = 999 else: hd = 999 #metric.hd(mask1, mask2) assd = 999 #metric.assd(mask1, mask2) asd = 999 #metric.asd(mask1, mask2) dc = metric.dc(mask1, mask2) precision = metric.precision(mask1, mask2) recall = metric.recall(mask1, mask2) ravd = metric.ravd(mask1, mask2) return true_positives, false_positives, dc, hd, precision, recall, ravd, assd, asd except: return 99999, 99999, 99999, 99999, 99999, 99999, 99999, 99999, 99999
for name in filenames: name_sp = name.split(sep='_') name_num = re.findall('\d+', name_sp[1])[0] # name_num = name[3:5] gt_nii = nb.load(os.path.join(gt_path, name_sp[0] + '_' + name_num + '.nii.gz')) gt = gt_nii.get_data() # tumorgt vxlspacing = gt_nii.header.get_zooms() seg = nb.load(os.path.join(seg_path, name)).get_data() seg = np.squeeze(seg) # seg = np.uint8(seg>1) if np.max(seg)>0: # gt = gt>1 # seg = getLargestCC(seg>0) seg_dice = round(metric.dc(seg>0, gt), 4) seg_RAVD = round(metric.ravd(seg, gt), 4) seg_ASSD = round(metric.binary.assd(seg, gt, vxlspacing), 4) seg_MSSD = round(metric.hd(seg, gt, vxlspacing), 4) else: seg_dice = 0.0 seg_RAVD = 0.0 seg_ASSD = 0.0 seg_MSSD = 0.0 seg_measures['image'].append(name) seg_measures['Dice'].append(seg_dice) seg_measures['RAVD'].append(seg_RAVD) seg_measures['ASSD'].append(seg_ASSD) seg_measures['MSSD'].append(seg_MSSD) print(name, seg_dice)
def metric_3d(logits3d, labels3d, required=None, **kwargs): """ Compute 3D metrics: * (Dice) Dice Coefficient * (VOE) Volumetric Overlap Error * (VD) Relative Volume Difference * (ASD) Average Symmetric Surface Distance * (RMSD) Root Mean Square Symmetric Surface Distance * (MSD) Maximum Symmetric Surface Distance Parameters ---------- logits3d: ndarray 3D binary prediction, shape is the same with `labels3d`, it should be an int array or boolean array. labels3d: ndarray 3D labels for segmentation, shape [None, None, None], it should be an int array or boolean array. If the dimensions of `logits3d` and `labels3d` are greater than 3, then `np.squeeze` will be applied to remove extra single dimension and then please make sure these two variables are still have 3 dimensions. For example, shape [None, None, None, 1] or [1, None, None, None, 1] are allowed. required: str or list a string or a list of string to specify which metrics need to be return, default this function will return all the metrics listed above. For example, if use ```python _metric_3D(logits3D, labels3D, require=["Dice", "VOE", "ASD"]) ``` then only these three metrics will be returned. kwargs: dict sampling: list the pixel resolution or pixel size. This is entered as an n-vector where n is equal to the number of dimensions in the segmentation i.e. 2D or 3D. The default value is 1 which means pixls are 1x1x1 mm in size Returns ------- metrics required Notes ----- Thanks to the code snippet from @MLNotebook's blog. [Blog link](https://mlnotebook.github.io/post/surface-distance-function/). """ metrics = ["Dice", "VOE", "RVD", "ASSD", "RMSD", "MSD"] need_dist_map = False if required is None: required = metrics elif isinstance(required, str): required = [required] if required[0] not in metrics: raise ValueError("Not supported metric: %s" % required[0]) elif required in metrics[3:]: need_dist_map = True else: need_dist_map = False for req in required: if req not in metrics: raise ValueError("Not supported metric: %s" % req) if (not need_dist_map) and req in metrics[3:]: need_dist_map = True if logits3d.ndim > 3: logits3d = np.squeeze(logits3d) if labels3d.ndim > 3: labels3d = np.squeeze(labels3d) assert logits3d.shape == labels3d.shape, ( "Shape mismatch of logits3D and labels3D. \n" "Logits3D has shape %r while labels3D has " "shape %r" % (logits3d.shape, labels3d.shape)) logits3d = logits3d.astype(np.bool) labels3d = labels3d.astype(np.bool) metrics_3d = {} sampling = kwargs.get("sampling", [1., 1., 1.]) if need_dist_map: from utils.surface import Surface if np.count_nonzero(logits3d) == 0 or np.count_nonzero(labels3d) == 0: metrics_3d['ASSD'] = 0 metrics_3d['MSD'] = 0 else: eval_surf = Surface(logits3d, labels3d, physical_voxel_spacing=sampling, mask_offset=[0., 0., 0.], reference_offset=[0., 0., 0.]) if "ASSD" in required: metrics_3d[ "ASSD"] = eval_surf.get_average_symmetric_surface_distance( ) required.remove("ASSD") if "MSD" in required: metrics_3d[ "MSD"] = eval_surf.get_maximum_symmetric_surface_distance( ) if "RMSD" in required: metrics_3d[ "RMSD"] = eval_surf.get_root_mean_square_symmetric_surface_distance( ) if required: if "Dice" in required: metrics_3d["Dice"] = mtr.dc(logits3d, labels3d) if "VOE" in required: metrics_3d["VOE"] = 1. - mtr.jc(logits3d, labels3d) if "RVD" in required: metrics_3d["RVD"] = mtr.ravd(logits3d, labels3d) return metrics_3d
overlap_measures_filter = sitk.LabelOverlapMeasuresImageFilter() hausdorff_distance_filter = sitk.HausdorffDistanceImageFilter() reference_segmentation = image # the refenrence image in this case is the tumor mask label = 255 # init signed mauerer distance as reference metrics reference_distance_map = sitk.SignedMaurerDistanceMap( reference_segmentation, squaredDistance=False, useImageSpacing=True) label_intensity_statistics_filter = sitk.LabelIntensityStatisticsImageFilter( ) ''' Calculate Overlap Metrics''' volscores = {} ref = sitk.GetArrayFromImage( image) # convert the sitk image to numpy array seg = sitk.GetArrayFromImage(segmentation) volscores['rvd'] = metric.ravd(seg, ref) volscores['dice'] = metric.dc(seg, ref) volscores['jaccard'] = metric.binary.jc(seg, ref) volscores['voe'] = 1. - volscores['jaccard'] ''' Add the Volume Overlap Metrics in the Enum vector ''' overlap_measures_filter.Execute(reference_segmentation, segmentation) overlap_results[0, OverlapMeasures.jaccard. value] = overlap_measures_filter.GetJaccardCoefficient() overlap_results[0, OverlapMeasures.dice. value] = overlap_measures_filter.GetDiceCoefficient() overlap_results[0, OverlapMeasures.volume_similarity. value] = overlap_measures_filter.GetVolumeSimilarity() overlap_results[ 0, OverlapMeasures.volumetric_overlap_error.value] = volscores['voe'] overlap_results[ 0, OverlapMeasures.relative_vol_difference.value] = volscores['rvd']