def calculate_metric_percase(pred, gt, num_classes): "二分类、多分类的指标统计" if num_classes is None: num_classes = len(np.unique(gt))#注意:gt不是onehot编码 print('np.unique(gt):',np.unique(gt)) if num_classes==2: dice = metric.binary.dc(pred, gt) jc = metric.binary.jc(pred, gt) hd = metric.binary.hd95(pred, gt) asd = metric.binary.asd(pred, gt) elif num_classes>2: gt_onehot = to_categorical(gt, num_classes) pred_onehot = to_categorical(pred, num_classes) dice = [] jc = [] hd = [] asd = [] for k in range(num_classes): pred_k = pred_onehot[...,k] gt_k = gt_onehot[...,k] dice += [metric.dc(result=pred_k, reference=gt_k)] jc += [metric.jc(result=pred_k, reference=gt_k)] hd += [metric.hd95(result=pred_k, reference=gt_k)] asd += [metric.asd(result=pred_k, reference=gt_k)] else: raise ValueError("pred和gt不能是onehot编码") return dice, jc, hd, asd
def housdorff_distance_95(logits, label): predis = F.softmax(logits, dim=1) n_classes = logits.shape[1] result = [] for cls in range(1, n_classes): result.append(hd95(predis[:, cls, ...].squeeze(), label == cls)) return result
def hausdorff(prediction: np.ndarray, reference: np.ndarray) -> float: try: return metric.hd95(prediction, reference) except Exception as e: print("Error: ", e) print( f"Prediction does not contain the same label as gt. " f"Pred labels {np.unique(prediction)} GT labels {np.unique(reference)}" ) return 373
def hausdorff_distance_95(test=None, reference=None, confusion_matrix=None, nan_for_nonexisting=True, voxel_spacing=None, connectivity=1, **kwargs): if confusion_matrix is None: confusion_matrix = ConfusionMatrix(test, reference) test_empty, test_full, reference_empty, reference_full = confusion_matrix.get_existence() if test_empty or test_full or reference_empty or reference_full: if nan_for_nonexisting: return float("NaN") else: return 0 test, reference = confusion_matrix.test, confusion_matrix.reference return metric.hd95(test, reference, voxel_spacing, connectivity)
def compute_BraTS_HD95(ref, pred): """ ref and gt are binary integer numpy.ndarray s spacing is assumed to be (1, 1, 1) :param ref: :param pred: :return: """ num_ref = np.sum(ref) num_pred = np.sum(pred) if num_ref == 0: if num_pred == 0: return 0 else: return 373.12866 elif num_pred == 0 and num_ref != 0: return 373.12866 else: return hd95(pred, ref, (1, 1, 1))
batch_x = X_test[i_c:i_c + 1, :, :, :, :].copy() batch_y = Y_test[i_c:i_c + 1, :, :, :, :].copy() for j_c in range(n_class): dice_c[i_c, j_c] = dk_seg.dice(batch_y[0, :, :, :, j_c] == 1, y_tr_pr_c == j_c) y_t_c = np.argmax( batch_y[0, :, :, :, :], axis=-1) #dk_aux.save_pred_thumbs(batch_x[0,:,:,:,0], y_t_c, y_tr_pr_c, False, i_c, i_eval, images_dir ) '''if i_eval==0: save_pred_mhds(batch_x[0,:,:,:,0], y_t_c, y_tr_pr_c, False, i_c, i_eval) else: save_pred_mhds(None, None, y_tr_pr_c, False, i_c, i_eval)''' dice_c[i_c, n_class] = hd95(y_t_c, y_tr_pr_c) dice_c[i_c, n_class+1] = asd(y_t_c, y_tr_pr_c) dice_c[i_c, n_class+2] = assd(y_t_c, y_tr_pr_c) y_tr_pr_soft= y_tr_pr_sum[:,:,:,1]/(y_tr_pr_cnt+1e-10) #dk_aux.save_pred_soft_thumbs(batch_x[0,:,:,:,0], y_t_c, y_tr_pr_c, y_tr_pr_soft, False, i_c, i_eval, images_dir) error_mask= dk_aux.seg_2_anulus(y_t_c, radius= 2.0) plot_save_path= None ECE, MCE, ECE_curve= dk_aux.estimate_ECE_and_MCE(y_t_c, y_tr_pr_soft, plot_save_path=plot_save_path) dice_c[i_c, n_class+3]= ECE dice_c[i_c, n_class+4]= MCE plot_save_path= None ECE, MCE, ECE_curve= dk_aux.estimate_ECE_and_MCE_masked(y_t_c, y_tr_pr_soft, error_mask, plot_save_path=plot_save_path)
def main(FLAGS): # set GPU device to use os.environ['CUDA_VISIBLE_DEVICES'] = '3' # get the models to evaluate ckpts = glob(os.path.join(FLAGS.ckpt_dir, '*.h5')) # get data files and sort them image_files = os.listdir(FLAGS.test_X_dir) anno_files = os.listdir(FLAGS.test_y_dir) image_files.sort() anno_files.sort() for ckpt in ckpts: K.clear_session() # set model to load ckpt_name = os.path.basename(ckpt) # create a location to store evaluation metrics metrics = np.zeros((len(image_files), FLAGS.classes, 8)) overall_accuracy = np.zeros((len(image_files), )) # create a file writer to store the metrics excel_name = os.path.splitext(os.path.basename(ckpt))[0] + '.xlsx' writer = pd.ExcelWriter(excel_name) model = load_model(ckpt) for i in range(len(image_files)): # define path to the test data test_path = os.path.join(FLAGS.test_X_dir, image_files[i]) generator = FCN2DDatasetGenerator( test_path, batch_size=FLAGS.batch_size, subset='test', normalization=FLAGS.normalization, categorical_labels=True, num_classes=FLAGS.classes) # check if the images and annotations are the correct files print(image_files[i], anno_files[i]) preds = model.predict_generator(generator.generate(), steps=len(generator)) stamp = datetime.datetime.fromtimestamp( time.time()).strftime('date_%Y_%m_%d_time_%H_%M_%S') write_hdf5('fcn_predictions_' + stamp + '.h5', preds) pred_file = glob(os.path.join(FLAGS.predictions_temp_dir, '*.h5'))[0] pt_name = image_files[i].split('.')[0] new_name_raw = pt_name + ckpt_name new_file_raw = os.path.join(FLAGS.predictions_final_dir, new_name_raw) os.rename(pred_file, new_file_raw) ref = read_hdf5_multientry( os.path.join(FLAGS.test_y_dir, anno_files[i])) ref = np.squeeze(np.asarray(ref)) preds = read_hdf5(new_file_raw) preds = np.argmax(preds, axis=-1) overall_accuracy[i] = skm.accuracy_score(ref.flatten(), preds.flatten()) for j in range(FLAGS.classes): organ_pred = (preds == j).astype(np.int64) organ_ref = (ref == j).astype(np.int64) if np.sum(organ_pred) == 0 or np.sum(organ_ref) == 0: metrics[i, j, 0] = 0. metrics[i, j, 1] = 0. metrics[i, j, 2] = 1. metrics[i, j, 3] = 0. metrics[i, j, 4] = 0. metrics[i, j, 5] = 0. metrics[i, j, 6] = np.inf metrics[i, j, 7] = np.inf else: metrics[i, j, 0] = jaccard_index(organ_ref, organ_pred) metrics[i, j, 1] = dice_similarity_coefficient( organ_ref, organ_pred) metrics[i, j, 2] = relative_volume_difference( organ_ref, organ_pred) metrics[i, j, 3] = precision(organ_ref, organ_pred) metrics[i, j, 4] = recall(organ_ref, organ_pred) metrics[i, j, 5] = matthews_correlation_coefficient( organ_ref, organ_pred) metrics[i, j, 6] = mpm.hd95(organ_pred, organ_ref) metrics[i, j, 7] = mpm.assd(organ_pred, organ_ref) print(overall_accuracy[i]) print(metrics[i]) for k in range(metrics.shape[-1]): data = pd.DataFrame(metrics[:, :, k], columns=['bg', 'met']) data.to_excel(writer, sheet_name=str(k)) acc = pd.DataFrame(overall_accuracy, columns=['acc']) acc.to_excel(writer, sheet_name='acc') writer.save()
def process(self, inputs, outputs): for input, output in zip(inputs, outputs): self.count += len(output["instances"]) gt_segm = self.annotations[input["file_name"]]["segm_per_region"] try: _ = output["instances"].pred_masks except AttributeError: continue pred_segm = downsample_points(output) doc_ahd = {cat: [] for cat in categories_list} doc_hd = {cat: [] for cat in categories_list} doc_hd95 = {cat: [] for cat in categories_list} doc_iou = {cat: [] for cat in categories_list} doc_acc = {cat: [] for cat in categories_list} for reg_type in range(len(categories_list)): gt, pred = gt_segm[reg_type], pred_segm[reg_type] # Both have points if len(gt) and len(pred): """"Peformed using medpy""" gt_mask = np.zeros((len(gt), input["height"], input["width"]), dtype=np.int8) for i in range(len(gt)): cv2.fillPoly(gt_mask[i], np.array([gt[i]]).astype(np.int32), 1) pred_mask = np.zeros((len(pred), input["height"], input["width"]), dtype=np.int8) for i in range(len(pred)): cv2.fillPoly(pred_mask[i], np.array([pred[i]]).astype(np.int32), 1) gt_mask = gt_mask.astype(np.uint8) gt_mask = (gt_mask * 255).astype(np.uint8) pred_mask = pred_mask.astype(np.uint8) pred_mask = (pred_mask * 255).astype(np.uint8) def compute_iou_and_accuracy(arrs, edge_mask1): intersection = cv2.bitwise_and(arrs, edge_mask1) union = cv2.bitwise_or(arrs, edge_mask1) intersection_sum = np.sum(intersection) union_sum = np.sum(union) iou = (intersection_sum) / (union_sum) total = np.sum(arrs) correct_predictions = intersection_sum accuracy = correct_predictions / total # print(iou, accuracy) return iou, accuracy iou_hd_dict = np.empty((len(gt), len(pred), 3)) # IOU, HD95, AHD for i, each_gt_instance in enumerate(gt_mask): for j, each_pred_instance in enumerate(pred_mask): inst_iou, _ = compute_iou_and_accuracy(gt_mask[i], pred_mask[j]) inst_hd95 = hd95(gt_mask[i], pred_mask[j]) inst_ahd = assd(gt_mask[i], pred_mask[j]) iou_hd_dict[i][j][0] = inst_iou iou_hd_dict[i][j][1] = inst_hd95 iou_hd_dict[i][j][2] = inst_ahd corr_matrix = np.empty((len(gt), 4)) corr_matrix[:, 0] = np.argmax(iou_hd_dict[:, :, 0], 1) for i, each_gt_instance_metric in enumerate(iou_hd_dict): corr_matrix[i, 1:] = each_gt_instance_metric[corr_matrix[i, 0].astype(np.int)] gt_mask_copy = gt_mask.copy() gt_mask_copy = np.repeat(gt_mask_copy[:, :, :, np.newaxis], 3, axis=3) def bbox2(img): rows = np.any(img, axis=1) cols = np.any(img, axis=0) rmin, rmax = np.where(rows)[0][[0, -1]] cmin, cmax = np.where(cols)[0][[0, -1]] return rmin, rmax, cmin, cmax for i, each_gt_instance in enumerate(gt_mask): rmin, rmax, cmin, cmax = bbox2(each_gt_instance) pos = (int(cmin), int((rmin + rmax) / 2)) gt_mask_copy[i] = cv2.cvtColor(each_gt_instance, cv2.COLOR_GRAY2RGB) cv2.putText(gt_mask_copy[i], f"{i}", pos, cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 0, 0), 2) gt_image = gt_mask_copy.sum(axis=0).clip(0, 255).astype(np.uint8) # x = plt.imshow(gt_image) # plt.show() pred_image = pred_mask.sum(axis=0).clip(0, 255) pred_image_copy = pred_image.copy() pred_image_copy = np.repeat(pred_image_copy[:, :, np.newaxis], 3, axis=2).astype(np.uint8) for i, each_pred_instance in enumerate(pred_mask): rmin, rmax, cmin, cmax = bbox2(each_pred_instance) pos = (int(cmin), int((rmin + rmax) / 2)) # pred_image_copy[i] = cv2.cvtColor(each_pred_instance, cv2.COLOR_GRAY2RGB) if i in corr_matrix[:, 0].astype(np.int): if self.write_to_csv: metrics = (corr_matrix[np.where(i == corr_matrix[:, 0]), 1:]).squeeze() if metrics.ndim > 1: metrics = metrics[np.argmax(metrics[:, 0])] gt_idx = np.where((metrics == corr_matrix[:, 1:]).all(axis=1))[0].item() self.metrics_for_csv.append( { "region_name": f"{osp.splitext('_'.join(input['file_name'].split('/')[-3:]))[0]}_gt_{reg_type}_{gt_idx}", "iou": metrics[0].round(2), "ahd": metrics[2].round(2), "hd95": metrics[1].round(2), } ) continue text_metrics = np.array2string( (corr_matrix[np.where(i == corr_matrix[:, 0]), 1:].round(4)).squeeze() ) cv2.putText( pred_image_copy, f"{i} - {text_metrics}", pos, cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 0, 0), 2, ) else: cv2.putText(pred_image_copy, f"{i}", pos, cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 0), 1) # x = plt.imshow(pred_image_copy) # plt.show() if not self.write_to_csv: save_path = "final_outputs/comparision/bmrcnn_masks_whew/" import os file_name = os.path.splitext("_".join(input["file_name"].split("/")[-3:]))[0] cv2.imwrite( save_path + f"{file_name}_gt_{reg_type}.jpg", cv2.cvtColor(gt_image, cv2.COLOR_RGB2BGR) ) cv2.imwrite( save_path + f"{file_name}_pred_{reg_type}.jpg", cv2.cvtColor(pred_image_copy, cv2.COLOR_RGB2BGR), ) # res_iou, res_accuracy = compute_iou_and_accuracy(pred_mask, gt_mask) # res_ahd, res_hd, res_hd95 = assd(pred_mask, gt_mask), hd(pred_mask, gt_mask), hd95(pred_mask, gt_mask) # self.ahd[categories_list[reg_type]].append(res_ahd) # self.hd[categories_list[reg_type]].append(res_hd) # self.hd95[categories_list[reg_type]].append(res_hd95) # self.hd[categories_list[reg_type]].append(hd(pred_mask, gt_mask)) # self.iou[categories_list[reg_type]].append(res_iou) # self.acc[categories_list[reg_type]].append(res_accuracy) # # doc_ahd[categories_list[reg_type]].append(res_ahd) # doc_hd[categories_list[reg_type]].append(res_hd) # doc_hd95[categories_list[reg_type]].append(res_hd95) # doc_hd[categories_list[reg_type]].append(hd(pred_mask, gt_mask)) # doc_iou[categories_list[reg_type]].append(res_iou) # doc_acc[categories_list[reg_type]].append(res_accuracy) # One has points # elif len(gt) ^ len(pred): # total_area = 0 # for each_pred in pred: # total_area += PolyArea(each_pred[:, 0], each_pred[:, 1]) # hd = total_area / 100 # self.ahd[categories_list[reg_type]].append(hd) # self.hd[categories_list[reg_type]].append(hd) # self.hd95[categories_list[reg_type]].append(hd) # self.iou[categories_list[reg_type]].append(0) # self.acc[categories_list[reg_type]].append(0) # Both Empty # elif len(gt) == 0 and len(pred) != 0: else: # self.hd[categories_list[reg_type]].append(0) pass # print("Over for doc") total_ahd = list() for l in doc_ahd.values(): total_ahd.extend(l) doc_ahd = np.mean(total_ahd) total_hd = list() for l in doc_hd.values(): total_hd.extend(l) doc_hd = np.mean(total_hd) total_hd95 = list() for l in doc_hd95.values(): total_hd95.extend(l) doc_hd95 = np.mean(total_hd95) total_iou = list() for l in doc_iou.values(): total_iou.extend(l) doc_iou = np.mean(total_iou) total_acc = list() for l in doc_acc.values(): total_acc.extend(l) doc_acc = np.mean(total_acc) self.doc_wise[input["file_name"]] = { "AHD": doc_ahd, "IOU": doc_iou, "HD": doc_hd, "HD95": doc_hd95, "ACC": doc_acc, }
def process(self, inputs, outputs): for input, output in zip(inputs, outputs): self.count += len(output['instances']) gt_segm = self.annotations[input['file_name']]['segm_per_region'] try: _ = output['instances'].pred_masks except AttributeError: continue pred_segm = downsample_points(output) doc_ahd = {cat: [] for cat in categories_list} doc_hd = {cat: [] for cat in categories_list} doc_hd95 = {cat: [] for cat in categories_list} doc_iou = {cat: [] for cat in categories_list} doc_acc = {cat: [] for cat in categories_list} for reg_type in range(len(categories_list)): gt, pred = gt_segm[reg_type], pred_segm[reg_type] # Both have points if len(gt) and len(pred): gt_mask = np.zeros((input['height'], input['width']), dtype=np.int8) for i in gt: cv2.fillPoly(gt_mask, np.array([i]).astype(np.int32), 1) pred_mask = np.zeros((input['height'], input['width']), dtype=np.int8) for i in pred: cv2.fillPoly(pred_mask, np.array([i]).astype(np.int32), 1) gt_mask = gt_mask.astype(np.uint8) gt_mask = (gt_mask * 255).astype(np.uint8) pred_mask = pred_mask.astype(np.uint8) pred_mask = (pred_mask * 255).astype(np.uint8) def compute_iou_and_accuracy(arrs, edge_mask1): intersection = cv2.bitwise_and(arrs, edge_mask1) union = cv2.bitwise_or(arrs, edge_mask1) intersection_sum = np.sum(intersection) union_sum = np.sum(union) iou = (intersection_sum) / (union_sum) total = np.sum(arrs) correct_predictions = intersection_sum accuracy = correct_predictions / total # print(iou, accuracy) return iou, accuracy res_iou, res_accuracy = compute_iou_and_accuracy( pred_mask, gt_mask) res_ahd, res_hd, res_hd95 = assd(pred_mask, gt_mask), hd( pred_mask, gt_mask), hd95(pred_mask, gt_mask) self.ahd[categories_list[reg_type]].append(res_ahd) self.hd[categories_list[reg_type]].append(res_hd) self.hd95[categories_list[reg_type]].append(res_hd95) self.hd[categories_list[reg_type]].append( hd(pred_mask, gt_mask)) self.iou[categories_list[reg_type]].append(res_iou) self.acc[categories_list[reg_type]].append(res_accuracy) doc_ahd[categories_list[reg_type]].append(res_ahd) doc_hd[categories_list[reg_type]].append(res_hd) doc_hd95[categories_list[reg_type]].append(res_hd95) doc_hd[categories_list[reg_type]].append( hd(pred_mask, gt_mask)) doc_iou[categories_list[reg_type]].append(res_iou) doc_acc[categories_list[reg_type]].append(res_accuracy) # One has points # elif len(gt) ^ len(pred): # total_area = 0 # for each_pred in pred: # total_area += PolyArea(each_pred[:, 0], each_pred[:, 1]) # hd = total_area / 100 # self.ahd[categories_list[reg_type]].append(hd) # self.hd[categories_list[reg_type]].append(hd) # self.hd95[categories_list[reg_type]].append(hd) # self.iou[categories_list[reg_type]].append(0) # self.acc[categories_list[reg_type]].append(0) # Both Empty # elif len(gt) == 0 and len(pred) != 0: else: # self.hd[categories_list[reg_type]].append(0) pass # print("Over for doc") total_ahd = list() for l in doc_ahd.values(): total_ahd.extend(l) doc_ahd = np.mean(total_ahd) total_hd = list() for l in doc_hd.values(): total_hd.extend(l) doc_hd = np.mean(total_hd) total_hd95 = list() for l in doc_hd95.values(): total_hd95.extend(l) doc_hd95 = np.mean(total_hd95) total_iou = list() for l in doc_iou.values(): total_iou.extend(l) doc_iou = np.mean(total_iou) total_acc = list() for l in doc_acc.values(): total_acc.extend(l) doc_acc = np.mean(total_acc) self.doc_wise[input['file_name']] = { "AHD": doc_ahd, "IOU": doc_iou, "HD": doc_hd, "HD95": doc_hd95, "ACC": doc_acc }
def main(FLAGS): # set GPU device to use os.environ['CUDA_VISIBLE_DEVICES'] = '3' # define the experiments encoders = FLAGS.encoders.split(',') losses = FLAGS.losses.split(',') alpha = FLAGS.loss_param1.split(',') experiments = [encoders, losses, alpha] print(experiments) # get data files and sort them image_files = os.listdir(FLAGS.test_X_dir) anno_files = os.listdir(FLAGS.test_y_dir) image_files.sort() anno_files.sort() for experiment in itertools.product(*experiments): # switch to activate training session do_eval = True # load the base configurations if experiment[0] == 'UNet': configs = load_config( os.path.join(FLAGS.base_configs_dir, 'unet2d.json')) elif experiment[0] == 'UNet3D': configs = load_config( os.path.join(FLAGS.base_configs_dir, 'unet3d.json')) elif experiment[0] == 'VGG16' or experiment[0] == 'VGG19': configs = load_config( os.path.join(FLAGS.base_configs_dir, 'vgg16_unet.json')) else: configs = load_config( os.path.join(FLAGS.base_configs_dir, 'xception_unet.json')) # set the path to the model (checkpoints are fine for this) ckpt_name = 'encoder_{}_loss_{}_alpha_{}_beta_{}_ckpt.h5'.format( experiment[0], experiment[1], experiment[2], 1. - literal_eval(experiment[2])) configs['paths']['load_model'] = os.path.join(FLAGS.ckpt_dir, ckpt_name) # switch to inference configs['config_file']['type_signal'] = 'Inference' # perform some preprocessing configs['preprocessing']['categorical_switch'] = 'True' configs['preprocessing']['minimum_image_intensity'] = '0.0' configs['preprocessing']['maximum_image_intensity'] = '2048.0' configs['preprocessing']['normalization_type'] = '{}'.format( FLAGS.normalization) # set some other configurations configs['training_configurations']['batch_size'] = '{}'.format( FLAGS.batch_size) configs['config_file']['input_shape'] = '({}, {}, {})'.format( FLAGS.height, FLAGS.width, FLAGS.channels) # ensure xentropy/jaccard/focal only used once per encoder if experiment[1] == 'sparse_categorical_crossentropy'\ or experiment[1] == 'categorical_crossentropy'\ or experiment[1] == 'jaccard'\ or experiment[1] == 'focal': if experiment[1] == 'focal': configs['loss_function']['parameter1'] = '0.75' configs['loss_function']['parameter2'] = '2.0' if experiment[1] == 'jaccard': configs['loss_function']['parameter1'] = '100.0' if experiment[2] == '0.3': configs['loss_function']['loss'] = experiment[1] else: do_eval = False elif experiment[1] == 'tversky': configs['loss_function']['loss'] = experiment[1] configs['loss_function']['parameter1'] = experiment[2] configs['loss_function']['parameter2'] = str( 1. - literal_eval(experiment[2])) else: do_eval = False # create a location to store evaluation metrics metrics = np.zeros((len(image_files), FLAGS.classes, 8)) overall_accuracy = np.zeros((len(image_files), )) # create a file writer to store the metrics excel_name = '{}_{}_{}_{}_metrics.xlsx'.format( experiment[0], experiment[1], experiment[2], 1. - literal_eval(experiment[2])) writer = pd.ExcelWriter(excel_name) for i in range(len(image_files)): K.clear_session() # define path to the test data configs['paths']['test_X'] = os.path.join(FLAGS.test_X_dir, image_files[i]) if do_eval is True: configs_lvl1, errors_lvl1, warnings_lvl1 = level_one_error_checking( configs) if any(warnings_lvl1): with open('errors.txt', 'a') as f: for warning in warnings_lvl1: f.write("%s\n" % warning) f.close() print('Level 1 warnings encountered.') print( "The following level 1 warnings were identified and corrected based on engine defaults:" ) for warning in warnings_lvl1: print(warning) if any(errors_lvl1): print('Level 1 errors encountered.') print( "Please fix the level 1 errors below before continuing:" ) for error in errors_lvl1: print(error) else: configs_lvl2, errors_lvl2, warnings_lvl2 = level_two_error_checking( configs_lvl1) if any(warnings_lvl2): print('Level 2 warnings encountered.') print( "The following level 2 warnings were identified and corrected based on engine defaults:" ) for warning in warnings_lvl2: print(warning) if any(errors_lvl2): print('Level 2 errors encountered.') print( "Please fix the level 2 errors below before continuing:" ) for error in errors_lvl2: print(error) else: engine = Dlae(configs) engine.run() if any(engine.errors): print('Level 3 errors encountered.') print( "Please fix the level 3 errors below before continuing:" ) for error in engine.errors: print(error) # check if the images and annotations are the correct files print(image_files[i], anno_files[i]) pred_file = glob( os.path.join(FLAGS.predictions_temp_dir, '*.h5'))[0] pt_name = image_files[i].split('.')[0] new_name_raw = pt_name + '_{}_{}_{}_{}_raw.h5'.format( experiment[0], experiment[1], experiment[2], 1. - literal_eval(experiment[2])) new_file_raw = os.path.join(FLAGS.predictions_final_dir, new_name_raw) os.rename(pred_file, new_file_raw) ref = read_hdf5_multientry( os.path.join(FLAGS.test_y_dir, anno_files[i])) ref = np.squeeze(np.asarray(ref)) preds = read_hdf5(new_file_raw) if experiment[0] == 'UNet3D': ref = np.transpose(ref, (1, 2, 0)) # stich the image back together first sw = SlidingWindow(ref, [96, 96, 40], [128, 128, 48]) preds = sw.stitch_patches(preds, sw.window_corner_coords, [128, 128, 48], sw.img_shape, FLAGS.classes) preds = np.argmax(preds, axis=-1) else: preds = np.argmax(preds, axis=-1) overall_accuracy[i] = skm.accuracy_score( ref.flatten(), preds.flatten()) for j in range(FLAGS.classes): organ_pred = (preds == j).astype(np.int64) organ_ref = (ref == j).astype(np.int64) if np.sum(organ_pred) == 0 or np.sum(organ_ref) == 0: metrics[i, j, 0] = 0. metrics[i, j, 1] = 0. metrics[i, j, 2] = 1. metrics[i, j, 3] = 0. metrics[i, j, 4] = 0. metrics[i, j, 5] = 0. metrics[i, j, 6] = np.inf metrics[i, j, 7] = np.inf else: metrics[i, j, 0] = jaccard_index(organ_ref, organ_pred) metrics[i, j, 1] = dice_similarity_coefficient( organ_ref, organ_pred) metrics[i, j, 2] = relative_volume_difference( organ_ref, organ_pred) metrics[i, j, 3] = precision(organ_ref, organ_pred) metrics[i, j, 4] = recall(organ_ref, organ_pred) metrics[i, j, 5] = matthews_correlation_coefficient( organ_ref, organ_pred) metrics[i, j, 6] = mpm.hd95(organ_pred, organ_ref) metrics[i, j, 7] = mpm.assd(organ_pred, organ_ref) print(overall_accuracy[i]) print(metrics[i]) else: pass if do_eval is True: for k in range(metrics.shape[-1]): data = pd.DataFrame( metrics[:, :, k], columns=['bg', 'pros', 'eus', 'sv', 'rect', 'blad']) data.to_excel(writer, sheet_name=str(k)) acc = pd.DataFrame(overall_accuracy, columns=['acc']) acc.to_excel(writer, sheet_name='acc') writer.save()