def validate(self, epoch_loss): # Report validation error valid_data = self.loader.load_all_modalities_concatenated(self.conf.split, 'validation', self.conf.image_downsample) valid_data.crop(self.conf.input_shape[:2]) images0 = valid_data.get_images_modi(0) images1 = valid_data.get_images_modi(1) real_mask0 = valid_data.get_masks_modi(0) real_mask1 = valid_data.get_masks_modi(1) s1 = self.model.Encoders_Anatomy[0].predict(images0) s2 = self.model.Encoders_Anatomy[1].predict(images1) s1_deformed, s_fused = self.model.Anatomy_Fuser.predict([s1, s2]) mask1 = self.model.Segmentor.predict(s1) mask2 = self.model.Segmentor.predict(s2) mask3 = self.model.Segmentor.predict(s1_deformed) mask4 = self.model.Segmentor.predict(s_fused) l_mod1 = (1 - costs.dice(real_mask0, mask1, binarise=True)) l_mod2 = (1 - costs.dice(real_mask1, mask2, binarise=True)) l_mod2_s1def = (1 - costs.dice(real_mask1, mask3, binarise=True)) l_mod2_fused = (1 - costs.dice(real_mask1, mask4, binarise=True)) epoch_loss['val_loss_mod2'].append(l_mod2) epoch_loss['val_loss_mod2_s1def'].append(l_mod2_s1def) epoch_loss['val_loss_mod2_fused'].append(l_mod2_fused) epoch_loss['val_loss_mod1'].append(l_mod1) epoch_loss['val_loss'].append(np.mean([l_mod1, l_mod2, l_mod2_s1def, l_mod2_fused]))
def on_epoch_end(self, epoch, logs=None): if not os.path.exists(self.folder): os.makedirs(self.folder) all_dice = [] for i in range(len(self.test_data)): d, m = self.test_data[i], self.test_masks[i] s, im = save_segmentation(self.folder, self.model, d, m, 'slc_%d' % i) all_dice.append(-dice(self.test_masks[i:i + 1], s)) if self.comet_experiment is not None: plt.figure() plt.plot(0, 0) # fake a line plot to upload to comet plt.imshow(im, cmap='gray') plt.xticks([]) plt.yticks([]) plt.tight_layout() self.comet_experiment.log_figure(figure_name='segmentation', figure=plt) plt.close() f = open(os.path.join(self.folder, 'test_error.txt'), 'a+') f.writelines("%d, %.3f\n" % (epoch, np.mean(all_dice))) f.close()
def validate(self, epoch_loss): """ Report validation error :param epoch_loss: dictionary of losses """ valid_data = self.loader.load_labelled_data(self.conf.split, 'validation') mask, _ = self.sdnet.Decomposer.predict(valid_data.images) assert mask.shape == valid_data.masks.shape epoch_loss['val_loss'].append((1 - costs.dice(valid_data.masks, mask)))
def on_epoch_end(self, epoch, logs=None): if not os.path.exists(self.folder): os.makedirs(self.folder) all_dice = [] for i in range(len(self.test_data)): d, m = self.test_data[i], self.test_masks[i] s = save_segmentation(self.folder, self.model, d, m, 'slc_%d' % i) all_dice.append(-dice(self.test_masks[i:i + 1], s)) f = open(os.path.join(self.folder, 'test_error.txt'), 'a+') f.writelines("%d, %.3f\n" % (epoch, np.mean(all_dice))) f.close()
def validate(self, epoch_loss): # Report validation error valid_data = \ self.loader.load_labelled_data(self.conf.split, 'validation', modality=self.conf.modality, downsample=self.conf.image_downsample) # harric added modality and segmentation_option arguments valid_data.crop(self.conf.input_shape[:2]) anatomy_segmentor = self.model.Segmentor anatomy_encoder = self.model.Enc_Anatomy pathology_segmentor = self.model.Enc_Pathology s = anatomy_encoder.predict(valid_data.images) anatomy_pred_mask = anatomy_segmentor.predict(s) pathology_pred_mask = pathology_segmentor.predict(np.concatenate([valid_data.images,anatomy_pred_mask], axis=-1)) assert pathology_pred_mask.shape[:-1] == valid_data.images.shape[:-1], \ str(valid_data.images.shape) + ' ' + str(pathology_pred_mask.shape) epoch_loss['Validate_Dice'].append(1 - costs.dice(valid_data.patho_masks, pathology_pred_mask)[0]) return costs.dice(valid_data.patho_masks,pathology_pred_mask)[0]
def validate(self, epoch_loss): # Report validation error valid_data = self.loader.load_labelled_data( self.conf.split, 'validation', modality=self.conf.modality, downsample=self.conf.image_downsample) valid_data.crop(self.conf.input_shape[:2]) mask = self.model.Segmentor.predict( self.model.Enc_Anatomy.predict(valid_data.images)) assert mask.shape[:-1] == valid_data.images.shape[:-1], str( valid_data.images.shape) + ' ' + str(mask.shape) epoch_loss['val_loss'].append((1 - costs.dice(valid_data.masks, mask)))
def test_modality_type(self, folder, modality_index, type, test_loader, test_data): assert type in ['simple', 'def', 'max', 'maxnostn'] samples = os.path.join(folder, 'samples') if not os.path.exists(samples): os.makedirs(samples) synth = [] im_dice = {} f = open(os.path.join(folder, 'results.csv'), 'w') f.writelines('Vol, Dice, ' + ', '.join(['Dice%d' % mi for mi in range(test_loader.num_masks)]) + '\n') for vol_i in test_data.volumes(): vol_folder = os.path.join(samples, 'vol_%s' % str(vol_i)) if not os.path.exists(vol_folder): os.makedirs(vol_folder) vol_image_mod1 = test_data.get_volume_images_modi(0, vol_i) vol_image_mod2 = test_data.get_volume_images_modi(1, vol_i) assert vol_image_mod1.shape[0] > 0 vol_mask = test_data.get_volume_masks_modi(modality_index, vol_i) prd_mask = self.model.predict_mask(modality_index, type, [vol_image_mod1, vol_image_mod2]) synth.append(prd_mask) im_dice[vol_i] = costs.dice(vol_mask, prd_mask, binarise=True) sep_dice = [costs.dice(vol_mask[..., mi:mi + 1], prd_mask[..., mi:mi + 1], binarise=True) for mi in range(test_loader.num_masks)] s = '%s, %.3f, ' + ', '.join(['%.3f'] * test_loader.num_masks) + '\n' d = (str(vol_i), im_dice[vol_i]) + tuple(sep_dice) f.writelines(s % d) self.plot_images(samples, vol_i, modality_index, prd_mask, vol_mask, [vol_image_mod1, vol_image_mod2]) print('%s - Dice score: %.3f' % (type, np.mean(list(im_dice.values())))) f.close()
def test(self): """ Evaluate a model on the test data. """ log.info('Evaluating model on test data') folder = os.path.join(self.conf.folder, 'test_results_%s' % self.conf.dataset_name) if not os.path.exists(folder): os.makedirs(folder) test_loader = loader_factory.init_loader(self.conf.dataset_name) test_data = test_loader.load_labelled_data(self.conf.split, 'test') synth = [] im_dice = {} samples = os.path.join(folder, 'samples') if not os.path.exists(samples): os.makedirs(samples) f = open(os.path.join(folder, 'results.csv'), 'w') f.writelines('Vol, Dice\n') for vol_i in test_data.volumes(): vol_folder = os.path.join(samples, 'vol_%s' % str(vol_i)) if not os.path.exists(vol_folder): os.makedirs(vol_folder) vol_image = test_data.get_volume_image(vol_i) vol_mask = test_data.get_volume_mask(vol_i) assert vol_image.shape[0] > 0 and vol_image.shape == vol_mask.shape pred, _ = self.sdnet.Decomposer.predict(vol_image) synth.append(pred) im_dice[vol_i] = costs.dice(vol_mask, pred) f.writelines('%s, %.3f\n' % (str(vol_i), im_dice[vol_i])) for i in range(vol_image.shape[0]): im = np.concatenate([ vol_image[i, :, :, 0], pred[i, :, :, 0], vol_mask[i, :, :, 0] ], axis=1) scipy.misc.imsave( os.path.join(vol_folder, 'test_vol%d_sl%d.png' % (vol_i, i)), im) print('Dice score: %.3f' % np.mean(list(im_dice.values()))) f.close()
def validate(self, epoch_loss): """ Calculate losses on validation set :param epoch_loss: the dictionary to save the losses """ valid_data = self.loader.load_all_modalities_concatenated( self.conf.split, 'validation', self.conf.image_downsample) if (hasattr(self.conf, 'randomise') and self.conf.randomise): valid_data.randomise_pairs(length=self.conf.n_pairs - 1) valid_data.crop(self.conf.input_shape[:2]) images0 = valid_data.get_images_modi(0) images1 = valid_data.get_images_modi(1) masks0 = valid_data.get_masks_modi(0) masks1 = valid_data.get_masks_modi(1) s1 = self.swa_Enc_Anatomy1.get_clone_model().predict(images0) s2 = self.swa_Enc_Anatomy2.get_clone_model().predict(images1) z1, _ = self.swa_Enc_Modality.get_clone_model().predict([s1, images0]) z2, _ = self.swa_Enc_Modality.get_clone_model().predict([s2, images1]) swa_Anatomy_Fuser_model = self.swa_Anatomy_Fuser.get_clone_model() s1_deformed, s2_fused = swa_Anatomy_Fuser_model.predict([s1, s2]) s2_deformed, s1_fused = swa_Anatomy_Fuser_model.predict([s2, s1]) swa_Segmentor_model = self.swa_Segmentor.get_clone_model() m1_s1 = swa_Segmentor_model.predict(s1) m2_s2 = swa_Segmentor_model.predict(s2) m2_s1def = swa_Segmentor_model.predict(s1_deformed) m1_s2def = swa_Segmentor_model.predict(s2_deformed) m2_fused = swa_Segmentor_model.predict(s2_fused) m1_fused = swa_Segmentor_model.predict(s1_fused) assert m1_s1.shape[:-1] == images0.shape[:-1], str( images0) + ' ' + str(m1_s1.shape) assert m2_s2.shape[:-1] == images0.shape[:-1], str( images0.shape) + ' ' + str(m2_s2.shape) assert m2_s1def.shape[:-1] == images0.shape[:-1], str( images0.shape) + ' ' + str(m2_s1def.shape) dice_m1s1 = (1 - costs.dice(masks0, m1_s1, binarise=True)) dice_m1s2def = (1 - costs.dice(masks0, m1_s2def, binarise=True)) dice_m1fused = (1 - costs.dice(masks0, m1_fused, binarise=True)) dice_m2s2 = (1 - costs.dice(masks1, m2_s2, binarise=True)) dice_m2s1def = (1 - costs.dice(masks1, m2_s1def, binarise=True)) dice_m2fused = (1 - costs.dice(masks1, m2_fused, binarise=True)) epoch_loss['val_loss_mod2'].append(dice_m2s2) epoch_loss['val_loss_mod2_mod1def'].append(dice_m2s1def) epoch_loss['val_loss_mod2_fused'].append(dice_m2fused) epoch_loss['val_loss_mod1_mod2def'].append(dice_m1s2def) epoch_loss['val_loss_mod1_fused'].append(dice_m1fused) epoch_loss['val_loss_mod1'].append(dice_m1s1) epoch_loss['val_loss'].append( np.mean([dice_m1s1, dice_m2s2, dice_m2s1def, dice_m2fused])) if self.conf.automatedpairing: valid_data.expand_pairs(self.conf.n_pairs - 1, 0, neighborhood=self.conf.n_pairs) images0 = valid_data.get_images_modi(0) images0_list = [ images0[..., i:i + 1] for i in range(images0.shape[-1]) ] images1 = valid_data.get_images_modi(1) s1_list = [ self.model.Encoders_Anatomy[0].predict(x) for x in images0_list ] s2 = self.model.Encoders_Anatomy[1].predict(images1) weights = self.model.Balancer.predict([s2] + s1_list) weights = [ np.mean(weights[..., j]) for j in range(weights.shape[-1]) ] for j in range(len(weights)): epoch_loss['val_weight_%d' % j].append(weights[j])
def test_modality(self, folder, modality, group, save_figs=True): test_loader = loader_factory.init_loader(self.conf.test_dataset) test_data = test_loader.load_labelled_data( self.conf.split, group, modality=modality, downsample=self.conf.image_downsample) anatomy_segmentor = self.model.get_anatomy_segmentor() pathology_segmentator = self.model.get_pathology_encoder() synth = [] im_dice_anato, im_false_negative_anato = {}, {} im_dice_patho, im_false_negative_patho = {}, {} sep_dice_list_anato, sep_false_negative_list_anato = [], [] sep_dice_list_patho, sep_false_negative_list_patho = [], [] anato_mask_num = len(test_data.anato_mask_names) patho_mask_num = len(test_data.patho_mask_names) for ii in range(anato_mask_num): sep_dice_list_anato.append([]) sep_false_negative_list_anato.append([]) for ii in range(patho_mask_num): sep_dice_list_patho.append([]) sep_false_negative_list_patho.append([]) f = open(os.path.join(folder, 'results.csv'), 'w') for vol_i in test_data.volumes(): vol_image = test_data.get_images(vol_i) vol_anato_mask = test_data.get_anato_masks(vol_i) vol_patho_mask = test_data.get_patho_masks(vol_i) vol_slice = test_data.get_slice(vol_i) assert vol_image.shape[ 0] > 0 and vol_image.shape[: -1] == vol_anato_mask.shape[: -1] and vol_image.shape[: -1] == vol_patho_mask.shape[: -1] anato_pred = anatomy_segmentor.predict(vol_image) patho_pred = pathology_segmentator.predict(vol_image) pred = [anato_pred, patho_pred] synth.append(pred) model_type = 'sdnet' im_dice_anato[vol_i], sep_dice_anato \ = dice(vol_anato_mask, pred[0]) im_false_negative_anato[vol_i], sep_false_negative_anato \ = calculate_false_negative(vol_anato_mask, pred[0]) im_dice_patho[vol_i], sep_dice_patho \ = dice(vol_patho_mask, pred[1]) im_false_negative_patho[vol_i], sep_false_negative_patho \ = calculate_false_negative(vol_patho_mask, pred[1]) # harric added to specify dice scores across different masks assert anato_mask_num == len( sep_dice_anato), 'Incorrect mask num !' assert patho_mask_num == len( sep_dice_patho), 'Incorrect mask num !' for ii in range(anato_mask_num): sep_dice_list_anato[ii].append(sep_dice_anato[ii]) sep_false_negative_list_anato[ii].append( sep_false_negative_anato[ii]) for ii in range(patho_mask_num): sep_dice_list_patho[ii].append(sep_dice_patho[ii]) sep_false_negative_list_patho[ii].append( sep_false_negative_patho[ii]) # harric added to specify dice scores across different masks s = 'Volume:%s, AnatomyDice:%.3f, AnatomyFN:%.3f, ' \ + 'PathologyDice:%.3f, PathologyFN:%.3f, ' \ + ', '.join(['%s, %.3f, %.3f, '] * len(test_data.anato_mask_names)) \ + ', '.join(['%s, %.3f, %.3f, '] * len(test_data.patho_mask_names)) \ + '\n' d = (str(vol_i), im_dice_anato[vol_i], im_false_negative_anato[vol_i]) d += (im_dice_patho[vol_i], im_false_negative_patho[vol_i]) for info_travesal in range(anato_mask_num): d += (test_data.anato_mask_names[info_travesal], sep_dice_anato[info_travesal], sep_false_negative_anato[info_travesal]) for info_travesal in range(patho_mask_num): d += (test_data.patho_mask_names[info_travesal], sep_dice_patho[info_travesal], sep_false_negative_patho[info_travesal]) f.writelines(s % d) if save_figs: for i in range(vol_image.shape[0]): d, m, mm = vol_image[i], vol_anato_mask[i], vol_patho_mask[ i] # d, m, mm = vol_image[10], vol_anato_mask[10], vol_patho_mask[10] s = vol_slice[i] im1 = save_segmentation(pred[0][i, :, :, :], d, m) im2 = save_segmentation(pred[1][i, :, :, :], d, mm) if im1.shape[1] > im2.shape[1]: im2 = np.concatenate([ im2, np.zeros(shape=(im2.shape[0], im1.shape[1] - im2.shape[1]), dtype=im2.dtype) ], axis=1) elif im1.shape[1] < im2.shape[1]: im1 = np.concatenate([ im1, np.zeros(shape=(im1.shape[0], im2.shape[1] - im1.shape[1]), dtype=im1.dtype) ], axis=1) im = np.concatenate([im1, im2], axis=0) imsave( os.path.join( folder, "vol%s_slice%s" % (str(vol_i), s) + '.png'), im) # harric added to specify dice scores across different masks print_info = group + ', AnatomyDice:%.3f, AnatoFN:%.3f, PathoDice:%.3f, PathoFN:%.3f,' % \ (np.mean(list(im_dice_anato.values())), np.mean(list(im_false_negative_anato.values())), np.mean(list(im_dice_patho.values())), np.mean(list(im_false_negative_patho.values()))) for ii in range(anato_mask_num): print_info += '%s, %.3f, %.3f,' % \ (test_data.anato_mask_names[ii], np.mean(sep_dice_list_anato[ii]), np.mean(sep_false_negative_list_anato[ii])) for ii in range(patho_mask_num): print_info += '%s, %.3f, %.3f' % \ (test_data.patho_mask_names[ii], np.mean(sep_dice_list_patho[ii]), np.mean(sep_false_negative_list_patho[ii])) print(print_info) f.write(print_info) f.close() return np.mean(list(im_dice_patho.values()))