def __init__(self, plans_file, fold, output_folder=None, dataset_directory=None, batch_dice=True, stage=None, unpack_data=True, deterministic=True, fp16=False): super().__init__(plans_file, fold, output_folder, dataset_directory, batch_dice, stage, unpack_data, deterministic, fp16) self.regions = get_brats_regions() self.regions_class_order = (1, 2, 3) self.loss = DC_and_BCE_loss({}, {'batch_dice': False, 'do_bg': True, 'smooth': 0})
def main(): args = get_args() log_file = args.log_file preprocessed_data_dir = args.preprocessed_data_dir output_folder = args.postprocessed_data_dir ground_truths = args.label_data_dir output_dtype = dtype_map[args.output_dtype] num_threads_nifti_save = args.num_threads_nifti_save all_in_gpu = "None" force_separate_z = None interp_order = 3 interp_order_z = 0 if not os.path.exists(output_folder): os.makedirs(output_folder) # Load necessary metadata. print("Loading necessary metadata...") with open(os.path.join(preprocessed_data_dir, "preprocessed_files.pkl"), "rb") as f: preprocessed_files = pickle.load(f) dictionaries = [] for preprocessed_file in preprocessed_files: with open( os.path.join(preprocessed_data_dir, preprocessed_file + ".pkl"), "rb") as f: dct = pickle.load(f)[1] dictionaries.append(dct) # Load predictions from loadgen accuracy log. print("Loading loadgen accuracy log...") predictions = load_loadgen_log(log_file, output_dtype, dictionaries) # Save predictions # This runs in multiprocess print("Running postprocessing with multiple threads...") save_predictions_MLPerf(predictions, output_folder, preprocessed_files, dictionaries, num_threads_nifti_save, all_in_gpu, force_separate_z, interp_order, interp_order_z) # Run evaluation print("Running evaluation...") evaluate_regions(output_folder, ground_truths, get_brats_regions()) # Load evaluation summary print("Loading evaluation summary...") with open(os.path.join(output_folder, "summary.csv")) as f: for line in f: words = line.split(",") if words[0] == "mean": whole = float(words[1]) core = float(words[2]) enhancing = float(words[3]) mean = (whole + core + enhancing) / 3 print( "Accuracy: mean = {:.5f}, whole tumor = {:.4f}, tumor core = {:.4f}, enhancing tumor = {:.4f}" .format(mean, whole, core, enhancing)) break print("Done!")
def __init__(self, plans_file, fold, output_folder=None, dataset_directory=None, batch_dice=True, stage=None, unpack_data=True, deterministic=True, fp16=False): super().__init__(plans_file, fold, output_folder, dataset_directory, batch_dice, stage, unpack_data, deterministic, fp16) self.validate_every = 1 self.evaluation_regions = get_brats_regions() self.num_val_batches_per_epoch = 0 # we dont need this because this does not evaluate on full images
def __init__(self, plans_file, fold, local_rank, output_folder=None, dataset_directory=None, batch_dice=True, stage=None, unpack_data=True, deterministic=True, distribute_batch_size=False, fp16=False): super().__init__(plans_file, fold, local_rank, output_folder, dataset_directory, batch_dice, stage, unpack_data, deterministic, distribute_batch_size, fp16) self.regions = get_brats_regions() self.regions_class_order = (1, 2, 3) self.loss = None self.ce_loss = nn.BCEWithLogitsLoss()
def collect_and_prepare(base_dir, num_processes = 12, clean=False): """ collect all cv_niftis, compute brats metrics, compute enh tumor thresholds and summarize in csv :param base_dir: :return: """ out = join(base_dir, 'cv_results') out_pp = join(base_dir, 'cv_results_pp') experiments = subfolders(base_dir, join=False, prefix='nnUNetTrainer') regions = get_brats_regions() gt_dir = join(base_dir, 'gt_niftis') replace_with = 2 failed = [] successful = [] for e in experiments: print(e) try: o = join(out, e) o_p = join(out_pp, e) maybe_mkdir_p(o) maybe_mkdir_p(o_p) collect_cv_niftis(join(base_dir, e), o) if clean or not isfile(join(o, 'summary.csv')): evaluate_regions(o, gt_dir, regions, num_processes) if clean or not isfile(join(o_p, 'threshold.pkl')): determine_brats_postprocessing(o, gt_dir, o_p, num_processes, thresholds=list(np.arange(0, 760, 10)), replace_with=replace_with) if clean or not isfile(join(o_p, 'summary.csv')): evaluate_regions(o_p, gt_dir, regions, num_processes) successful.append(e) except Exception as ex: print("\nERROR\n", e, ex, "\n") failed.append(e) # we are interested in the mean (nan is 1) column with open(join(base_dir, 'cv_summary.csv'), 'w') as f: f.write('name,whole,core,enh,mean\n') for e in successful: expected_nopp = join(out, e, 'summary.csv') expected_pp = join(out, out_pp, e, 'summary.csv') if isfile(expected_nopp): res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_noPP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) if isfile(expected_pp): res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_PP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) # this just crawls the folders and evaluates what it finds with open(join(base_dir, 'cv_summary2.csv'), 'w') as f: for folder in ['cv_results', 'cv_results_pp']: for ex in subdirs(join(base_dir, folder), join=False): print(folder, ex) expected = join(base_dir, folder, ex, 'summary.csv') if clean or not isfile(expected): evaluate_regions(join(base_dir, folder, ex), gt_dir, regions, num_processes) if isfile(expected): res = np.loadtxt(expected, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write('%s__%s,' % (folder, ex)) f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) f.write('name,whole,core,enh,mean\n') for e in successful: expected_nopp = join(out, e, 'summary.csv') expected_pp = join(out, out_pp, e, 'summary.csv') if isfile(expected_nopp): res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_noPP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) if isfile(expected_pp): res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2] as_numeric = [float(i) for i in res[1:]] f.write(e + '_PP,') f.write("%0.4f," % as_numeric[0]) f.write("%0.4f," % as_numeric[1]) f.write("%0.4f," % as_numeric[2]) f.write("%0.4f\n" % np.mean(as_numeric)) # apply threshold to val set expected_num_cases = 125 missing_valset = [] has_val_pred = [] for e in successful: if isdir(join(base_dir, 'predVal', e)): currdir = join(base_dir, 'predVal', e) files = subfiles(currdir, suffix='.nii.gz', join=False) if len(files) != expected_num_cases: print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases)) continue output_folder = join(base_dir, 'predVal_PP', e) maybe_mkdir_p(output_folder) threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2] if threshold > 1000: threshold = 750 # don't make it too big! apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes) has_val_pred.append(e) else: print(e, 'has no valset predictions') missing_valset.append(e) # 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment e = 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5' currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold') output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold') maybe_mkdir_p(output_folder) threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2] if threshold > 1000: threshold = 750 # don't make it too big! apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes) # 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment e = 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5' currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold') output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold') maybe_mkdir_p(output_folder) threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2] if threshold > 1000: threshold = 750 # don't make it too big! apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes) # convert val set to brats labels for submission output_converted = join(base_dir, 'converted_valSet') for source in ['predVal', 'predVal_PP']: for e in has_val_pred + ['nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold']: expected_source_folder = join(base_dir, source, e) if not isdir(expected_source_folder): print(e, 'has no', source) raise RuntimeError() files = subfiles(expected_source_folder, suffix='.nii.gz', join=False) if len(files) != expected_num_cases: print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases)) continue target_folder = join(output_converted, source, e) maybe_mkdir_p(target_folder) convert_labels_back_to_BraTS_2018_2019_convention(expected_source_folder, target_folder) summarize_validation_set_predictions(output_converted)
def run(self): print("Run inference for accuracy") setup(self.args.data_location, self.args.input_graph) graph = tf.Graph() with graph.as_default(): graph_def = tf.compat.v1.GraphDef() with open(self.args.input_graph, "rb") as f: graph_def.ParseFromString(f.read()) output_graph = optimize_for_inference( graph_def, [INPUTS], [OUTPUTS], dtypes.float32.as_datatype_enum, False) tf.import_graph_def(output_graph, name="") input_tensor = graph.get_tensor_by_name('input:0') output_tensor = graph.get_tensor_by_name('Identity:0') config = tf.compat.v1.ConfigProto() config.intra_op_parallelism_threads = self.args.num_intra_threads config.inter_op_parallelism_threads = self.args.num_inter_threads config.graph_options.rewrite_options.auto_mixed_precision_mkl = rewriter_config_pb2.RewriterConfig.ON sess = tf.compat.v1.Session(graph=graph, config=config) if (self.args.accuracy_only): print("Inference with real data") preprocessed_data_dir = "build/preprocessed_data" with open( os.path.join(preprocessed_data_dir, "preprocessed_files.pkl"), "rb") as f: preprocessed_files = pickle.load(f) dictionaries = [] for preprocessed_file in preprocessed_files: with open( os.path.join(preprocessed_data_dir, preprocessed_file + ".pkl"), "rb") as f: dct = pickle.load(f)[1] dictionaries.append(dct) count = len(preprocessed_files) predictions = [None] * count validation_indices = list(range(0, count)) print("Found {:d} preprocessed files".format(count)) loaded_files = {} batch_size = self.args.batch_size # Get the number of steps based on batch size steps = count #math.ceil(count/batch_size) for i in range(steps): print("Iteration {} ...".format(i)) test_data_index = validation_indices[ i] #validation_indices[i * batch_size:(i + 1) * batch_size] file_name = preprocessed_files[test_data_index] with open( os.path.join(preprocessed_data_dir, "{:}.pkl".format(file_name)), "rb") as f: data = pickle.load(f)[0] predictions[i] = sess.run( output_tensor, feed_dict={input_tensor: data[np.newaxis, ...]})[0].astype(np.float32) output_folder = "build/postprocessed_data" output_files = preprocessed_files # Post Process postprocess_output(predictions, dictionaries, validation_indices, output_folder, output_files) ground_truths = "build/raw_data/nnUNet_raw_data/Task043_BraTS2019/labelsTr" # Run evaluation print("Running evaluation...") evaluate_regions(output_folder, ground_truths, get_brats_regions()) # Load evaluation summary print("Loading evaluation summary...") with open(os.path.join(output_folder, "summary.csv")) as f: for line in f: words = line.split(",") if words[0] == "mean": whole = float(words[1]) core = float(words[2]) enhancing = float(words[3]) mean = (whole + core + enhancing) / 3 print( "Accuracy: mean = {:.5f}, whole tumor = {:.4f}, tumor core = {:.4f}, enhancing tumor = {:.4f}" .format(mean, whole, core, enhancing)) break print("Done!")