def __init__(self, plans_file, fold, output_folder=None, dataset_directory=None, batch_dice=True, stage=None,
unpack_data=True, deterministic=True, fp16=False):
super().__init__(plans_file, fold, output_folder, dataset_directory, batch_dice, stage, unpack_data,
deterministic, fp16)
self.regions = get_brats_regions()
self.regions_class_order = (1, 2, 3)
self.loss = DC_and_BCE_loss({}, {'batch_dice': False, 'do_bg': True, 'smooth': 0})
def main():
args = get_args()
log_file = args.log_file
preprocessed_data_dir = args.preprocessed_data_dir
output_folder = args.postprocessed_data_dir
ground_truths = args.label_data_dir
output_dtype = dtype_map[args.output_dtype]
num_threads_nifti_save = args.num_threads_nifti_save
all_in_gpu = "None"
force_separate_z = None
interp_order = 3
interp_order_z = 0
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Load necessary metadata.
print("Loading necessary metadata...")
with open(os.path.join(preprocessed_data_dir, "preprocessed_files.pkl"),
"rb") as f:
preprocessed_files = pickle.load(f)
dictionaries = []
for preprocessed_file in preprocessed_files:
with open(
os.path.join(preprocessed_data_dir,
preprocessed_file + ".pkl"), "rb") as f:
dct = pickle.load(f)[1]
dictionaries.append(dct)
# Load predictions from loadgen accuracy log.
print("Loading loadgen accuracy log...")
predictions = load_loadgen_log(log_file, output_dtype, dictionaries)
# Save predictions
# This runs in multiprocess
print("Running postprocessing with multiple threads...")
save_predictions_MLPerf(predictions, output_folder, preprocessed_files,
dictionaries, num_threads_nifti_save, all_in_gpu,
force_separate_z, interp_order, interp_order_z)
# Run evaluation
print("Running evaluation...")
evaluate_regions(output_folder, ground_truths, get_brats_regions())
# Load evaluation summary
print("Loading evaluation summary...")
with open(os.path.join(output_folder, "summary.csv")) as f:
for line in f:
words = line.split(",")
if words[0] == "mean":
whole = float(words[1])
core = float(words[2])
enhancing = float(words[3])
mean = (whole + core + enhancing) / 3
print(
"Accuracy: mean = {:.5f}, whole tumor = {:.4f}, tumor core = {:.4f}, enhancing tumor = {:.4f}"
.format(mean, whole, core, enhancing))
break
print("Done!")
def __init__(self, plans_file, fold, output_folder=None, dataset_directory=None, batch_dice=True, stage=None,
unpack_data=True, deterministic=True, fp16=False):
super().__init__(plans_file, fold, output_folder, dataset_directory, batch_dice, stage, unpack_data,
deterministic, fp16)
self.validate_every = 1
self.evaluation_regions = get_brats_regions()
self.num_val_batches_per_epoch = 0 # we dont need this because this does not evaluate on full images
def __init__(self, plans_file, fold, local_rank, output_folder=None, dataset_directory=None, batch_dice=True,
stage=None,
unpack_data=True, deterministic=True, distribute_batch_size=False, fp16=False):
super().__init__(plans_file, fold, local_rank, output_folder, dataset_directory, batch_dice, stage, unpack_data,
deterministic, distribute_batch_size, fp16)
self.regions = get_brats_regions()
self.regions_class_order = (1, 2, 3)
self.loss = None
self.ce_loss = nn.BCEWithLogitsLoss()
def collect_and_prepare(base_dir, num_processes = 12, clean=False):
"""
collect all cv_niftis, compute brats metrics, compute enh tumor thresholds and summarize in csv
:param base_dir:
:return:
"""
out = join(base_dir, 'cv_results')
out_pp = join(base_dir, 'cv_results_pp')
experiments = subfolders(base_dir, join=False, prefix='nnUNetTrainer')
regions = get_brats_regions()
gt_dir = join(base_dir, 'gt_niftis')
replace_with = 2
failed = []
successful = []
for e in experiments:
print(e)
try:
o = join(out, e)
o_p = join(out_pp, e)
maybe_mkdir_p(o)
maybe_mkdir_p(o_p)
collect_cv_niftis(join(base_dir, e), o)
if clean or not isfile(join(o, 'summary.csv')):
evaluate_regions(o, gt_dir, regions, num_processes)
if clean or not isfile(join(o_p, 'threshold.pkl')):
determine_brats_postprocessing(o, gt_dir, o_p, num_processes, thresholds=list(np.arange(0, 760, 10)), replace_with=replace_with)
if clean or not isfile(join(o_p, 'summary.csv')):
evaluate_regions(o_p, gt_dir, regions, num_processes)
successful.append(e)
except Exception as ex:
print("\nERROR\n", e, ex, "\n")
failed.append(e)
# we are interested in the mean (nan is 1) column
with open(join(base_dir, 'cv_summary.csv'), 'w') as f:
f.write('name,whole,core,enh,mean\n')
for e in successful:
expected_nopp = join(out, e, 'summary.csv')
expected_pp = join(out, out_pp, e, 'summary.csv')
if isfile(expected_nopp):
res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_noPP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
if isfile(expected_pp):
res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_PP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
# this just crawls the folders and evaluates what it finds
with open(join(base_dir, 'cv_summary2.csv'), 'w') as f:
for folder in ['cv_results', 'cv_results_pp']:
for ex in subdirs(join(base_dir, folder), join=False):
print(folder, ex)
expected = join(base_dir, folder, ex, 'summary.csv')
if clean or not isfile(expected):
evaluate_regions(join(base_dir, folder, ex), gt_dir, regions, num_processes)
if isfile(expected):
res = np.loadtxt(expected, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write('%s__%s,' % (folder, ex))
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
f.write('name,whole,core,enh,mean\n')
for e in successful:
expected_nopp = join(out, e, 'summary.csv')
expected_pp = join(out, out_pp, e, 'summary.csv')
if isfile(expected_nopp):
res = np.loadtxt(expected_nopp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_noPP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
if isfile(expected_pp):
res = np.loadtxt(expected_pp, dtype=str, skiprows=0, delimiter=',')[-2]
as_numeric = [float(i) for i in res[1:]]
f.write(e + '_PP,')
f.write("%0.4f," % as_numeric[0])
f.write("%0.4f," % as_numeric[1])
f.write("%0.4f," % as_numeric[2])
f.write("%0.4f\n" % np.mean(as_numeric))
# apply threshold to val set
expected_num_cases = 125
missing_valset = []
has_val_pred = []
for e in successful:
if isdir(join(base_dir, 'predVal', e)):
currdir = join(base_dir, 'predVal', e)
files = subfiles(currdir, suffix='.nii.gz', join=False)
if len(files) != expected_num_cases:
print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases))
continue
output_folder = join(base_dir, 'predVal_PP', e)
maybe_mkdir_p(output_folder)
threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2]
if threshold > 1000: threshold = 750 # don't make it too big!
apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes)
has_val_pred.append(e)
else:
print(e, 'has no valset predictions')
missing_valset.append(e)
# 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment
e = 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5'
currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold')
output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold')
maybe_mkdir_p(output_folder)
threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2]
if threshold > 1000: threshold = 750 # don't make it too big!
apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes)
# 'nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold' needs special treatment
e = 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5'
currdir = join(base_dir, 'predVal', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold')
output_folder = join(base_dir, 'predVal_PP', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold')
maybe_mkdir_p(output_folder)
threshold = load_pickle(join(out_pp, e, 'threshold.pkl'))[2]
if threshold > 1000: threshold = 750 # don't make it too big!
apply_threshold_to_folder(currdir, output_folder, threshold, replace_with, num_processes)
# convert val set to brats labels for submission
output_converted = join(base_dir, 'converted_valSet')
for source in ['predVal', 'predVal_PP']:
for e in has_val_pred + ['nnUNetTrainerV2BraTSRegions_DA3_BN__nnUNetPlansv2.1_bs5_15fold', 'nnUNetTrainerV2BraTSRegions_DA4_BN__nnUNetPlansv2.1_bs5_15fold']:
expected_source_folder = join(base_dir, source, e)
if not isdir(expected_source_folder):
print(e, 'has no', source)
raise RuntimeError()
files = subfiles(expected_source_folder, suffix='.nii.gz', join=False)
if len(files) != expected_num_cases:
print(e, 'prediction not done, found %d files, expected %s' % (len(files), expected_num_cases))
continue
target_folder = join(output_converted, source, e)
maybe_mkdir_p(target_folder)
convert_labels_back_to_BraTS_2018_2019_convention(expected_source_folder, target_folder)
summarize_validation_set_predictions(output_converted)
def run(self):
print("Run inference for accuracy")
setup(self.args.data_location, self.args.input_graph)
graph = tf.Graph()
with graph.as_default():
graph_def = tf.compat.v1.GraphDef()
with open(self.args.input_graph, "rb") as f:
graph_def.ParseFromString(f.read())
output_graph = optimize_for_inference(
graph_def, [INPUTS], [OUTPUTS],
dtypes.float32.as_datatype_enum, False)
tf.import_graph_def(output_graph, name="")
input_tensor = graph.get_tensor_by_name('input:0')
output_tensor = graph.get_tensor_by_name('Identity:0')
config = tf.compat.v1.ConfigProto()
config.intra_op_parallelism_threads = self.args.num_intra_threads
config.inter_op_parallelism_threads = self.args.num_inter_threads
config.graph_options.rewrite_options.auto_mixed_precision_mkl = rewriter_config_pb2.RewriterConfig.ON
sess = tf.compat.v1.Session(graph=graph, config=config)
if (self.args.accuracy_only):
print("Inference with real data")
preprocessed_data_dir = "build/preprocessed_data"
with open(
os.path.join(preprocessed_data_dir,
"preprocessed_files.pkl"), "rb") as f:
preprocessed_files = pickle.load(f)
dictionaries = []
for preprocessed_file in preprocessed_files:
with open(
os.path.join(preprocessed_data_dir,
preprocessed_file + ".pkl"), "rb") as f:
dct = pickle.load(f)[1]
dictionaries.append(dct)
count = len(preprocessed_files)
predictions = [None] * count
validation_indices = list(range(0, count))
print("Found {:d} preprocessed files".format(count))
loaded_files = {}
batch_size = self.args.batch_size
# Get the number of steps based on batch size
steps = count #math.ceil(count/batch_size)
for i in range(steps):
print("Iteration {} ...".format(i))
test_data_index = validation_indices[
i] #validation_indices[i * batch_size:(i + 1) * batch_size]
file_name = preprocessed_files[test_data_index]
with open(
os.path.join(preprocessed_data_dir,
"{:}.pkl".format(file_name)), "rb") as f:
data = pickle.load(f)[0]
predictions[i] = sess.run(
output_tensor,
feed_dict={input_tensor: data[np.newaxis,
...]})[0].astype(np.float32)
output_folder = "build/postprocessed_data"
output_files = preprocessed_files
# Post Process
postprocess_output(predictions, dictionaries, validation_indices,
output_folder, output_files)
ground_truths = "build/raw_data/nnUNet_raw_data/Task043_BraTS2019/labelsTr"
# Run evaluation
print("Running evaluation...")
evaluate_regions(output_folder, ground_truths, get_brats_regions())
# Load evaluation summary
print("Loading evaluation summary...")
with open(os.path.join(output_folder, "summary.csv")) as f:
for line in f:
words = line.split(",")
if words[0] == "mean":
whole = float(words[1])
core = float(words[2])
enhancing = float(words[3])
mean = (whole + core + enhancing) / 3
print(
"Accuracy: mean = {:.5f}, whole tumor = {:.4f}, tumor core = {:.4f}, enhancing tumor = {:.4f}"
.format(mean, whole, core, enhancing))
break
print("Done!")
