def normalize_data_storage_val(data_storage,
offset=0.1,
mul_factor=100,
save_file='../data/mean_std.pkl'):
print('normalize validation data storage...')
if not os.path.exists(save_file):
print_red('There\'s no mean_std.pkl file.')
return
with open(save_file, 'rb') as f:
mean_std_values = pickle.load(f)
for modality_storage in data_storage:
n_subs = modality_storage.shape[0]
mean = mean_std_values[modality_storage.name + '_mean']
std = mean_std_values[modality_storage.name + '_std']
# pdb.set_trace()
for i in tqdm(range(n_subs)):
brain_index = np.nonzero(modality_storage[i])
temp_img = np.copy(modality_storage[i])
temp_img[brain_index] = (minmax_normalize(
(modality_storage[i][brain_index] - mean) / std) +
offset) * mul_factor
modality_storage[i] = temp_img
print('normalization FINISHED')
return
def add_data_to_storage(data_storage, brain_width_storage, subject_data,
brain_width, modality_names):
# pdb.set_trace()
for i in range(len(modality_names)):
if data_storage[i].name != modality_names[i]:
print_red('modality_storage.name != modality_name')
return 1
data_storage[i].append(
np.asarray(subject_data[i])[np.newaxis][np.newaxis])
brain_width_storage.append(
np.asarray(brain_width, dtype=np.uint8)[np.newaxis])
return 0
def write_image_data_to_file(image_files,
storage_list,
image_shape,
modality_names,
truth_dtype=np.uint8,
trivial_check=True):
'''
trivial_check: to see if all images share the same affine info and pad_width, the incompliance file names
would be printed in red lines.
Also to check the order of modalities when added to the .h5
'''
# pdb.set_trace()
affine_0 = None
save_affine = True
print('write_image_data_to_file...')
for set_of_files in tqdm(image_files):
if trivial_check:
if not [
os.path.basename(img_file).split('.')[0]
for img_file in set_of_files
] == modality_names + ['truth']:
print('wrong order of modalities')
print_red(image_nii_path)
subject_data = []
brain_widths = []
for i, image_nii_path in enumerate(set_of_files):
img = nib.load(image_nii_path)
affine = img.affine
if affine_0 is None:
affine_0 = affine
if trivial_check:
if np.sum(affine_0 - affine):
print('affine incompliance:')
print_red(image_nii_path)
save_affine = False
img_npy = img.get_data()
subject_data.append(img_npy)
if i < len(set_of_files
) - 1: # we don't calculate brain_width for truth
brain_widths.append(cal_outline(img_npy))
else:
tumor_width = cal_outline(img_npy)
start_edge = np.min(brain_widths, axis=0)[0]
end_edge = np.max(brain_widths, axis=0)[1]
brain_width = np.vstack((start_edge, end_edge))
if add_data_to_storage(storage_list,
subject_data,
brain_width,
tumor_width,
truth_dtype,
modality_names=modality_names):
print_red('modality_storage.name != modality_name')
print_red(set_of_files)
print('write_image_data_to_file...FINISHED')
if save_affine:
np.save('affine', affine_0)
return
def write_image_data_to_file(image_files,
data_storage,
brain_width_storage,
image_shape,
modality_names,
trivial_check=True):
'''
trivial_check: to see if all images share the same affine info and pad_width, the incompliance file names
would be printed in red lines.
Also to check the order of modalities when added to the .h5
'''
# pdb.set_trace()
affine_0 = np.load('affine.npy')
# temp = 0
print('write_image_data_to_file...')
for set_of_files in tqdm(image_files):
if trivial_check:
if not [
os.path.basename(img_file).split('.')[0]
for img_file in set_of_files
] == modality_names:
print('wrong order of modalities')
print_red(image_nii_path)
subject_data = []
brain_widths = []
for i, image_nii_path in enumerate(set_of_files):
img = nib.load(image_nii_path)
affine = img.affine
if trivial_check:
if np.sum(affine_0 - affine):
print('affine incompliance:')
print_red(image_nii_path)
img_npy = img.get_data()
subject_data.append(img_npy)
brain_widths.append(cal_outline(img_npy))
start_edge = np.min(brain_widths, axis=0)[0]
end_edge = np.max(brain_widths, axis=0)[1]
brain_width = np.vstack((start_edge, end_edge))
if add_data_to_storage(data_storage,
brain_width_storage,
subject_data,
brain_width,
modality_names=modality_names):
print_red('modality_storage.name != modality_name')
print_red(set_of_files)
print('write_image_data_to_file...FINISHED')
return data_storage
def add_data_to_storage(storage_list, subject_data, brain_width, tumor_width,
truth_dtype, modality_names):
# pdb.set_trace()
modality_storage_list, truth_storage, brain_width_storage, tumor_width_storage = storage_list
for i in range(len(modality_names)):
if modality_storage_list[i].name != modality_names[i]:
print_red('modality_storage.name != modality_name')
return 1
modality_storage_list[i].append(
np.asarray(subject_data[i])[np.newaxis][np.newaxis])
if truth_storage.name != 'truth':
print_red('truth_storage.name != truth')
return 1
truth_storage.append(
np.asarray(subject_data[-1],
dtype=truth_dtype)[np.newaxis][np.newaxis])
brain_width_storage.append(
np.asarray(brain_width, dtype=truth_dtype)[np.newaxis])
tumor_width_storage.append(
np.asarray(tumor_width, dtype=truth_dtype)[np.newaxis])
return 0
def prediction_to_image(prediction,
affine,
brain_mask,
threshold=0.5,
labels=None,
output_dir='',
overlap_label=False):
'''
for multi categories classification please refer to Isensee's repository.
'''
# pdb.set_trace()
pdb_set = False
if prediction.shape[1] == 3:
if overlap_label:
data = get_prediction_labels_overlap(prediction,
threshold=threshold)
else:
data = get_prediction_labels(prediction,
threshold=threshold,
labels=labels)
else:
raise RuntimeError("Invalid prediction array shape: {0}".format(
prediction.shape))
masked_output = data * brain_mask
if np.sum(masked_output - data):
if pdb_set:
print_red('changed after mask')
print_red(output_dir)
print_red(np.array(np.where(masked_output != data)).shape[1])
nib.Nifti1Image(data, affine).to_filename(
os.path.join(output_dir, "prediction_before_mask.nii.gz"))
return nib.Nifti1Image(masked_output, affine)
def compute_patch_indices_for_prediction(image_shape,
patch_size,
center_patch=True):
# pdb.set_trace()
pdb_set = False
if pdb_set:
if np.any(
np.array(2 * np.array(patch_size) -
np.array(image_shape)) <= 0):
print_red('error patch: too large')
if np.any(np.array(image_shape - patch_size) <= 0):
print_red('error patch: too small')
start_2 = np.asarray(image_shape - patch_size)
start_2[start_2 < 0] = 0
patches = np.array([[0, 0, 0], [start_2[0], 0, 0], [0, start_2[1], 0],
[0, 0, start_2[2]], [start_2[0], start_2[1], 0],
[start_2[0], start_2[1], start_2[2]],
[start_2[0], 0, start_2[2]],
[0, start_2[1], start_2[2]]])
if center_patch:
patches = np.vstack((patches, (image_shape - patch_size) // 2))
return patches
def convert_brats_folder(in_folder, out_folder, truth_name='seg', no_bias_correction_modalities=None, bias_correct=True):
# pdb.set_trace()
for name in config["all_modalities"]:
try:
image_file = get_image(in_folder, name)
except RuntimeError as error:
if name == 't1ce':
print_red(in_fold)
image_file = get_image(in_folder, 't1Gd')
truth_name = "GlistrBoost_ManuallyCorrected"
else:
raise error
out_file = os.path.abspath(os.path.join(out_folder, name + ".nii.gz"))
if bias_correct:
perform_bias_correction = no_bias_correction_modalities and name not in no_bias_correction_modalities
normalize_image(image_file, out_file, bias_correction=perform_bias_correction)
else:
if not os.path.exists(out_file):
shutil.copy(image_file, out_file)
# copy the truth file only for training dataset
if in_folder.split('/')[-2] == 'val':
return
try:
truth_file = get_image(in_folder, truth_name)
except RuntimeError:
truth_file = get_image(in_folder, truth_name.split("_")[0])
out_file = os.path.abspath(os.path.join(out_folder, "truth.nii.gz"))
if not os.path.exists(out_file):
shutil.copy(truth_file, out_file)
check_origin(out_file, get_image(in_folder, config["all_modalities"][0]))
return
def get_xy(csv_file, save_name, gtr_only=True, for_val=True):
# pdb.set_trace()
if not for_val:
if os.path.exists(save_name):
print(save_name,'exists already.')
return
# df = pd.read_csv('os_data/gtr_only.csv' if gtr_only else 'os_data/all.csv')
df = pd.read_csv(csv_file)
X = []
Y = []
for i in tqdm(range(len(df))):
sub_id = df.iloc[i]['BraTS19ID']
# print(sub_id)
if for_val:
img,truth = get_truth_val(sub_id)
else:
img,truth = get_truth(sub_id)
size_brain = np.sum(np.nonzero(img))
size_et = np.sum(truth==4)
size_tc = size_et + np.sum(truth==1)
size_wt = size_tc + np.sum(truth==2)
prop_et = size_et/size_brain
prop_tc = size_tc/size_brain
prop_wt = size_wt/size_brain
prop_et_tc = size_et/size_tc
prop_et_wt = size_et/size_wt
prop_tc_wt = size_tc/size_wt
bound_et = np.gradient((truth==4).astype(np.float32))
length_et = get_length(bound_et)
bound_tc = np.gradient(np.logical_or(truth==1,truth==4).astype(np.float32))
length_tc = get_length(bound_tc)
bound_wt = np.gradient(np.logical_or(np.logical_or(truth==1,truth==4),truth==2).astype(np.float32))
length_wt = get_length(bound_wt)
age = df.iloc[i]['Age']
features = [size_brain,size_et,size_tc,size_wt,
prop_et,prop_tc,prop_wt,
prop_et_tc,prop_et_wt,prop_tc_wt,
length_et,length_tc,length_wt,age]
if not gtr_only:
status = df.iloc[i]['ResectionStatus']
if status == '-1':
features.append(0)
features.append(0)
elif status == 'GTR':
features.append(1)
features.append(0)
elif status == 'STR':
features.append(0)
features.append(1)
else:
pdb.set_trace()
print_red('wrong ResectionStatus'+sub_id)
X.append(features)
if not for_val:
Y.append(int(df.iloc[i]['Survival']))
np.savez(save_name,X,Y)
return