def get_max_values(patients, data_dir, feature_file_names):
print('Getting max values for normalization...')
maxs_dict = {}
for i, patient in enumerate(patients):
if i % 100 == 0:
print(str(i), '/', len(patients))
features = [helper.load_nifti_mat_from_file(
os.path.join(data_dir, patient + feature_file_name), printout=False) for feature_file_name
in feature_file_names]
num_channels = len(features)
for ch in range(num_channels):
if ch not in maxs_dict.keys():
maxs_dict[ch] = []
maxs_dict[ch].append(np.max(features[ch]))
max_list = [max(maxs_list) for ch, maxs_list in maxs_dict.items()]
print('Max values per channel:', max_list)
return max_list
def main(img_filepath, graph_json_filepath, model_filepath,
annotation_results_filepath, washed_annotation_results_filepath):
img = helper.load_nifti_mat_from_file(img_filepath)
graph_json = helper.load_json(graph_json_filepath)
volume_dimensions = img.shape
background_value = bravenet_config.ARTER_DICTS[
bravenet_config.H_LEVEL]['background']
num_classes = bravenet_config.NUM_CLASSES
patch_size_x = bravenet_config.PATCH_SIZE_X
patch_size_y = bravenet_config.PATCH_SIZE_Y
patch_size_z = bravenet_config.PATCH_SIZE_Z
max_values_for_normalization = bravenet_config.MAX_FEATURE_VALUES
# create volume with vessel skeleton with radius values
print('Creating skeleton with radius volume...')
skeleton_radius = helper.skeleton_radius_from_json_graph(
graph_json=graph_json,
volume_dimensions=volume_dimensions,
background_value=background_value,
radius_dtype='float32')
# create volume with vessel skeleton with vessel segments coded in y-axis depth
print('Creating skeleton with vessel segments volume...')
skeleton_segments, _, _ = helper.skeleton_segments_from_json_graph(
graph_json=graph_json,
volume_dimensions=volume_dimensions,
background_value=background_value,
segments_dtype='int32',
axis='y')
feature_volumes = [img, skeleton_radius]
model = load_model(model_filepath, compile=False)
predict(feature_volumes=feature_volumes,
model=model,
num_classes=num_classes,
patch_size_x=patch_size_x,
patch_size_y=patch_size_y,
patch_size_z=patch_size_z,
annotation_save_path=annotation_results_filepath,
max_values_for_normalization=max_values_for_normalization,
skeleton_segments=skeleton_segments,
washed_annotation_save_path=washed_annotation_results_filepath)
return True
def evaluate_and_save_to_csv(version, n_estimators,
n_patients_train, data_dir, dataset, models_dir, predictions_dir,
patients, result_file, result_file_per_patient, label_load_name,
washed):
"""
:param version: The Unet version name. String.
:param n_estimators: Number of trees in forest. Positive integer.
:param n_patients_train: The number of patient used for training. Positive integer.
:param data_dir: The path to data dirs. String.
:param dataset: The dataset. E.g. train/val/set
:param models_dir: Directory where trained models are saved. String.
:param predictions_dir: Directory where predictions are saved. String.
:param patients: The names of patients in the given dataset. List of strings.
:param result_file: The path to the csv file where to store the results of the performance calculation calculated
as average over all patient in dataset. String.
:param result_file_per_patient: The path to the csv file where to store the results of the performance calculation
per patient. String.
:param label_load_name: The file name of the ground-truth label. String.
:param washed: True for evaluation of predictions washed with vessel segments.
:return:
"""
print('model version', version)
print('number of estimators', n_estimators)
print('label load name', label_load_name)
print('washed', washed)
starttime_row = helper.start_time_measuring(what_to_measure='model evaluation')
# create the name of current run
run_name = rf_config.get_run_name(version=version, n_estimators=n_estimators, n_patients_train=n_patients_train)
# -----------------------------------------------------------
# TRAINING PARAMS
# -----------------------------------------------------------
try:
train_metadata_filepath = rf_config.get_train_metadata_filepath(models_dir, run_name)
with open(train_metadata_filepath, 'rb') as handle:
train_metadata = pickle.load(handle)
print('Train params:')
print(train_metadata['params'])
except FileNotFoundError:
print('Unexpected error by reading train metadata:', sys.exc_info()[0])
# -----------------------------------------------------------
# DATASET RESULTS (TRAIN / VAL / TEST)
# -----------------------------------------------------------
# initialize empty list for saving measures for each patient
acc_list = []
f1_macro_list = []
f1_macro_wo_0_list = []
f1_macro_wo_0_and_1_list = []
avg_acc_list = []
avg_acc_wo_0_list = []
avg_acc_wo_0_and_1_list = []
f1_bin_list = []
f1_class_dict = {} # for saving f1 for each class
for cls in range(rf_config.NUM_CLASSES):
f1_class_dict[cls] = []
# calculate the performance per patient
for patient in patients:
# load patient ground truth and prediction
print('-----------------------------------------------------------------')
print(patient)
print('> Loading label...')
label = helper.load_nifti_mat_from_file(os.path.join(data_dir, patient + label_load_name))
print('> Loading prediction...')
if washed:
prediction_path = rf_config.get_washed_annotation_filepath(predictions_dir, run_name, patient, dataset)
else:
prediction_path = rf_config.get_annotation_filepath(predictions_dir, run_name, patient, dataset)
if not os.path.exists(prediction_path) and prediction_path.endswith('.gz'):
prediction_path = prediction_path[:-3]
prediction = helper.load_nifti_mat_from_file(prediction_path)
# flatten the 3d volumes to 1d vector, necessary for performance calculation with sklearn functions
label_f = label.flatten()
prediction_f = prediction.flatten().astype(np.int16)
label_f_bin = np.clip(label_f, 0, 1)
prediction_f_bin = np.clip(prediction_f, 0, 1)
print('Computing performance measures...')
# classification accuracy
acc = accuracy_score(label_f, prediction_f)
# dice score for multiclass classification
f1_per_classes = f1_score(label_f, prediction_f,
average=None) # f1 for each present class label in ground truth and prediction
f1_macro = np.mean(f1_per_classes) # averaged f1 over all present class labels
f1_macro_wo_0 = np.mean(
f1_per_classes[1:]) # averaged f1 over all present class labels except background class label
f1_macro_wo_0_and_1 = np.mean(
f1_per_classes[
2:]) # averaged f1 over all present class labels except background and not-annotated class label
# average class accuracy for multiclass classification
avg_acc, avg_acc_per_classes = balanced_accuracy_score(label_f, prediction_f)
avg_acc_wo_0 = np.mean(avg_acc_per_classes[1:])
avg_acc_wo_0_and_1 = np.mean(avg_acc_per_classes[2:])
# dice for binary prediction -> labels converted to 1 for annotated vessels and 0 for background
f1_bin = f1_score(label_f_bin, prediction_f_bin)
patient_f1_class_list = ['-'] * rf_config.NUM_CLASSES # for saving f1 for each class
# print out to console
print('acc:', acc)
print('f1 all classes:', f1_macro)
print('f1 without background class:', f1_macro_wo_0)
print('f1 without background and not-annotated class:', f1_macro_wo_0_and_1)
print('avg_acc all classes:', avg_acc)
print('avg_acc without background class:', avg_acc_wo_0)
print('avg_acc without background and not-annotated class:', avg_acc_wo_0_and_1)
print('f1 binary predictions:', f1_bin)
print('f1 per classes', f1_per_classes, 'size', len(f1_per_classes))
# find what labels are in ground-truth and what labels were predicted
unique_labels = np.unique(label_f)
unique_prediction = np.unique(prediction_f)
print('label unique', unique_labels, 'size', len(unique_labels))
print('predicted annotation unique', unique_prediction, 'size', len(unique_prediction))
# save results for patient to the lists
acc_list.append(acc)
f1_macro_list.append(f1_macro)
f1_macro_wo_0_list.append(f1_macro_wo_0)
f1_macro_wo_0_and_1_list.append(f1_macro_wo_0_and_1)
avg_acc_list.append(avg_acc)
avg_acc_wo_0_list.append(avg_acc_wo_0)
avg_acc_wo_0_and_1_list.append(avg_acc_wo_0_and_1)
f1_bin_list.append(f1_bin)
all_classes = np.concatenate((unique_labels, unique_prediction))
unique_classes = np.unique(all_classes)
for i, cls in enumerate(unique_classes):
f1_class_dict[cls].append(f1_per_classes[i])
patient_f1_class_list[cls] = f1_per_classes[i]
# create row for saving to csv file with details for each patient and write to the csv file
row_per_patient = [n_estimators, patient,
acc,
f1_macro,
f1_macro_wo_0,
f1_macro_wo_0_and_1,
avg_acc,
avg_acc_wo_0,
avg_acc_wo_0_and_1,
f1_bin] + patient_f1_class_list
helper.write_to_csv(result_file_per_patient, [row_per_patient])
# calculate patient mean and std for each class
f1_class_list_mean = []
f1_class_list_std = []
for cls, class_f1_list in f1_class_dict.items():
f1_class_list_mean.append(np.mean(class_f1_list))
f1_class_list_std.append(np.std(class_f1_list))
# create row for saving to csv file with averages over whole set and write to the csv file
row_avg = [n_estimators, len(patients),
'AVG',
np.mean(acc_list),
np.mean(f1_macro_list),
np.mean(f1_macro_wo_0_list),
np.mean(f1_macro_wo_0_and_1_list),
np.mean(avg_acc_list),
np.mean(avg_acc_wo_0_list),
np.mean(avg_acc_wo_0_and_1_list),
np.mean(f1_bin_list)] + f1_class_list_mean
row_std = [n_estimators, len(patients),
'STD',
np.std(acc_list),
np.std(f1_macro_list),
np.std(f1_macro_wo_0_list),
np.std(f1_macro_wo_0_and_1_list),
np.std(avg_acc_list),
np.std(avg_acc_wo_0_list),
np.std(avg_acc_wo_0_and_1_list),
np.std(f1_bin_list)] + f1_class_list_std
print('AVG:', row_avg)
print('STD:', row_std)
helper.write_to_csv(result_file, [row_avg, row_std])
# print out how long did the calculations take
helper.end_time_measuring(starttime_row, what_to_measure='model evaluation')
def predict_and_save(version,
n_estimators,
n_patients_train,
patient,
data_dir,
dataset,
feature_filenames,
models_dir,
predictions_dir,
run_name=None,
model=None,
train_metadata=None):
"""
:param version: The name describes the SVM version. String.
:param n_estimators: Number of trees in forest. Positive integer.
:param n_patients_train: The number of patient used for training. Positive integer.
:param patient: The patient name. String.
:param data_dir: The path to data dirs. String.
:param dataset: The dataset. E.g. train/val/set
:param feature_filenames: List of file names of the feature inputs to the network. List of strings.
:param models_dir: Directory where trained models are saved. String.
:param predictions_dir: Directory where predictions are saved. String.
"""
print('model version', version)
print('number of estimators', n_estimators)
print('patient:', patient)
print('feature file names', feature_filenames)
# create the name of current run
if not run_name:
run_name = rf_config.get_run_name(version=version,
n_estimators=n_estimators,
n_patients_train=n_patients_train)
# -----------------------------------------------------------
# LOADING MODEL, RESULTS AND WHOLE BRAIN MATRICES
# -----------------------------------------------------------
try:
if not model:
model_filepath = rf_config.get_model_filepath(models_dir, run_name)
print('Model path:', model_filepath)
model = pickle.load(open(model_filepath, 'rb'))
# -----------------------------------------------------------
# TRAINING PARAMS AND FEATURES
# -----------------------------------------------------------
if not train_metadata:
try:
train_metadata_filepath = rf_config.get_train_metadata_filepath(
models_dir, run_name)
with open(train_metadata_filepath, 'rb') as handle:
train_metadata = pickle.load(handle)
print('Train params:')
print(train_metadata['params'])
except FileNotFoundError:
print('Unexpected error by reading train metadata:',
sys.exc_info()[0])
print('> Loading features...')
loaded_feature_list = [
helper.load_nifti_mat_from_file(os.path.join(
data_dir, patient + f),
printout=True)
for f in feature_filenames
]
# -----------------------------------------------------------
# PREDICTION
# -----------------------------------------------------------
print('Predicting...')
starttime_predict = helper.start_time_measuring(
what_to_measure='patient prediction')
# find all vessel voxel indices
vessel_inds = np.where(loaded_feature_list[0] > 0)
# extract features and labels per voxel and predict
prediction_3D = np.zeros(loaded_feature_list[0].shape, dtype='uint8')
for voxel in range(len(vessel_inds[0])):
x, y, z = vessel_inds[0][voxel], vessel_inds[1][
voxel], vessel_inds[2][voxel] # vessel voxel coordinates
features = [[x, y, z]]
for i, feature in enumerate(loaded_feature_list):
features[0].append(feature[x, y, z])
# scale data
scaler = train_metadata['params']['scaler']
features = scaler.transform(features)
# predict
prediction = model.predict(features)
# rebuild the 3D volume
prediction_3D[x, y, z] = prediction
# how long does the prediction take for a patient
helper.end_time_measuring(starttime_predict,
what_to_measure='patient prediction')
# -----------------------------------------------------------
# SAVE AS NIFTI
# -----------------------------------------------------------
print(predictions_dir)
save_path = rf_config.get_annotation_filepath(predictions_dir,
run_name, patient,
dataset)
helper.create_and_save_nifti(prediction_3D, save_path)
except FileNotFoundError:
print('Unexpected error by reading model:', sys.exc_info()[0])
def main(label_filepath, prediction_filepath, mode, segments_filepath=''):
print('> Loading label...')
label = helper.load_nifti_mat_from_file(label_filepath)
print('> Loading prediction...')
prediction = helper.load_nifti_mat_from_file(prediction_filepath)
# If mode is segment-wise, prepare the segment-wise data points.
if mode == 'segment-wise':
print('> Loading segments...')
skel_seg = helper.load_nifti_mat_from_file(segments_filepath)
# Get label and prediction segment datapoints.
label_segment_datapoints = []
prediction_segment_datapoints = []
unique_segments = np.unique(skel_seg)
for seg in unique_segments:
segment_inds = np.where(skel_seg == seg)
# Get label segment data points.
label_segment_classes = label[segment_inds]
label_segment_unique_classes, label_segment_classes_counts = np.unique(
label_segment_classes, return_counts=True)
if len(label_segment_unique_classes) > 1:
print(
colored(('Label - Segment id:', seg, ' Unique classes:',
label_segment_unique_classes, ' Classes counts: ',
label_segment_classes_counts), 'red'))
label_segment_class = label_segment_unique_classes[np.argmax(
label_segment_classes_counts)]
else:
label_segment_class = label_segment_unique_classes[0]
label_segment_datapoints.append(label_segment_class)
# Get prediction segment data points.
prediction_segment_classes = prediction[segment_inds]
prediction_segment_unique_classes, prediction_segment_classes_counts = np.unique(
prediction_segment_classes, return_counts=True)
if len(prediction_segment_unique_classes) > 1:
print(
colored(
('Prediction - Segment id:', seg, ' Unique classes:',
prediction_segment_unique_classes,
' Classes counts: ',
prediction_segment_classes_counts), 'red'))
prediction_segment_class = prediction_segment_unique_classes[
np.argmax(prediction_segment_classes_counts)]
else:
prediction_segment_class = prediction_segment_unique_classes[0]
prediction_segment_datapoints.append(prediction_segment_class)
# assign the segment datapoint lists to flatten variables
label_f = label_segment_datapoints
prediction_f = prediction_segment_datapoints
elif mode == 'voxel-wise':
# flatten the 3d volumes to 1d vector, necessary for performance calculation with sklearn functions
label_f = label.flatten()
prediction_f = prediction.flatten()
else:
raise ValueError(
'Mode can be only one of the values ["voxel-wise", "segment-wise"].'
)
scores = evaluate_volume(label_f, prediction_f)
return scores
def evaluate_set(version,
n_epochs,
batch_size,
learning_rate,
dropout_rate,
l1,
l2,
batch_normalization,
deconvolution,
n_base_filters,
n_patients_train,
n_patients_val,
patients,
data_dir,
predictions_dir,
models_dir,
dataset,
result_file,
result_file_per_patient,
label_load_name,
washed=False,
washing_version='score',
mode='voxel-wise',
segment_load_name=''):
"""
:param version: The net version name. String.
:param n_epochs: The number of epochs. Positive integer.
:param batch_size: The size of one mini-batch. Positive integer.
:param learning_rate: The learning rate. Positive float.
:param dropout_rate: The dropout rate. Positive float or None.
:param l1: The L1 regularization. Positive float or None.
:param l2: The L2 regularization. Positive float or None.
:param batch_normalization: Whether to train with batch normalization. Boolean.
:param deconvolution: Whether to use deconvolution instead of up-sampling layer. Boolean.
:param n_base_filters: The number of filters in the first convolutional layer of the net. Positive integer.
:param n_patients_train: The number of patient used for training. Positive integer.
:param n_patients_val: The number of patient used for validation. Positive integer.
:param patients: The names of patients in the given dataset. List of strings.
:param data_dir: The path to data dir. String.
:param predictions_dir: Path to directory where to save predictions and results. String.
:param models_dir: Path to directory where to save models. String.
:param dataset: The dataset. E.g. train/val/set
:param result_file: The path to the csv file where to store the results of the performance calculation calculated
as average over all patient in dataset. String.
:param result_file_per_patient: The path to the csv file where to store the results of the performance calculation
per patient. String.
:param label_load_name: The file name of the ground-truth label. String.
:param washed: True for washed predictions with vessels segments.
:param washing_version: The name of the washing version. Can be empty. String.
:param mode: One of the values ['voxel-wise', 'segment-wise']. Voxel-wise: voxel-wise scores are calculated.
Segment-wise: segment-wise scores are calculated. String.
:param segment_load_name: Filename regex for files containing the skeletons with vessel segments.
:return:
"""
print('label load name', label_load_name)
print('washed', washed)
print('mode', mode)
starttime_set = helper.start_time_measuring(
'performance assessment in set')
# create the name of current run
run_name = bravenet_config.get_run_name(
version=version,
n_epochs=n_epochs,
batch_size=batch_size,
learning_rate=learning_rate,
dropout_rate=dropout_rate,
l1=l1,
l2=l2,
batch_normalization=batch_normalization,
deconvolution=deconvolution,
n_base_filters=n_base_filters,
n_patients_train=n_patients_train,
n_patients_val=n_patients_val)
# -----------------------------------------------------------
# TRAINING PARAMS
# -----------------------------------------------------------
try:
train_metadata_filepath = bravenet_config.get_train_metadata_filepath(
models_dir, run_name)
with open(train_metadata_filepath, 'rb') as handle:
train_metadata = pickle.load(handle)
print('Train params:', train_metadata['params'])
except FileNotFoundError:
train_metadata = None
print('Unexpected error by reading train metadata:', sys.exc_info()[0])
# -----------------------------------------------------------
# DATASET RESULTS (TRAIN / VAL / TEST)
# -----------------------------------------------------------
num_epochs = train_metadata['params'][
'epochs'] if train_metadata else n_epochs
# initialize empty list for saving measures for each patient
acc_list = []
f1_macro_list = []
f1_macro_wo_0_list = []
f1_macro_wo_0_and_1_list = []
avg_acc_list = []
avg_acc_wo_0_list = []
avg_acc_wo_0_and_1_list = []
f1_bin_list = []
f1_class_dict = {} # for saving f1 for each class
for cls in range(bravenet_config.NUM_CLASSES):
f1_class_dict[cls] = []
# calculate the performance per patient
for p, patient in enumerate(patients):
# load patient ground truth and prediction
print(
'-----------------------------------------------------------------'
)
print('PATIENT:', patient, ',', str(p + 1) + '/' + str(len(patients)))
print('> Loading label...')
label = helper.load_nifti_mat_from_file(
os.path.join(data_dir, patient + label_load_name))
print('> Loading prediction...')
if washed:
prediction_path = bravenet_config.get_washed_annotation_filepath(
predictions_dir,
run_name,
patient,
dataset,
washing_version=washing_version)
else:
prediction_path = bravenet_config.get_annotation_filepath(
predictions_dir, run_name, patient, dataset)
if not os.path.exists(prediction_path) and prediction_path.endswith(
'.gz'):
prediction_path = prediction_path[:-3]
prediction = helper.load_nifti_mat_from_file(prediction_path)
# If mode is segment-wise, prepare the segment-wise data points.
if mode == 'segment-wise':
print('> Loading segments...')
skel_seg = helper.load_nifti_mat_from_file(
os.path.join(data_dir, patient + segment_load_name))
# Get label and prediction segment datapoints.
label_segment_datapoints = []
prediction_segment_datapoints = []
unique_segments = np.unique(skel_seg)
for seg in unique_segments:
segment_inds = np.where(skel_seg == seg)
# Get label segment data points.
label_segment_classes = label[segment_inds]
label_segment_unique_classes, label_segment_classes_counts = np.unique(
label_segment_classes, return_counts=True)
if len(label_segment_unique_classes) > 1:
label_segment_class = label_segment_unique_classes[
np.argmax(label_segment_classes_counts)]
else:
label_segment_class = label_segment_unique_classes[0]
label_segment_datapoints.append(label_segment_class)
# Get prediction segment data points.
prediction_segment_classes = prediction[segment_inds]
prediction_segment_unique_classes, prediction_segment_classes_counts = np.unique(
prediction_segment_classes, return_counts=True)
if len(prediction_segment_unique_classes) > 1:
prediction_segment_class = prediction_segment_unique_classes[
np.argmax(prediction_segment_classes_counts)]
else:
prediction_segment_class = prediction_segment_unique_classes[
0]
prediction_segment_datapoints.append(prediction_segment_class)
# assign the segment datapoint lists to flatten variables
label_f = label_segment_datapoints
prediction_f = prediction_segment_datapoints
elif mode == 'voxel-wise':
# flatten the 3d volumes to 1d vector, necessary for performance calculation with sklearn functions
label_f = label.flatten()
prediction_f = prediction.flatten()
else:
raise ValueError(
'Mode can be only one of the values ["voxel-wise", "segment-wise"].'
)
# Calculate scores.
scores = evaluate_volume(label_f, prediction_f)
# find what labels are in ground-truth and what labels were predicted
unique_labels = np.unique(label_f)
unique_prediction = np.unique(prediction_f)
print('label unique', unique_labels, 'size', len(unique_labels))
print('predicted annotation unique', unique_prediction, 'size',
len(unique_prediction))
# save results for patient to the lists
acc_list.append(scores['acc'])
f1_macro_list.append(scores['f1_macro'])
f1_macro_wo_0_list.append(scores['f1_macro_wo_0'])
f1_macro_wo_0_and_1_list.append(scores['f1_macro_wo_0_and_1'])
avg_acc_list.append(scores['avg_acc'])
avg_acc_wo_0_list.append(scores['avg_acc_wo_0'])
avg_acc_wo_0_and_1_list.append(scores['avg_acc_wo_0_and_1'])
f1_bin_list.append(scores['f1_bin'])
all_classes = np.concatenate((unique_labels, unique_prediction))
unique_classes = np.unique(all_classes)
f1_per_classes = scores['f1_per_classes']
patient_f1_class_list = [
'-'
] * bravenet_config.NUM_CLASSES # for saving f1 for each class
for i, cls in enumerate(unique_classes):
f1_class_dict[cls].append(f1_per_classes[i])
patient_f1_class_list[cls] = f1_per_classes[i]
# create row for saving to csv file with details for each patient and write to the csv file
row_per_patient = [
num_epochs, batch_size, learning_rate, dropout_rate, l1, l2,
batch_normalization, deconvolution, n_base_filters, patient,
scores['acc'], scores['f1_macro'], scores['f1_macro_wo_0'],
scores['f1_macro_wo_0_and_1'], scores['avg_acc'],
scores['avg_acc_wo_0'], scores['avg_acc_wo_0_and_1'],
scores['f1_bin']
] + patient_f1_class_list
print('Writing to per patient csv...')
helper.write_to_csv(result_file_per_patient, [row_per_patient])
# calculate patient mean and std for each class
f1_class_list_mean = []
f1_class_list_std = []
for cls, class_f1_list in f1_class_dict.items():
f1_class_list_mean.append(np.mean(class_f1_list))
f1_class_list_std.append(np.std(class_f1_list))
# create row for saving to csv file with averages over whole set and write to the csv file
row_avg = [
num_epochs, batch_size, learning_rate, dropout_rate, l1, l2,
batch_normalization, deconvolution, n_base_filters,
len(patients), 'AVG',
np.mean(acc_list),
np.mean(f1_macro_list),
np.mean(f1_macro_wo_0_list),
np.mean(f1_macro_wo_0_and_1_list),
np.mean(avg_acc_list),
np.mean(avg_acc_wo_0_list),
np.mean(avg_acc_wo_0_and_1_list),
np.mean(f1_bin_list)
] + f1_class_list_mean
row_std = [
num_epochs, batch_size, learning_rate, dropout_rate, l1, l2,
batch_normalization, deconvolution, n_base_filters,
len(patients), 'STD',
np.std(acc_list),
np.std(f1_macro_list),
np.std(f1_macro_wo_0_list),
np.std(f1_macro_wo_0_and_1_list),
np.std(avg_acc_list),
np.std(avg_acc_wo_0_list),
np.std(avg_acc_wo_0_and_1_list),
np.std(f1_bin_list)
] + f1_class_list_std
print('AVG:', row_avg)
print('STD:', row_std)
print('Writing to csv...')
helper.write_to_csv(result_file, [row_avg, row_std])
# print out how long did the calculations take
helper.end_time_measuring(starttime_set,
what_to_measure='performance assessment in set')
print('Patient files:', patient_files)
# check number of patients in datasets
num_patients = len(patient_files)
print('Number of patients:', num_patients)
save_directory = os.path.join(data_dir, rf_config.VESSEL_VOXEL_DATA_DIR)
if not os.path.exists(save_directory):
os.makedirs(save_directory)
# extract features and labels voxel-wise from patient in train dataset
for patient in patient_files:
print('PATIENT: ', patient)
# load image, label and skeleton
feature_list = [
helper.load_nifti_mat_from_file(os.path.join(data_dir, patient + f),
printout=True)
for f in feature_filenames
]
label = helper.load_nifti_mat_from_file(os.path.join(
data_dir, patient + label_filename),
printout=True)
# check if the skeletons of the different features and target match
helper.nonzero_indices_match(feature_list + [label])
# find all vessel voxel indices
vessel_inds = np.where(feature_list[0] > 0)
num_voxels = len(vessel_inds[0])
# prepare matrices for saving
features = []
helper.check_num_patients(patients)
all_patients = patients['working']
num_patients = len(all_patients)
print('Number of patients:', num_patients)
patch_directory = bravenet_config.SAMPLES_PATH
if not os.path.exists(patch_directory):
os.makedirs(patch_directory)
for patient in all_patients:
print('DATA SET:', dataset)
print('PATIENT:', patient)
patient_class_list = []
# load data
print('> Loading features...')
features = [helper.load_nifti_mat_from_file(
os.path.join(data_dir, patient + feature_file_name)) for feature_file_name in feature_file_names]
print('> Loading label...')
label = helper.load_nifti_mat_from_file(os.path.join(data_dir, patient + label_name))
print('Patient ', patient, ': features and label loaded.')
helper.check_dimensions(features, label.shape)
# sizes
min_x = 0
min_y = 0
min_z = 0
max_x = label.shape[0]
max_y = label.shape[1]
max_z = label.shape[2]
# for each class except background and not-annotated extract given number of patches
patch = 0
def pipeline(class_dict, datasets, predictions_dir, models_dir, data_dir, batch_size_list, learning_rate_list,
dropout_rate_list, l1_list, l2_list, batch_normalization_list, deconvolution_list, nr_base_filters_list,
version, nr_epochs, depth, filter_size, activation, final_activation, num_classes, optimizer,
loss_function, metrics, n_train_patients, n_val_patients, checkpoint_model, train_feature_files,
train_feature_files_big, train_label_files, train_label_files_big, val_feature_files,
val_feature_files_big, val_label_files, val_label_files_big, normalize_features,
max_values_for_normalization, transfer_learning, transfer_weights_path, feature_file_names,
patch_size_x, patch_size_y, patch_size_z, label_file_name,
washing_version, patients, select_min_loss):
# -----------------------------------------------------------
# PREPARE FILES FOR STORING RESULTS
# -----------------------------------------------------------
all_classes = [*class_dict]
all_classes = list(map(str, all_classes))
result_files_dict = {}
result_files_dict_washed = {}
result_files_dict_segment_wise = {}
for dataset in datasets:
if not os.path.exists(os.path.join(predictions_dir, dataset)):
os.makedirs(os.path.join(predictions_dir, dataset))
result_file = bravenet_config.get_complete_result_table_filepath(predictions_dir, dataset)
result_file_per_patient = bravenet_config.get_complete_result_table_per_patient_filepath(predictions_dir, dataset)
result_file_washed = bravenet_config.get_complete_result_table_filepath(predictions_dir, dataset, washed=True)
result_file_per_patient_washed = bravenet_config.get_complete_result_table_per_patient_filepath(predictions_dir,
dataset,
washed=True)
result_file_segment_wise = bravenet_config.get_complete_result_table_filepath(predictions_dir, dataset,
washed=True, segment_wise=True)
result_file_per_patient_segment_wise = bravenet_config.get_complete_result_table_per_patient_filepath(
predictions_dir, dataset, washed=True, segment_wise=True)
header_row, header_row_patient = get_csv_headers(dataset, all_classes)
helper.write_to_csv(result_file, [header_row])
helper.write_to_csv(result_file_per_patient, [header_row_patient])
helper.write_to_csv(result_file_washed, [header_row])
helper.write_to_csv(result_file_per_patient_washed, [header_row_patient])
helper.write_to_csv(result_file_segment_wise, [header_row])
helper.write_to_csv(result_file_per_patient_segment_wise, [header_row_patient])
result_files_dict[dataset] = [result_file, result_file_per_patient]
result_files_dict_washed[dataset] = [result_file_washed, result_file_per_patient_washed]
result_files_dict_segment_wise[dataset] = [result_file_segment_wise, result_file_per_patient_segment_wise]
# -----------------------------------------------------------
# FINE GRID FOR PARAMETER TUNING
# -----------------------------------------------------------
hyperparam_grid = []
for bs in batch_size_list:
for lr in learning_rate_list:
for dr in dropout_rate_list:
for l1 in l1_list:
for l2 in l2_list:
for bn in batch_normalization_list:
for deconv in deconvolution_list:
for nr_base_filt in nr_base_filters_list:
hyperparam_grid.append((bs, lr, dr, l1, l2, bn, deconv, nr_base_filt))
for bs, lr, dr, l1, l2, bn, deconv, nr_base_filt in hyperparam_grid:
# -----------------------------------------------------------
# TRAIN BRAVENET AND SAVE MODEL AND RESULTS
# -----------------------------------------------------------
train(version, n_epochs=nr_epochs, batch_size=bs, lr=lr, dr=dr, l1=l1, l2=l2, bn=bn,
n_base_filters=nr_base_filt, depth=depth, filter_size=filter_size, activation=activation,
final_activation=final_activation, n_classes=num_classes, optimizer=optimizer,
loss_function=loss_function, metrics=metrics, deconvolution=deconv,
n_train_patients=n_train_patients, n_val_patients=n_val_patients,
checkpoint_model=checkpoint_model, models_dir=models_dir,
train_feature_files=train_feature_files, train_feature_files_big=train_feature_files_big,
train_label_files=train_label_files, train_label_files_big=train_label_files_big,
val_feature_files=val_feature_files, val_feature_files_big=val_feature_files_big,
val_label_files=val_label_files, val_label_files_big=val_label_files_big,
normalize_features=normalize_features, max_values_for_normalization=max_values_for_normalization,
transfer_learning=transfer_learning, transfer_weights_path=transfer_weights_path)
print('_' * 100)
print('Training completed.')
print('_' * 100)
# -----------------------------------------------------------
# PREDICT AND EVALUATE
# -----------------------------------------------------------
print('Starting prediction...')
print('_' * 100)
run_name = bravenet_config.get_run_name(version=version, n_epochs=nr_epochs, batch_size=bs, learning_rate=lr,
dropout_rate=dr, l1=l1, l2=l2, batch_normalization=bn,
deconvolution=deconv, n_base_filters=nr_base_filt,
n_patients_train=n_train_patients, n_patients_val=n_val_patients)
train_metadata_filepath = bravenet_config.get_train_metadata_filepath(models_dir, run_name)
train_metadata = helper.load_model_metadata(train_metadata_filepath)
# Select model with min val loss from last 3 epochs.
print('Last 3 epochs:', train_metadata['history']['val_loss'][-3:])
if select_min_loss:
selected_epoch = np.argmin(train_metadata['history']['val_loss'][-3:]) + nr_epochs - 2
else:
selected_epoch = nr_epochs
print('Selected epoch:', selected_epoch)
run_name = bravenet_config.get_run_name(version=version, n_epochs=selected_epoch, batch_size=bs,
learning_rate=lr, dropout_rate=dr, l1=l1, l2=l2, batch_normalization=bn,
deconvolution=deconv, n_base_filters=nr_base_filt,
n_patients_train=n_train_patients, n_patients_val=n_val_patients)
model, _ = load_model_and_metadata(run_name=run_name, models_dir=models_dir)
for dataset in datasets:
print('DATASET', dataset)
set_patients = patients[dataset]['working']
if not os.path.exists(os.path.join(predictions_dir, dataset)):
os.makedirs(os.path.join(predictions_dir, dataset))
for patient in set_patients:
print('-' * 100)
feature_volumes = helper.load_feature_volumes(feature_file_names=feature_file_names, data_dir=data_dir,
patient=patient,
num_features_for_check=bravenet_config.NUM_FEATURES)
print('> Loading segmented skeleton...')
skeleton_segments_path = os.path.join(data_dir, patient + '_skel_seg_y.nii.gz')
skeleton_segments = helper.load_nifti_mat_from_file(skeleton_segments_path)
washed_annotation_save_path = bravenet_config.get_washed_annotation_filepath(
predictions_dir=predictions_dir, run_name=run_name, patient=patient, dataset=dataset)
annotation_save_path = bravenet_config.get_annotation_filepath(predictions_dir=predictions_dir,
run_name=run_name, patient=patient,
dataset=dataset)
predict(feature_volumes=feature_volumes, model=model, num_classes=num_classes,
patch_size_x=patch_size_x, patch_size_y=patch_size_y, patch_size_z=patch_size_z,
annotation_save_path=annotation_save_path,
max_values_for_normalization=max_values_for_normalization, skeleton_segments=skeleton_segments,
washed_annotation_save_path=washed_annotation_save_path)
print('_' * 100)
print('Prediction in %s dataset completed.' % dataset)
print('_' * 100)
# -----------------------------------------------------------
# EVALUATE
# -----------------------------------------------------------
print('Starting evaluation in %s dataset...' % dataset)
print('_' * 100)
evaluate_set(version=version, n_epochs=selected_epoch, batch_size=bs, learning_rate=lr, dropout_rate=dr,
l1=l1,
l2=l2, batch_normalization=bn, deconvolution=deconv, n_base_filters=nr_base_filt,
n_patients_train=n_train_patients, n_patients_val=n_val_patients, patients=set_patients,
data_dir=data_dir, predictions_dir=predictions_dir, models_dir=models_dir, dataset=dataset,
result_file=result_files_dict[dataset][0],
result_file_per_patient=result_files_dict[dataset][1],
label_load_name=label_file_name, washed=False, washing_version=washing_version,
mode='voxel-wise')
evaluate_set(version=version, n_epochs=selected_epoch, batch_size=bs, learning_rate=lr, dropout_rate=dr,
l1=l1,
l2=l2, batch_normalization=bn, deconvolution=deconv, n_base_filters=nr_base_filt,
n_patients_train=n_train_patients, n_patients_val=n_val_patients, patients=set_patients,
data_dir=data_dir, predictions_dir=predictions_dir, models_dir=models_dir, dataset=dataset,
result_file=result_files_dict_washed[dataset][0],
result_file_per_patient=result_files_dict_washed[dataset][1], label_load_name=label_file_name,
washed=True, washing_version=washing_version, mode='voxel-wise')
evaluate_set(version=version, n_epochs=selected_epoch, batch_size=bs, learning_rate=lr,
dropout_rate=dr, l1=l1, l2=l2, batch_normalization=bn, deconvolution=deconv,
n_base_filters=nr_base_filt, n_patients_train=n_train_patients, n_patients_val=n_val_patients,
patients=set_patients, data_dir=data_dir, predictions_dir=predictions_dir,
models_dir=models_dir, dataset=dataset, result_file=result_files_dict_segment_wise[dataset][0],
result_file_per_patient=result_files_dict_segment_wise[dataset][1],
label_load_name=label_file_name, washed=True, washing_version=washing_version,
mode='segment-wise', segment_load_name='_skel_seg_y.nii.gz')
print('_' * 100)
print('Evaluation in %s dataset completed.' % dataset)
print('_' * 100)
K.clear_session()
return True