def main():
file_train = h5py.File(
"/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5",
"r")
clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id")
patient_list = list(file_train.keys())
patient_list = [p for p in patient_list if p not in ["PatientLC_63"]]
patient_list_train, patient_list_val = get_trainval_patient_list(
clinical_df, patient_list)
data_val = get_tf_data(
file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=False,
centered_on_gtvt=True,
patient_list_copy=patient_list_val,
).cache().batch(2)
data_train = get_tf_data(file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=True,
random_shift=20,
n_repeat=10,
num_parallel_calls='auto',
oversample_plc_neg=True,
patient_list_copy=patient_list_train).batch(bs)
for x, y, plc_status in data_val.as_numpy_iterator():
print(
f"voici, voilé le x {x.shape}, le y {y.shape} et le plc_status {plc_status}"
)
file_train.close()
def main():
file_train = h5py.File(
"/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5",
"r")
file_test = h5py.File(
"/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/test.hdf5",
"r")
clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id")
if checkpoint:
model_ = unet_model(3, input_shape=image_size + (3, ))
model = CustomModel(model_.inputs, model_.outputs)
else:
model = tf.keras.models.load_model(model_path, compile=False)
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
model.compile(
optimizer=optimizer,
run_eagerly=False,
)
if checkpoint:
model.load_weights(model_path)
data_test = get_tf_data(
file_test,
clinical_df,
output_shape_image=(256, 256),
random_slice=False,
centered_on_gtvt=True,
).cache().batch(2)
results = evaluate_model(
model,
data_test,
str_df="test",
w_gtvl=4,
w_gtvt=1,
w_lung=1,
alpha=0.75,
s_gtvl=40,
)
print(f" REstusl : {results}")
file_train.close()
file_test.close()
def main():
# file_train = h5py.File(
# "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5",
# "r")
file_test = h5py.File(
"/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/test.hdf5",
"r")
clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id")
model = tf.keras.models.load_model(model_path, compile=False)
data_test = get_tf_data(
file_test,
clinical_df,
output_shape_image=(256, 256),
random_slice=False,
centered_on_gtvt=True,
return_complete_gtvl=True,
return_patient=True,
return_plc_status=False,
).batch(4)
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
model.compile(
loss=CustomLoss(),
optimizer=optimizer,
run_eagerly=False,
)
for x, y_true, patient_id in data_test.take(1).as_numpy_iterator():
y_pred = model(x).numpy()
for i in range(x.shape[0]):
print(y_pred.shape)
# file_train.close()
file_test.close()
def main():
file = h5py.File(
"/home/valentin/python_wkspce/plc_seg/data/processed/2d_pet_normalized/data.hdf5",
"r")
clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id")
patient_list = list(file.keys())
patient_list = [p for p in patient_list if p not in ["PatientLC_63"]]
patient_list_train, patient_list_val, patient_list_test = get_split_patient_lists(
clinical_df, patient_list)
data_val = get_tf_data(
file,
clinical_df,
output_shape=(256, 256),
random_slice=False,
centered_on_gtvt=False,
patient_list=patient_list_val,
output_type=task_type, # "segmentation" "plc_status"
).cache().batch(2)
data_train = get_tf_data(
file,
clinical_df,
output_shape=(256, 256),
random_slice=True,
random_shift=20,
n_repeat=10,
num_parallel_calls='auto',
oversample_plc_neg=True,
output_type=task_type, # "segmentation" "plc_status"
patient_list=patient_list_train,
random_angle=15,
).batch(bs)
model = model_dict[task_type](3, input_shape=image_size + (3, ))
# model = UnetClassif()
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
early_stop_callback = tf.keras.callbacks.EarlyStopping(
monitor='val_loss',
min_delta=0,
patience=15,
verbose=0,
mode='auto',
restore_best_weights=True)
def gtvl_metrics(y_true, y_pred):
return gtvl_loss(y_true, y_pred, scaling=s_gtvl, alpha=alpha)
model.compile(
loss=losses_dict[task_type],
# metrics=[gtvl_metrics],
optimizer=optimizer,
run_eagerly=False,
)
evaluator = evaluator_dict[task_type](
model,
file,
clinical_df,
output_shape=(256, 256),
)
auc_train = evaluator(patient_list_train)
auc_val = evaluator(patient_list_val)
auc_test = evaluator(patient_list_test)
print(f"======================== "
f"That's the AUC before fitting"
f" of the train, val and test resp.: "
f"{auc_train}, {auc_val} and {auc_test}")
model.fit(
x=data_train,
epochs=n_epochs,
validation_data=data_val,
callbacks=[early_stop_callback, tensorboard_callback],
)
auc_train = evaluator(
patient_list_train,
path_to_save_nii=
"/home/valentin/python_wkspce/plc_seg/data/reports/train")
auc_val = evaluator(
patient_list_val,
path_to_save_nii="/home/valentin/python_wkspce/plc_seg/data/reports/val"
)
auc_test = evaluator(
patient_list_test,
path_to_save_nii=
"/home/valentin/python_wkspce/plc_seg/data/reports/test")
print(f"That's the AUC after fitting"
f" of the train, val and test resp.: "
f"{auc_train}, {auc_val} and {auc_test}")
# model.save(path_model)
file.close()
def main():
file_train = h5py.File(
"/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5",
"r")
clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id")
patient_list = list(file_train.keys())
patient_list = [p for p in patient_list if p not in ["PatientLC_63"]]
patient_list_train, patient_list_val = get_trainval_patient_list(
clinical_df, patient_list)
data_val = get_tf_data(
file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=False,
label_to_contain="GTV T",
patient_list=patient_list_val,
).cache().batch(2)
data_train = get_tf_data(file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=True,
random_shift=20,
n_repeat=10,
num_parallel_calls='auto',
oversample_plc_neg=True,
patient_list=patient_list_train).batch(bs)
tuner = kt.Hyperband(model_builder,
objective=kt.Objective("val_gtvl_metric",
direction="min"),
max_epochs=50,
factor=3,
directory='my_dir',
project_name='intro_to_kt')
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
early_stop_callback = tf.keras.callbacks.EarlyStopping(
monitor='val_loss',
min_delta=0,
patience=5,
verbose=0,
mode='auto',
restore_best_weights=True)
tuner.search(x=data_train,
epochs=50,
validation_data=data_val,
callbacks=[early_stop_callback])
# Get the optimal hyperparameters
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"""
The hyperparameter search is complete. The optimal
hyperparameters are alpha: {best_hps.get('alpha')},
s_gtvl: {best_hps.get('s_gtvl')},
w_gtvl: {best_hps.get('w_gtvl')},
w_gtvt: {best_hps.get('w_gtvt')},
w_lung: {best_hps.get('w_lung')},
and the optimal learning rate for the optimizer
is {best_hps.get('learning_rate')}.
""")
model = tuner.hypermodel.build(best_hps)
model.fit(
x=data_train,
epochs=n_epochs,
validation_data=data_val,
callbacks=[early_stop_callback, tensorboard_callback],
)
model.save(path_model)
file_train.close()
def main():
file_train = h5py.File(
"/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5",
"r")
# file_test = h5py.File(
# "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/test.hdf5",
# "r")
clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id")
patient_list = list(file_train.keys())
patient_list = [p for p in patient_list if p not in ["PatientLC_63"]]
patient_list_train, patient_list_val = get_trainval_patient_list(
clinical_df, patient_list)
data_val = get_tf_data(
file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=False,
centered_on_gtvt=True,
patient_list=patient_list_val,
).cache().batch(2)
data_train = get_tf_data(file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=True,
random_shift=20,
n_repeat=10,
num_parallel_calls='auto',
oversample_plc_neg=True,
patient_list=patient_list_train).batch(bs)
data_train_eval = get_tf_data(
file_train,
clinical_df,
output_shape_image=(256, 256),
random_slice=False,
oversample_plc_neg=False,
centered_on_gtvt=True,
patient_list=patient_list_train).cache().batch(bs)
results_df = pd.DataFrame()
for alpha, w_gtvl, s_gtvl, w_gtvt, w_lung, rep in product(
alphas, ws_gtvl, ss_gtvl, ws_gtvt, ws_lung, reps):
model_name = (
f"model_{alpha:0.2f}_alpha_{w_gtvl:0.2f}_wplc_"
f"{w_gtvt:0.2f}_wt_{w_lung:0.2f}_wl_{s_gtvl:0.2f}_splc_early_stop_rep{rep}"
.replace(".", "-"))
model_path = project_dir / f"models/early_stop/{model_name}/final/"
checkpoint_filepath = project_dir / (f"models/early_stop/{model_name}"
f"/checkpoint/checkpoint")
results_path = project_dir / f"data/results/{model_name}.csv"
model_ = unet_model(3, input_shape=image_size + (3, ))
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
model = CustomModel(
model_.inputs,
model_.outputs,
alpha=alpha,
w_lung=w_lung,
w_gtvt=w_gtvt,
w_gtvl=w_gtvl,
s_gtvl=s_gtvl,
)
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=True,
monitor='val_loss',
mode='min',
save_best_only=False,
# save_freq=1,
)
early_stop_callback = tf.keras.callbacks.EarlyStopping(
monitor='val_gtvl_loss',
min_delta=0,
patience=5,
verbose=0,
mode='auto',
restore_best_weights=True)
model.compile(
loss=custom_loss,
optimizer=optimizer,
run_eagerly=True,
)
model.fit(
x=data_train,
epochs=n_epochs,
validation_data=data_val,
callbacks=[model_checkpoint_callback, early_stop_callback],
)
model.save(model_path)
res_train = pd.DataFrame(
evaluate_model(
model,
data_train_eval,
str_df="train",
w_gtvl=w_gtvl,
w_gtvt=w_gtvt,
w_lung=w_lung,
alpha=alpha,
s_gtvl=s_gtvl,
))
res_val = pd.DataFrame(
evaluate_model(
model,
data_val,
str_df="val",
w_gtvl=w_gtvl,
w_gtvt=w_gtvt,
w_lung=w_lung,
alpha=alpha,
s_gtvl=s_gtvl,
))
df = pd.concat([
res_train, res_val,
pd.DataFrame({
"alpha": [alpha],
"w_gtvl": [w_gtvl],
"s_gtvl": [s_gtvl],
"w_gtvt": [w_gtvt],
"w_lung": [w_lung]
})
],
axis=1)
df.to_csv(results_path)
results_df = results_df.append(df, ignore_index=True)
del model # Just to be sure
file_train.close()
results_df.to_csv(project_dir /
"data/results/all_results_early_stop_rep.csv")
def train_one_rep(model, losses, file, clinical_df, task_type):
patient_list = list(file.keys())
patient_list = [p for p in patient_list if p not in ["PatientLC_63"]]
patient_list_train, patient_list_val, patient_list_test = get_split_patient_lists(
clinical_df, patient_list)
data_val = get_tf_data(
file,
clinical_df,
output_shape=(256, 256),
random_slice=False,
centered_on_gtvt=False,
patient_list=patient_list_val,
output_type=task_type, # "segmentation" "plc_status"
).cache().batch(2)
data_train = get_tf_data(
file,
clinical_df,
output_shape=(256, 256),
random_slice=True,
random_shift=20,
n_repeat=10,
num_parallel_calls='auto',
oversample_plc_neg=True,
output_type=task_type, # "segmentation" "plc_status"
patient_list=patient_list_train,
# random_angle=0,
).batch(bs)
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
early_stop_callback = tf.keras.callbacks.EarlyStopping(
monitor='val_loss',
min_delta=0,
patience=15,
verbose=0,
mode='auto',
restore_best_weights=True)
model.compile(
loss=losses,
# metrics=[gtvl_metrics],
optimizer=optimizer,
run_eagerly=False,
)
evaluator = evaluator_dict[task_type](
model,
file,
clinical_df,
output_shape=(256, 256),
)
model.fit(
x=data_train,
epochs=n_epochs,
validation_data=data_val,
callbacks=[early_stop_callback, tensorboard_callback],
)
auc_train = evaluator(patient_list_train)
auc_val = evaluator(patient_list_val)
auc_test = evaluator(patient_list_test)
print(f"That's the AUC after fitting"
f" of the train, val and test resp.: "
f"{auc_train}, {auc_val} and {auc_test}")
return auc_train, auc_val, auc_test