def main(): file_train = h5py.File( "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5", "r") clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id") patient_list = list(file_train.keys()) patient_list = [p for p in patient_list if p not in ["PatientLC_63"]] patient_list_train, patient_list_val = get_trainval_patient_list( clinical_df, patient_list) data_val = get_tf_data( file_train, clinical_df, output_shape_image=(256, 256), random_slice=False, centered_on_gtvt=True, patient_list_copy=patient_list_val, ).cache().batch(2) data_train = get_tf_data(file_train, clinical_df, output_shape_image=(256, 256), random_slice=True, random_shift=20, n_repeat=10, num_parallel_calls='auto', oversample_plc_neg=True, patient_list_copy=patient_list_train).batch(bs) for x, y, plc_status in data_val.as_numpy_iterator(): print( f"voici, voilé le x {x.shape}, le y {y.shape} et le plc_status {plc_status}" ) file_train.close()
def main(): file_train = h5py.File( "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5", "r") file_test = h5py.File( "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/test.hdf5", "r") clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id") if checkpoint: model_ = unet_model(3, input_shape=image_size + (3, )) model = CustomModel(model_.inputs, model_.outputs) else: model = tf.keras.models.load_model(model_path, compile=False) optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) model.compile( optimizer=optimizer, run_eagerly=False, ) if checkpoint: model.load_weights(model_path) data_test = get_tf_data( file_test, clinical_df, output_shape_image=(256, 256), random_slice=False, centered_on_gtvt=True, ).cache().batch(2) results = evaluate_model( model, data_test, str_df="test", w_gtvl=4, w_gtvt=1, w_lung=1, alpha=0.75, s_gtvl=40, ) print(f" REstusl : {results}") file_train.close() file_test.close()
def main(): # file_train = h5py.File( # "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5", # "r") file_test = h5py.File( "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/test.hdf5", "r") clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id") model = tf.keras.models.load_model(model_path, compile=False) data_test = get_tf_data( file_test, clinical_df, output_shape_image=(256, 256), random_slice=False, centered_on_gtvt=True, return_complete_gtvl=True, return_patient=True, return_plc_status=False, ).batch(4) optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) model.compile( loss=CustomLoss(), optimizer=optimizer, run_eagerly=False, ) for x, y_true, patient_id in data_test.take(1).as_numpy_iterator(): y_pred = model(x).numpy() for i in range(x.shape[0]): print(y_pred.shape) # file_train.close() file_test.close()
def main(): file = h5py.File( "/home/valentin/python_wkspce/plc_seg/data/processed/2d_pet_normalized/data.hdf5", "r") clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id") patient_list = list(file.keys()) patient_list = [p for p in patient_list if p not in ["PatientLC_63"]] patient_list_train, patient_list_val, patient_list_test = get_split_patient_lists( clinical_df, patient_list) data_val = get_tf_data( file, clinical_df, output_shape=(256, 256), random_slice=False, centered_on_gtvt=False, patient_list=patient_list_val, output_type=task_type, # "segmentation" "plc_status" ).cache().batch(2) data_train = get_tf_data( file, clinical_df, output_shape=(256, 256), random_slice=True, random_shift=20, n_repeat=10, num_parallel_calls='auto', oversample_plc_neg=True, output_type=task_type, # "segmentation" "plc_status" patient_list=patient_list_train, random_angle=15, ).batch(bs) model = model_dict[task_type](3, input_shape=image_size + (3, )) # model = UnetClassif() optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) early_stop_callback = tf.keras.callbacks.EarlyStopping( monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='auto', restore_best_weights=True) def gtvl_metrics(y_true, y_pred): return gtvl_loss(y_true, y_pred, scaling=s_gtvl, alpha=alpha) model.compile( loss=losses_dict[task_type], # metrics=[gtvl_metrics], optimizer=optimizer, run_eagerly=False, ) evaluator = evaluator_dict[task_type]( model, file, clinical_df, output_shape=(256, 256), ) auc_train = evaluator(patient_list_train) auc_val = evaluator(patient_list_val) auc_test = evaluator(patient_list_test) print(f"======================== " f"That's the AUC before fitting" f" of the train, val and test resp.: " f"{auc_train}, {auc_val} and {auc_test}") model.fit( x=data_train, epochs=n_epochs, validation_data=data_val, callbacks=[early_stop_callback, tensorboard_callback], ) auc_train = evaluator( patient_list_train, path_to_save_nii= "/home/valentin/python_wkspce/plc_seg/data/reports/train") auc_val = evaluator( patient_list_val, path_to_save_nii="/home/valentin/python_wkspce/plc_seg/data/reports/val" ) auc_test = evaluator( patient_list_test, path_to_save_nii= "/home/valentin/python_wkspce/plc_seg/data/reports/test") print(f"That's the AUC after fitting" f" of the train, val and test resp.: " f"{auc_train}, {auc_val} and {auc_test}") # model.save(path_model) file.close()
def main(): file_train = h5py.File( "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5", "r") clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id") patient_list = list(file_train.keys()) patient_list = [p for p in patient_list if p not in ["PatientLC_63"]] patient_list_train, patient_list_val = get_trainval_patient_list( clinical_df, patient_list) data_val = get_tf_data( file_train, clinical_df, output_shape_image=(256, 256), random_slice=False, label_to_contain="GTV T", patient_list=patient_list_val, ).cache().batch(2) data_train = get_tf_data(file_train, clinical_df, output_shape_image=(256, 256), random_slice=True, random_shift=20, n_repeat=10, num_parallel_calls='auto', oversample_plc_neg=True, patient_list=patient_list_train).batch(bs) tuner = kt.Hyperband(model_builder, objective=kt.Objective("val_gtvl_metric", direction="min"), max_epochs=50, factor=3, directory='my_dir', project_name='intro_to_kt') tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) early_stop_callback = tf.keras.callbacks.EarlyStopping( monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto', restore_best_weights=True) tuner.search(x=data_train, epochs=50, validation_data=data_val, callbacks=[early_stop_callback]) # Get the optimal hyperparameters best_hps = tuner.get_best_hyperparameters(num_trials=1)[0] print(f""" The hyperparameter search is complete. The optimal hyperparameters are alpha: {best_hps.get('alpha')}, s_gtvl: {best_hps.get('s_gtvl')}, w_gtvl: {best_hps.get('w_gtvl')}, w_gtvt: {best_hps.get('w_gtvt')}, w_lung: {best_hps.get('w_lung')}, and the optimal learning rate for the optimizer is {best_hps.get('learning_rate')}. """) model = tuner.hypermodel.build(best_hps) model.fit( x=data_train, epochs=n_epochs, validation_data=data_val, callbacks=[early_stop_callback, tensorboard_callback], ) model.save(path_model) file_train.close()
def main(): file_train = h5py.File( "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/train.hdf5", "r") # file_test = h5py.File( # "/home/val/python_wkspce/plc_seg/data/processed/2d_pet_normalized/test.hdf5", # "r") clinical_df = pd.read_csv(path_clinical_info).set_index("patient_id") patient_list = list(file_train.keys()) patient_list = [p for p in patient_list if p not in ["PatientLC_63"]] patient_list_train, patient_list_val = get_trainval_patient_list( clinical_df, patient_list) data_val = get_tf_data( file_train, clinical_df, output_shape_image=(256, 256), random_slice=False, centered_on_gtvt=True, patient_list=patient_list_val, ).cache().batch(2) data_train = get_tf_data(file_train, clinical_df, output_shape_image=(256, 256), random_slice=True, random_shift=20, n_repeat=10, num_parallel_calls='auto', oversample_plc_neg=True, patient_list=patient_list_train).batch(bs) data_train_eval = get_tf_data( file_train, clinical_df, output_shape_image=(256, 256), random_slice=False, oversample_plc_neg=False, centered_on_gtvt=True, patient_list=patient_list_train).cache().batch(bs) results_df = pd.DataFrame() for alpha, w_gtvl, s_gtvl, w_gtvt, w_lung, rep in product( alphas, ws_gtvl, ss_gtvl, ws_gtvt, ws_lung, reps): model_name = ( f"model_{alpha:0.2f}_alpha_{w_gtvl:0.2f}_wplc_" f"{w_gtvt:0.2f}_wt_{w_lung:0.2f}_wl_{s_gtvl:0.2f}_splc_early_stop_rep{rep}" .replace(".", "-")) model_path = project_dir / f"models/early_stop/{model_name}/final/" checkpoint_filepath = project_dir / (f"models/early_stop/{model_name}" f"/checkpoint/checkpoint") results_path = project_dir / f"data/results/{model_name}.csv" model_ = unet_model(3, input_shape=image_size + (3, )) optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) model = CustomModel( model_.inputs, model_.outputs, alpha=alpha, w_lung=w_lung, w_gtvt=w_gtvt, w_gtvl=w_gtvl, s_gtvl=s_gtvl, ) model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_filepath, save_weights_only=True, monitor='val_loss', mode='min', save_best_only=False, # save_freq=1, ) early_stop_callback = tf.keras.callbacks.EarlyStopping( monitor='val_gtvl_loss', min_delta=0, patience=5, verbose=0, mode='auto', restore_best_weights=True) model.compile( loss=custom_loss, optimizer=optimizer, run_eagerly=True, ) model.fit( x=data_train, epochs=n_epochs, validation_data=data_val, callbacks=[model_checkpoint_callback, early_stop_callback], ) model.save(model_path) res_train = pd.DataFrame( evaluate_model( model, data_train_eval, str_df="train", w_gtvl=w_gtvl, w_gtvt=w_gtvt, w_lung=w_lung, alpha=alpha, s_gtvl=s_gtvl, )) res_val = pd.DataFrame( evaluate_model( model, data_val, str_df="val", w_gtvl=w_gtvl, w_gtvt=w_gtvt, w_lung=w_lung, alpha=alpha, s_gtvl=s_gtvl, )) df = pd.concat([ res_train, res_val, pd.DataFrame({ "alpha": [alpha], "w_gtvl": [w_gtvl], "s_gtvl": [s_gtvl], "w_gtvt": [w_gtvt], "w_lung": [w_lung] }) ], axis=1) df.to_csv(results_path) results_df = results_df.append(df, ignore_index=True) del model # Just to be sure file_train.close() results_df.to_csv(project_dir / "data/results/all_results_early_stop_rep.csv")
def train_one_rep(model, losses, file, clinical_df, task_type): patient_list = list(file.keys()) patient_list = [p for p in patient_list if p not in ["PatientLC_63"]] patient_list_train, patient_list_val, patient_list_test = get_split_patient_lists( clinical_df, patient_list) data_val = get_tf_data( file, clinical_df, output_shape=(256, 256), random_slice=False, centered_on_gtvt=False, patient_list=patient_list_val, output_type=task_type, # "segmentation" "plc_status" ).cache().batch(2) data_train = get_tf_data( file, clinical_df, output_shape=(256, 256), random_slice=True, random_shift=20, n_repeat=10, num_parallel_calls='auto', oversample_plc_neg=True, output_type=task_type, # "segmentation" "plc_status" patient_list=patient_list_train, # random_angle=0, ).batch(bs) optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) early_stop_callback = tf.keras.callbacks.EarlyStopping( monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='auto', restore_best_weights=True) model.compile( loss=losses, # metrics=[gtvl_metrics], optimizer=optimizer, run_eagerly=False, ) evaluator = evaluator_dict[task_type]( model, file, clinical_df, output_shape=(256, 256), ) model.fit( x=data_train, epochs=n_epochs, validation_data=data_val, callbacks=[early_stop_callback, tensorboard_callback], ) auc_train = evaluator(patient_list_train) auc_val = evaluator(patient_list_val) auc_test = evaluator(patient_list_test) print(f"That's the AUC after fitting" f" of the train, val and test resp.: " f"{auc_train}, {auc_val} and {auc_test}") return auc_train, auc_val, auc_test