def modelpredict(): # parser config config_file = "./config.ini" cp = ConfigParser() cp.read(config_file) # default config output_dir = cp["DEFAULT"].get("output_dir") class_names = cp["DEFAULT"].get("class_names").split(",") # test config batch_size = cp["DEV"].getint("batch_size") use_best_weights = cp["DEV"].getboolean("use_best_weights") # parse weights file path output_weights_name = cp["TRAIN"].get("output_weights_name") weights_path = os.path.join(output_dir, output_weights_name) best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}") # get test sample count test_counts, _ = get_sample_counts(output_dir, "dev", class_names) symlink_dir_name = "image_links" dev_data_path = f"{output_dir}/{symlink_dir_name}/dev/" step_test = int(test_counts / batch_size) print("** load dev generator **") test_generator = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1. / 255), dev_data_path, batch_size=batch_size, class_names=class_names, ) x, y = load_generator_data(test_generator, step_test) print("** load model **") model = get_model(class_names) if use_best_weights: print("** use best weights **") model.load_weights(best_weights_path) else: print("** use last weights **") model.load_weights(weights_path) print("** make prediction **") y_hat = model.predict(x) # print(y_hat) # dev_log_path = os.path.join(output_dir, "dev.log") # print(f"** write log to {dev_log_path} **") # aurocs = [] # with open(dev_log_path, "w") as f: # for i in range(len(class_names)): # try: # score = roc_auc_score(y[i], y_hat[i]) # aurocs.append(score) # except ValueError: # score = 0 # f.write(f"{class_names[i]}: {score}\n") # mean_auroc = np.mean(aurocs) # f.write("-------------------------\n") # f.write(f"mean auroc: {mean_auroc}\n") return y, y_hat
def main(): # parser config config_file = "./config.ini" cp = ConfigParser() cp.read(config_file) # default config output_dir = cp["DEFAULT"].get("output_dir") image_source_dir = cp["DEFAULT"].get("image_source_dir") train_patient_count = cp["DEFAULT"].getint("train_patient_count") dev_patient_count = cp["DEFAULT"].getint("dev_patient_count") data_entry_file = cp["DEFAULT"].get("data_entry_file") class_names = cp["DEFAULT"].get("class_names").split(",") # train config use_base_model_weights = cp["TRAIN"].getboolean("use_base_model_weights") use_trained_model_weights = cp["TRAIN"].getboolean( "use_trained_model_weights") use_best_weights = cp["TRAIN"].getboolean("use_best_weights") output_weights_name = cp["TRAIN"].get("output_weights_name") epochs = cp["TRAIN"].getint("epochs") batch_size = cp["TRAIN"].getint("batch_size") initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate") train_steps = cp["TRAIN"].get("train_steps") patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr") validation_steps = cp["TRAIN"].get("validation_steps") positive_weights_multiply = cp["TRAIN"].getfloat( "positive_weights_multiply") use_class_balancing = cp["TRAIN"].getboolean("use_class_balancing") use_default_split = cp["TRAIN"].getboolean("use_default_split") # if previously trained weights is used, never re-split if use_trained_model_weights: # resuming mode print( "** use trained model weights, turn on use_skip_split automatically **" ) use_skip_split = True # load training status for resuming training_stats_file = os.path.join(output_dir, ".training_stats.json") if os.path.isfile(training_stats_file): # TODO: add loading previous learning rate? training_stats = json.load(open(training_stats_file)) else: training_stats = {} else: # start over use_skip_split = cp["TRAIN"].getboolean("use_skip_split ") training_stats = {} split_dataset_random_state = cp["TRAIN"].getint( "split_dataset_random_state") show_model_summary = cp["TRAIN"].getboolean("show_model_summary") # end parser config # check output_dir, create it if not exists if not os.path.isdir(output_dir): os.makedirs(output_dir) running_flag_file = os.path.join(output_dir, ".training.lock") if os.path.isfile(running_flag_file): raise RuntimeError("A process is running in this directory!!!") else: open(running_flag_file, "a").close() try: print(f"backup config file to {output_dir}") shutil.copy(config_file, os.path.join(output_dir, os.path.split(config_file)[1])) # split train/dev/test if use_default_split: datasets = ["train", "dev", "test"] for dataset in datasets: shutil.copy(f"./data/default_split/{dataset}.csv", output_dir) elif not use_skip_split: print("** split dataset **") split_data( data_entry_file, class_names, train_patient_count, dev_patient_count, output_dir, split_dataset_random_state, ) # get train/dev sample counts train_counts, train_pos_counts = get_sample_counts( output_dir, "train", class_names) dev_counts, _ = get_sample_counts(output_dir, "dev", class_names) # compute steps if train_steps == "auto": train_steps = int(train_counts / batch_size) else: try: train_steps = int(train_steps) except ValueError: raise ValueError(f""" train_steps: {train_steps} is invalid, please use 'auto' or integer. """) print(f"** train_steps: {train_steps} **") if validation_steps == "auto": validation_steps = int(dev_counts / batch_size) else: try: validation_steps = int(validation_steps) except ValueError: raise ValueError(f""" validation_steps: {validation_steps} is invalid, please use 'auto' or integer. """) print(f"** validation_steps: {validation_steps} **") # compute class weights print("** compute class weights from training data **") class_weights = get_class_weights( train_counts, train_pos_counts, multiply=positive_weights_multiply, use_class_balancing=use_class_balancing) print("** class_weights **") for c, w in class_weights.items(): print(f" {c}: {w}") print("** load model **") if use_base_model_weights: base_model_weights_file = cp["TRAIN"].get( "base_model_weights_file") else: base_model_weights_file = None if use_trained_model_weights: if use_best_weights: model_weights_file = os.path.join( output_dir, f"best_{output_weights_name}") else: model_weights_file = os.path.join(output_dir, output_weights_name) else: model_weights_file = None model = get_model(class_names, base_model_weights_file, model_weights_file) if show_model_summary: print(model.summary()) # recreate symlink folder for ImageDataGenerator symlink_dir_name = "image_links" create_symlink(image_source_dir, output_dir, symlink_dir_name) print("** create image generators **") train_data_path = f"{output_dir}/{symlink_dir_name}/train/" train_generator = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1. / 255), train_data_path, batch_size=batch_size, class_names=class_names, ) dev_data_path = f"{output_dir}/{symlink_dir_name}/dev/" dev_generator = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1. / 255), dev_data_path, batch_size=batch_size, class_names=class_names, ) output_weights_path = os.path.join(output_dir, output_weights_name) print(f"** set output weights path to: {output_weights_path} **") print("** check multiple gpu availability **") gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "1").split(",")) if gpus > 1: print(f"** multi_gpu_model is used! gpus={gpus} **") model_train = multi_gpu_model(model, gpus) # FIXME: currently (Keras 2.1.2) checkpoint doesn't work with multi_gpu_model checkpoint = MultiGPUModelCheckpoint( filepath=output_weights_path, base_model=model, ) else: model_train = model checkpoint = ModelCheckpoint(output_weights_path) print("** compile model with class weights **") optimizer = Adam(lr=initial_learning_rate) model_train.compile(optimizer=optimizer, loss="binary_crossentropy") auroc = MultipleClassAUROC( generator=dev_generator, steps=validation_steps, class_names=class_names, weights_path=output_weights_path, stats=training_stats, ) callbacks = [ checkpoint, TensorBoard(log_dir=os.path.join(output_dir, "logs"), batch_size=batch_size), ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=patience_reduce_lr, verbose=1), auroc, ] print("** training start **") history = model_train.fit_generator( generator=train_generator, steps_per_epoch=train_steps, epochs=epochs, validation_data=dev_generator, validation_steps=validation_steps, callbacks=callbacks, class_weight=class_weights, ) # dump history print("** dump history **") with open(os.path.join(output_dir, "history.pkl"), "wb") as f: pickle.dump({ "history": history.history, "auroc": auroc.aurocs, }, f) print("** done! **") finally: os.remove(running_flag_file)
def main(): # parser config config_file = "./config.ini" cp = ConfigParser() cp.read(config_file) # default config output_dir = cp["DEFAULT"].get("output_dir") class_names = cp["DEFAULT"].get("class_names").split(",") # test config batch_size = cp["TEST"].getint("batch_size") use_best_weights = cp["TEST"].getboolean("use_best_weights") # parse weights file path output_weights_name = cp["TRAIN"].get("output_weights_name") weights_path = os.path.join(output_dir, output_weights_name) best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}") # get test sample count test_counts, _ = get_sample_counts(output_dir, "test", class_names) symlink_dir_name = "image_links" test_data_path = f"{output_dir}/{symlink_dir_name}/test/" step_test = int(test_counts / batch_size) print("** load test generator **") test_generator = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1. / 255), test_data_path, batch_size=batch_size, class_names=class_names, cam=False ) test_generator_orig = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1. / 255), test_data_path, batch_size=batch_size, class_names=class_names, cam=True) x, y = load_generator_data(test_generator, step_test, len(class_names)) x_orig, _ = load_generator_data(test_generator_orig, step_test, len(class_names)) print("** load model **") model = get_model(class_names) if use_best_weights: print("** use best weights **") model.load_weights(best_weights_path) else: print("** use last weights **") model.load_weights(weights_path) print("** make prediction **") y_hat = model.predict(x, verbose=1) test_log_path = os.path.join(output_dir, "test.log") print(f"** write log to {test_log_path} **") aurocs = [] with open(test_log_path, "w") as f: for i in range(len(class_names)): try: score = roc_auc_score(y[i], y_hat[i]) aurocs.append(score) except ValueError: score = 0 f.write(f"{class_names[i]}: {score}\n") mean_auroc = np.mean(aurocs) f.write("-------------------------\n") f.write(f"mean auroc: {mean_auroc}\n") grad_cam(model, class_names, y, y_hat, x, x_orig)
def main(): # parser config config_file = "./config.ini" cp = ConfigParser() cp.read(config_file) # default config output_dir = cp["DEFAULT"].get("output_dir") image_source_dir = cp["DEFAULT"].get("image_source_dir") train_patient_count = cp["DEFAULT"].getint("train_patient_count") dev_patient_count = cp["DEFAULT"].getint("dev_patient_count") data_entry_file = cp["DEFAULT"].get("data_entry_file") class_names = cp["DEFAULT"].get("class_names").split(",") # train config use_base_model_weights = cp["TRAIN"].getboolean("use_base_model_weights") use_trained_model_weights = cp["TRAIN"].getboolean("use_trained_model_weights") use_best_weights = cp["TRAIN"].getboolean("use_best_weights") output_weights_name = cp["TRAIN"].get("output_weights_name") epochs = cp["TRAIN"].getint("epochs") batch_size = cp["TRAIN"].getint("batch_size") initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate") train_steps = cp["TRAIN"].get("train_steps") patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr") validation_steps = cp["TRAIN"].get("validation_steps") positive_weights_multiply = cp["TRAIN"].getfloat("positive_weights_multiply") use_class_balancing = cp["TRAIN"].getboolean("use_class_balancing") use_default_split = cp["TRAIN"].getboolean("use_default_split") # if previously trained weights is used, never re-split if use_trained_model_weights: # resuming mode print("** use trained model weights, turn on use_skip_split automatically **") use_skip_split = True # load training status for resuming training_stats_file = os.path.join(output_dir, ".training_stats.json") if os.path.isfile(training_stats_file): # TODO: add loading previous learning rate? training_stats = json.load(open(training_stats_file)) else: training_stats = {} else: # start over use_skip_split = cp["TRAIN"].getboolean("use_skip_split ") training_stats = {} split_dataset_random_state = cp["TRAIN"].getint("split_dataset_random_state") show_model_summary = cp["TRAIN"].getboolean("show_model_summary") # end parser config # check output_dir, create it if not exists if not os.path.isdir(output_dir): os.makedirs(output_dir) running_flag_file = os.path.join(output_dir, ".training.lock") if os.path.isfile(running_flag_file): raise RuntimeError("A process is running in this directory!!!") else: open(running_flag_file, "a").close() try: print(f"backup config file to {output_dir}") shutil.copy(config_file, os.path.join(output_dir, os.path.split(config_file)[1])) # split train/dev/test if use_default_split: datasets = ["train", "dev", "test"] for dataset in datasets: shutil.copy(f"./data/default_split/{dataset}.csv", output_dir) elif not use_skip_split: print("** split dataset **") split_data( data_entry_file, class_names, train_patient_count, dev_patient_count, output_dir, split_dataset_random_state, ) # get train/dev sample counts train_counts, train_pos_counts = get_sample_counts(output_dir, "train", class_names) dev_counts, _ = get_sample_counts(output_dir, "dev", class_names) # compute steps if train_steps == "auto": train_steps = int(train_counts / batch_size) else: try: train_steps = int(train_steps) except ValueError: raise ValueError(f""" train_steps: {train_steps} is invalid, please use 'auto' or integer. """) print(f"** train_steps: {train_steps} **") if validation_steps == "auto": validation_steps = int(dev_counts / batch_size) else: try: validation_steps = int(validation_steps) except ValueError: raise ValueError(f""" validation_steps: {validation_steps} is invalid, please use 'auto' or integer. """) print(f"** validation_steps: {validation_steps} **") # compute class weights print("** compute class weights from training data **") class_weights = get_class_weights( train_counts, train_pos_counts, multiply=positive_weights_multiply, use_class_balancing=use_class_balancing ) print("** class_weights **") for c, w in class_weights.items(): print(f" {c}: {w}") print("** load model **") if use_base_model_weights: base_model_weights_file = cp["TRAIN"].get("base_model_weights_file") else: base_model_weights_file = None if use_trained_model_weights: if use_best_weights: model_weights_file = os.path.join(output_dir, f"best_{output_weights_name}") else: model_weights_file = os.path.join(output_dir, output_weights_name) else: model_weights_file = None model = get_model(class_names, base_model_weights_file, model_weights_file) if show_model_summary: print(model.summary()) # recreate symlink folder for ImageDataGenerator symlink_dir_name = "image_links" create_symlink(image_source_dir, output_dir, symlink_dir_name) print("** create image generators **") train_data_path = f"{output_dir}/{symlink_dir_name}/train/" train_generator = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1./255), train_data_path, batch_size=batch_size, class_names=class_names, ) dev_data_path = f"{output_dir}/{symlink_dir_name}/dev/" dev_generator = custom_image_generator( ImageDataGenerator(horizontal_flip=True, rescale=1./255), dev_data_path, batch_size=batch_size, class_names=class_names, ) output_weights_path = os.path.join(output_dir, output_weights_name) print(f"** set output weights path to: {output_weights_path} **") print("** check multiple gpu availability **") gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "1").split(",")) if gpus > 1: print(f"** multi_gpu_model is used! gpus={gpus} **") model_train = multi_gpu_model(model, gpus) # FIXME: currently (Keras 2.1.2) checkpoint doesn't work with multi_gpu_model checkpoint = MultiGPUModelCheckpoint( filepath=output_weights_path, base_model=model, ) else: model_train = model checkpoint = ModelCheckpoint(output_weights_path) print("** compile model with class weights **") optimizer = Adam(lr=initial_learning_rate) model_train.compile(optimizer=optimizer, loss="binary_crossentropy") auroc = MultipleClassAUROC( generator=dev_generator, steps=validation_steps, class_names=class_names, weights_path=output_weights_path, stats=training_stats, ) callbacks = [ checkpoint, TensorBoard(log_dir=os.path.join(output_dir, "logs"), batch_size=batch_size), ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=patience_reduce_lr, verbose=1), auroc, ] print("** training start **") history = model_train.fit_generator( generator=train_generator, steps_per_epoch=train_steps, epochs=epochs, validation_data=dev_generator, validation_steps=validation_steps, callbacks=callbacks, class_weight=class_weights, ) # dump history print("** dump history **") with open(os.path.join(output_dir, "history.pkl"), "wb") as f: pickle.dump({ "history": history.history, "auroc": auroc.aurocs, }, f) print("** done! **") finally: os.remove(running_flag_file)