def test_model_checkpoint(save_best_only, max_save, n_files): wide = Wide(100, 1) deepdense = DeepDense( hidden_layers=[32, 16], dropout=[0.5, 0.5], deep_column_idx=deep_column_idx, embed_input=embed_input, continuous_cols=colnames[-5:], ) model = WideDeep(wide=wide, deepdense=deepdense) model.compile( method="binary", callbacks=[ ModelCheckpoint("weights/test_weights", save_best_only=save_best_only, max_save=max_save) ], verbose=0, ) model.fit(X_wide=X_wide, X_deep=X_deep, target=target, n_epochs=5, val_split=0.2) n_saved = len(os.listdir("weights/")) for f in os.listdir("weights/"): os.remove("weights/" + f) assert n_saved <= n_files
def test_model_checkpoint(save_best_only, max_save, n_files): wide = Wide(np.unique(X_wide).shape[0], 1) deeptabular = TabMlp( mlp_hidden_dims=[32, 16], mlp_dropout=[0.5, 0.5], column_idx=column_idx, embed_input=embed_input, continuous_cols=colnames[-5:], ) model = WideDeep(wide=wide, deeptabular=deeptabular) trainer = Trainer( model=model, objective="binary", callbacks=[ ModelCheckpoint( "tests/test_model_functioning/weights/test_weights", save_best_only=save_best_only, max_save=max_save, ) ], verbose=0, ) trainer.fit(X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, val_split=0.2) n_saved = len(os.listdir("tests/test_model_functioning/weights/")) shutil.rmtree("tests/test_model_functioning/weights/") assert n_saved <= n_files
def test_filepath_error(): wide = Wide(np.unique(X_wide).shape[0], 1) deepdense = DeepDense( hidden_layers=[16, 4], deep_column_idx=deep_column_idx, embed_input=embed_input, continuous_cols=colnames[-5:], ) model = WideDeep(wide=wide, deepdense=deepdense) with pytest.raises(ValueError): model.compile( method="binary", callbacks=[ModelCheckpoint(filepath="wrong_file_path")], verbose=0, )
def test_filepath_error(): wide = Wide(np.unique(X_wide).shape[0], 1) deeptabular = TabMlp( mlp_hidden_dims=[16, 4], column_idx=column_idx, embed_input=embed_input, continuous_cols=colnames[-5:], ) model = WideDeep(wide=wide, deeptabular=deeptabular) with pytest.raises(ValueError): trainer = Trainer( # noqa: F841 model=model, objective="binary", callbacks=[ModelCheckpoint(filepath="wrong_file_path")], verbose=0, )
"deeptabular": deep_sch, "deeptext": text_sch, "deepimage": img_sch, } initializers = { "wide": KaimingNormal, "deeptabular": KaimingNormal, "deeptext": KaimingNormal, "deepimage": KaimingNormal, } mean = [0.406, 0.456, 0.485] # BGR std = [0.225, 0.224, 0.229] # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [ EarlyStopping, ModelCheckpoint(filepath="model_weights/wd_out.pt") ] trainer = wd.Trainer( model, objective="regression", initializers=initializers, optimizers=optimizers, lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms, ) trainer.fit( X_wide=X_wide, X_tab=X_tab,
model = WideDeep(deeptabular=deeptabular) optimizers = set_optimizer(model, args) steps_per_epoch = (X_train.shape[0] // args.batch_size) + 1 lr_schedulers = set_lr_scheduler(optimizers, steps_per_epoch, args) early_stopping = EarlyStopping( monitor=args.monitor, min_delta=args.early_stop_delta, patience=args.early_stop_patience, ) model_checkpoint = ModelCheckpoint( filepath=str(MODELS_DIR / "best_model"), monitor=args.monitor, save_best_only=True, max_save=1, ) trainer = Trainer( model, objective="binary", optimizers=optimizers, lr_schedulers=lr_schedulers, reducelronplateau_criterion=args.monitor.split("_")[-1], callbacks=[early_stopping, model_checkpoint, LRHistory(n_epochs=args.n_epochs)], metrics=[Accuracy], ) start = time() trainer.fit(
def run_experiment_and_save( model, model_name, results_dir, models_dir, args, X_train, X_test, y_train, y_test, ): optimizers = set_optimizer(model, args) steps_per_epoch = (X_train.shape[0] // args.batch_size) + 1 lr_schedulers = set_lr_scheduler(optimizers, steps_per_epoch, args) early_stopping = EarlyStopping( monitor=args.monitor, min_delta=args.early_stop_delta, patience=args.early_stop_patience, ) model_checkpoint = ModelCheckpoint( filepath=str(models_dir / "best_model"), monitor=args.monitor, save_best_only=True, max_save=1, ) trainer = Trainer( model, objective="regression", optimizers=optimizers, lr_schedulers=lr_schedulers, reducelronplateau_criterion=args.monitor.split("_")[-1], callbacks=[ early_stopping, model_checkpoint, LRHistory(n_epochs=args.n_epochs) ], ) start = time() trainer.fit( X_train={ "X_tab": X_train, "target": y_train }, X_val={ "X_tab": X_test, "target": y_test }, n_epochs=args.n_epochs, batch_size=args.batch_size, validation_freq=args.eval_every, ) runtime = time() - start y_pred = trainer.predict(X_tab=X_test) rmse = np.sqrt(mean_squared_error(y_test, y_pred)) r2 = r2_score(y_test, y_pred) print(f"rmse with the best model: {rmse}") if args.save_results: suffix = str(datetime.now()).replace(" ", "_").split(".")[:-1][0] filename = "_".join(["fb_comments", model_name, "best", suffix]) + ".p" results_d = {} results_d["args"] = args results_d["rmse"] = rmse results_d["r2"] = r2 results_d["early_stopping"] = early_stopping results_d["trainer_history"] = trainer.history results_d["trainer_lr_history"] = trainer.lr_history results_d["runtime"] = runtime with open(results_dir / filename, "wb") as f: pickle.dump(results_d, f)
def run_experiment_and_save( model, model_name, results_dir, models_dir, args, X_train, X_test, y_train, y_test, fl_exp_indx: int = 0, ): try: if args.focal_loss: alpha, gamma = load_focal_loss_params(results_dir, fl_exp_indx) focal_loss = True else: alpha = 0.25 gamma = 2 focal_loss = False except AttributeError: alpha = 0.25 gamma = 2 focal_loss = False optimizers = set_optimizer(model, args) steps_per_epoch = (X_train.shape[0] // args.batch_size) + 1 lr_schedulers = set_lr_scheduler(optimizers, steps_per_epoch, args) early_stopping = EarlyStopping( monitor=args.monitor, min_delta=args.early_stop_delta, patience=args.early_stop_patience, ) model_checkpoint = ModelCheckpoint( filepath=str(models_dir / "best_model"), monitor=args.monitor, save_best_only=True, max_save=1, ) trainer = Trainer( model, objective="binary_focal_loss" if focal_loss else "binary", optimizers=optimizers, lr_schedulers=lr_schedulers, reducelronplateau_criterion=args.monitor.split("_")[-1], callbacks=[early_stopping, model_checkpoint, LRHistory(n_epochs=args.n_epochs)], metrics=[Accuracy, F1Score], alpha=alpha, gamma=gamma, ) start = time() trainer.fit( X_train={"X_tab": X_train, "target": y_train}, X_val={"X_tab": X_test, "target": y_test}, n_epochs=args.n_epochs, batch_size=args.batch_size, validation_freq=args.eval_every, ) runtime = time() - start y_pred = trainer.predict(X_tab=X_test) acc = accuracy_score(y_test, y_pred) auc = roc_auc_score(y_test, y_pred) f1 = f1_score(y_test, y_pred) print(f"Accuracy: {acc}. F1: {f1}. ROC_AUC: {auc}") print(confusion_matrix(y_test, y_pred)) if args.save_results: suffix = str(datetime.now()).replace(" ", "_").split(".")[:-1][0] filename = "_".join(["bankm", model_name, "best", suffix]) + ".p" results_d = {} results_d["args"] = args results_d["acc"] = acc results_d["auc"] = auc results_d["f1"] = f1 results_d["early_stopping"] = early_stopping results_d["trainer_history"] = trainer.history results_d["trainer_lr_history"] = trainer.lr_history results_d["runtime"] = runtime with open(results_dir / filename, "wb") as f: pickle.dump(results_d, f)
schedulers = { "wide": wide_sch, "deepdense": deep_sch, "deeptext": text_sch, "deepimage": img_sch, } initializers = { "wide": KaimingNormal, "deepdense": KaimingNormal, "deeptext": KaimingNormal, "deepimage": KaimingNormal, } mean = [0.406, 0.456, 0.485] # BGR std = [0.225, 0.224, 0.229] # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [EarlyStopping, ModelCheckpoint(filepath="model_weights/wd_out.pt")] model.compile( method="regression", initializers=initializers, optimizers=optimizers, lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms, ) model.fit( X_wide=X_wide, X_deep=X_deep, X_text=X_text, X_img=X_images,