def __init__(self, net, early_stopping: EarlyStopping = None, **l1_regularized_nnr_params): self.dataset = None self.early_stopping = early_stopping self.cpu_count = psutil.cpu_count(logical=True) cp['checkpoint'] = Checkpoint(dirname=tempfile.gettempdir()) if 'train_split' in l1_regularized_nnr_params: train_split = l1_regularized_nnr_params.get('train_split') if train_split is None: cp['checkpoint'].monitor = 'train_loss_best' if early_stopping is not None: if early_stopping.monitor == 'valid_loss': raise ValueError( # pylint: disable=line-too-long 'Select a valid train_split or disable early_stopping! A valid train_split needs to be selected when valid_loss monitor is selected as early stopping criteria.' ) if 'device' in l1_regularized_nnr_params: device = l1_regularized_nnr_params.get('device') else: device = 'cuda' if torch.cuda.is_available() else 'cpu' l1_regularized_nnr_params['device'] = device self.l1_regularized_nnr_params = l1_regularized_nnr_params self.ttr = TransformedTargetRegressor(regressor=self._get_pipeline( net, cp['checkpoint']), transformer=Scaler())
def train(data_folder: str, out_model: str): out_model = Path(out_model) out_model.mkdir() data_paths = list(Path(data_folder).rglob("*.npy")) train_paths, valid_paths = train_test_split(data_paths, train_size=0.7) train_dataset = LibriSpeechDataset( train_paths, Path(data_folder).parent / "SPEAKERS.TXT", Compose([ExtractStft(), RandomCrop(constants.STFT_CROP_WIDTH)])) valid_dataset = LibriSpeechDataset( valid_paths, Path(data_folder).parent / "SPEAKERS.TXT", Compose([ExtractStft(), RandomCrop(constants.STFT_CROP_WIDTH)])) net = NeuralNet(Classifier, module__n_classes=constants.NUMBER_OF_CLASSES, criterion=nn.CrossEntropyLoss, batch_size=8, max_epochs=100, optimizer=optim.Adam, lr=0.001, iterator_train__shuffle=True, iterator_train__num_workers=2, iterator_valid__shuffle=False, iterator_valid__num_workers=2, train_split=predefined_split(valid_dataset), device="cuda", callbacks=[ Checkpoint( f_params=(out_model / "params.pt").as_posix(), f_optimizer=(out_model / "optim.pt").as_posix(), f_history=(out_model / "history.pt").as_posix()), ProgressBar(postfix_keys=["train_loss", "train_acc"]), EarlyStopping(), EpochScoring(acc, name="val_acc", lower_is_better=False, on_train=False), EpochScoring(acc, name="train_acc", lower_is_better=False, on_train=True), Tensorboard((out_model / "train").as_posix(), metrics={"acc": acc_as_metric}, is_training=True), Tensorboard((out_model / "valid").as_posix(), metrics={"acc": acc_as_metric}, is_training=False), ]) net.fit(train_dataset)
def main(args): # Step 1: Fetch Datasets for training/validation/test datasets train_set, val_set, test_set = get_datasets(args) # Step 2: Initialize callbacks for NeuralNetClassifier lrscheduler = LRScheduler( policy='StepLR', step_size=7, gamma=0.1) checkpoint = Checkpoint( f_params='best_model.pt', monitor='valid_acc_best') freezer = Freezer(lambda x: not x.startswith('model.classifier.1')) net = NeuralNetClassifier( TwoClassSqueezeNet, criterion=nn.CrossEntropyLoss, batch_size=args.batch_size, max_epochs=args.num_epochs, module__num_classes=args.num_classes, optimizer=optim.SGD, iterator_train__shuffle=True, iterator_train__num_workers=args.num_workers, iterator_valid__shuffle=True, iterator_valid__num_workers=args.num_workers, # train_split fixes bug in skorch library, see: # https://github.com/skorch-dev/skorch/issues/599 train_split=None, device='cuda' # comment to train on cpu ) params = { 'optimizer__lr': [1e-5, 1e-4, 1e-3], 'optimizer__momentum': [0.5, 0.9, 0.99, 0.999], 'optimizer__nesterov': [True, False] } gs = GridSearchCV(net, params, refit=False, cv=3, scoring='accuracy', verbose=10) X_sl = SliceDataset(train_set, idx=0) # idx=0 is the default y_sl = SliceDataset(train_set, idx=1) # net.fit(train_set, y=None) gs.fit(X_sl, y_sl) print(gs.best_score_, gs.best_params_)
def calorie_model(val_ds): lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1) checkpoint = Checkpoint(f_params='models/calorie_net.pt', monitor='valid_acc_best') return NeuralNet(CalorieNet, criterion=nn.MSELoss(), lr=0.001, batch_size=64, max_epochs=25, optimizer=optim.SGD, optimizer__momentum=0.9, iterator_train__shuffle=True, iterator_train__num_workers=4, iterator_valid__shuffle=True, iterator_valid__num_workers=4, train_split=predefined_split(val_ds), callbacks=[lrscheduler, checkpoint], device='cuda')
def _default_callbacks(self): default_cb_list = [('epoch_timer', EpochTimer()), ('train_loss', BatchScoring(train_loss_score, name='train_loss', on_train=True, target_extractor=noop)), ('valid_loss', BatchScoring(valid_loss_score, name='valid_loss', target_extractor=noop)), ('valid_acc', EpochScoring( 'accuracy', name='valid_acc', lower_is_better=False, )), ('checkpoint', Checkpoint(dirname=self.model_path)), ('end_checkpoint', TrainEndCheckpoint(dirname=self.model_path)), ('report', ReportLog()), ('progressbar', ProgressBar())] if 'stop_patience' in self.hyperparamters.keys() and \ self.hyperparamters['stop_patience']: earlystop_cb = ('earlystop', EarlyStopping( patience=self.hyperparamters['stop_patience'], threshold=1e-4)) default_cb_list.append(earlystop_cb) if 'lr_step' in self.hyperparamters.keys() and \ self.hyperparamters['lr_step']: lr_callback = ('lr_schedule', DecayLR(self.hyperparamters['lr'], self.hyperparamters['lr_step'], gamma=0.5)) default_cb_list.append(lr_callback) return default_cb_list
def prepare_NN(self, num_classes): class ThreeLayerNN(nn.Module): def __init__(self, hidden, num_classes, activation): super(ThreeLayerNN, self).__init__() self.hidden = hidden self.num_classes = num_classes self.activation = activation self.Layer1 = nn.Linear(768, self.hidden, bias=True) self.Layer2 = nn.Linear(self.hidden, self.hidden, bias=True) self.Output = nn.Linear(self.hidden, self.num_classes, bias=True) def forward(self, x): z1 = self.Layer1(x) a1 = self.activation(z1) z2 = self.Layer2(a1) a2 = self.activation(z2) y_hat = F.softmax(a2) return(y_hat) monitor = lambda net: all(net.history[-1, ("train_loss_best", "valid_loss_best")]) net = NeuralNetClassifier(module=ThreeLayerNN, module__hidden=self.hidden, module__num_classes=num_classes, module__activation=self.activation, max_epochs=1000, lr=0.1, iterator_train__shuffle=True, callbacks=[Checkpoint(monitor=monitor), EarlyStopping(patience=5, threshold=0.0001), LRScheduler()], verbose=0, device=self.device, batch_size=32) return(net)
drop1=0.2): super().init() model = torchvision.models.mobilenet_v2(pretrained=False) n_inputs = model.classifier[1].in_features model.classifier = nn.Sequential(nn.Dropout(p=drop1), nn.Linear(n_inputs, output_features)) self.model = model def forward(self, x): return self.model(x) lr_scheduler_mobilenet = LRScheduler(policy='StepLR', step_size=8, gamma=0.2) # callback for saving the best on validation accuracy model checkpoint_mobilenet = Checkpoint( f_params='/content/drive/MyDrive/chess_weights/best_model_mobilenet.pkl', monitor='valid_acc_best') # callback for freezing all layer of the model except the last layer #freezer_vgg = Freezer(lambda x: not x.startswith('model.classifier')) # callback for early stopping early_stopping_mobilenet = EarlyStopping(patience=10) mobilenet = NeuralNetClassifier( # pretrained ResNet50 + custom classifier module=MobileNet, # fine tuning model's inner parameters module__output_features=13, module__num_units=512, module__drop=0.5, module__num_units1=512, module__drop1=0.5, # criterion
#print('Passed Thru VGG', x1) y = self.final_layer(x1) #print(y, 'y') #y_pred=self.log_softmax(y) #print(y_pred, 'y_pred') return y model = Vgg16Module() #print(vgg16) #### Model Callbacks lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1) checkpoint = Checkpoint(dirname='exp', f_params='best_model.pt', monitor='train_loss_best') #### Neural Net Classifier net = NeuralNetClassifier( module=model, criterion=nn.CrossEntropyLoss, lr=args.learning_rate, batch_size=args.batch_size, max_epochs=args.num_epochs, optimizer=optim.SGD, optimizer__momentum=args.momentum, train_split=None, #callbacks=[lrscheduler], device=args.device # comment to train on cpu )
def train_calc(inputs): images, filename, file_dir, Gs, lj, forcesonly, scaling = inputs class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params( "./results/checkpoints/{}_params.pt".format(filename)) cp = Checkpoint( monitor="forces_score_best", fn_prefix="./results/checkpoints/{}_".format(filename), ) if not os.path.exists(file_dir): os.makedirs(file_dir, exist_ok=True) forcetraining = True training_data = AtomsDataset( images, SNN_Gaussian, Gs, forcetraining=forcetraining, label=filename, cores=1, lj_data=None, scaling=scaling, ) unique_atoms = training_data.elements fp_length = training_data.fp_length device = "cpu" torch.set_num_threads(1) net = NeuralNetRegressor( module=FullNN(unique_atoms, [fp_length, 3, 20], device, forcetraining=forcetraining), criterion=CustomMSELoss, criterion__force_coefficient=0.04, optimizer=torch.optim.LBFGS, lr=1e-1, batch_size=len(training_data), max_epochs=200, iterator_train__collate_fn=collate_amp, iterator_train__shuffle=False, iterator_valid__collate_fn=collate_amp, iterator_valid__shuffle=False, device=device, train_split=CVSplit(cv=5, random_state=1), callbacks=[ EpochScoring( forces_score, on_train=False, use_caching=True, target_extractor=target_extractor, ), EpochScoring( energy_score, on_train=False, use_caching=True, target_extractor=target_extractor, ), ], ) calc = AMP(training_data, net, label=filename) calc.train() return [training_data, net, filename]
from braindecode import EEGClassifier # These values we found good for shallow network: lr = 0.0625 * 0.01 weight_decay = 0 # For deep4 they should be: # lr = 1 * 0.01 # weight_decay = 0.5 * 0.001 batch_size = 64 n_epochs = 100 from skorch.callbacks import Checkpoint, EarlyStopping, EpochScoring early_stopping = EarlyStopping(patience=30) cp = Checkpoint(dirname='', f_criterion=None, f_optimizer=None, f_history=None) clf = EEGClassifier( model, # criterion=torch.nn.NLLLoss, criterion=torch.nn.CrossEntropyLoss, optimizer=torch.optim.AdamW, train_split=predefined_split(valid_set), # using valid_set for validation optimizer__lr=lr, optimizer__weight_decay=weight_decay, batch_size=batch_size, callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)),('cp', cp),('patience', early_stopping) ], device=device,
def train_model_adam(dic_param, log_exp_run, wdir, device, tensor_embedding, train_data, test_data, gscv_best_model): # Defining a param distribution for hyperparameter-tuning for model and fit params param_grid = { 'lr': dic_param['alpha_distribution'], 'mode': ["Adam"] # Modes: Adam,SGD } fit_param = { 'patientia': dic_param['sgd_early_stopping_patientia'], 'min_diference': dic_param['sgd_min_difference'], 'checkpoint_path': wdir + "checkpoints/" } checkpoint = Checkpoint(dirname=fit_param['checkpoint_path'], f_params=dic_param['f_params_name'], f_optimizer=dic_param['f_optimizer_name'], f_history=dic_param['f_history_name'], f_criterion=dic_param['f_criterion_name'], monitor=None) load_state = LoadInitState(checkpoint) # Defining skorch-based neural network model = Trainer( module=ModelCNN, module__word_embedding_size=dic_param['word_embedding_size'], module__labels=dic_param['labels'], module__weights_tensor=tensor_embedding, module__batch_size=dic_param['sgd_batch_size'], max_epochs=dic_param['epochs_gs_cv'], iterator_train__shuffle=True, criterion=torch.nn.CrossEntropyLoss, train_split=None, device=device, callbacks=[checkpoint], optimizer=torch.optim.Adam, mode="Adam" # optimizer__weight_decay=dic_param['l2_reg'] #L2 regularization ) # model.initialize() # print(summary(model.module_,torch.zeros((1,1000),dtype=torch.long), show_input=True)) # Defining GridSearch using k-fold cross validation log_exp_run.experiments( "GridSearch using k-fold cross validation with for Adam") start_time = time.time() gs = GridSearchCV(model, param_grid, cv=dic_param['grid_search_cross_val_cv'], verbose=2) if gscv_best_model is None: gs.fit(train_data, fit_param=fit_param) log_exp_run.experiments( "Time elapsed for GridSearch using k-fold cross validation with k=5 for Adam: " + str(time.time() - start_time)) log_exp_run.experiments("Best param estimated for Adam: ") log_exp_run.experiments(gs.best_params_) log_exp_run.experiments("Best score for Adam: ") log_exp_run.experiments(gs.best_score_) log_exp_run.experiments("GridSearch scores") log_exp_run.experiments(gs.cv_results_) gscv_best_model = gs.best_estimator_ best_model = gscv_best_model best_model.set_params(max_epochs=dic_param['epochs']) start_time = time.time() best_model.fit(train_data, fit_param=fit_param) log_exp_run.experiments("Time elapsed for Adam : " + str(time.time() - start_time)) best_model.score(test_data) best_model.score(train_data) log_exp_run.experiments("Adam as optimizer: Process ends successfully!") log_exp_run.experiments("--------------------------\n\n\n") return gscv_best_model
# for i in range(NUM_GPUs): # CUDA_VISIBLE_DEVICES=i dask-worker 127.0.0.1:8786 --nthreads 1 # CUDA_VISIBLE_DEVICES=i dask-worker 127.0.0.1:8786 --nthreads 1 ####################################################################### # CALLBACK FUNCTIONS ####################################################################### ################## skorch_callbacks ################################### # checkpoint saver # monitor = lambda net: all(net.history[-1, ( # 'train_loss_best', 'valid_loss_best')]) cp_best_model = Checkpoint( monitor='valid_loss_best', # monitor = monitor, dirname='results/model_checkpoints/best_model', ) cp_best_train = Checkpoint( monitor='train_loss_best', # monitor = monitor, dirname='results/model_checkpoints/best_train', ) # learning rate scheduler cyclicLR = skorch.callbacks.LRScheduler(policy='CyclicLR', ) # lr = skorch.callbacks.CyclicLR(optimizer = Adam) # lr_scheduler = LRScheduler(policy="StepLR", step_size=7, gamma=0.1) #display progressbar == True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print("Using " + torch.cuda.get_device_name(device)) # Prepare data print("Prepare data") train_feat = get_audio_feature( train_dir, negative_num=100, positive_num=100) test_feat = get_audio_feature( test_dir, test_num=100) train_dataset = AudioDataset( train_feat=train_feat, test_feat=test_feat, negative_num=100, positive_num=100) cp_from_final = Checkpoint( dirname='checkpoints/' + run, fn_prefix='from_train_end_') cp = Checkpoint( dirname='checkpoints/' + run) train_end_cp = TrainEndCheckpoint(dirname='checkpoints/'+run) load_state = LoadInitState(train_end_cp) net = NeuralNetBinaryClassifier( module=AudioNet, criterion=nn.BCEWithLogitsLoss, max_epochs=5000, lr=0.01, optimizer=optim.SGD, optimizer__momentum=0.9, batch_size=160, device=device, callbacks=[
net = Net( # Architecture module=MLP, module__input_dim=input_dim, module__output_dim=output_dim, module__hidden_dim=opt['hidden_layer_dim'], optimizer__weight_decay=opt['l2_weight'], module__dropout=opt['dropout'], device='cuda', # Training max_epochs=opt['max_epochs'], batch_size=opt['batch_size'], callbacks=[ Checkpoint(dirname=save_dir, f_params='params.pt', f_optimizer=None, f_history=None, monitor='valid_loss_best') ], # train_split is validation data train_split=predefined_split(Dataset(X_val, y_val)), # Optimizer optimizer=optim.Adam, lr=opt['learning_rate'], # Data iterator_train__shuffle=True, verbose=(runs == 1)) net.fit(X_train, y_train) # Reload best valid loss checkpoint
binary=False, save_df=True) print("Preprocessing: Preparing for stratified sampling") y_train = np.array([y for _, y in tqdm(iter(ds))]) print("Preprocessing: Done") net = NeuralNetClassifier( module=DPROMModule, module__num_classes=2, module__seqs_length=ds.seqs_length, criterion=torch.nn.CrossEntropyLoss, max_epochs=50, lr=0.001, callbacks=[ EarlyStopping(patience=5), ProgressBar(), Checkpoint(dirname=model_folder, f_params='model.pt') ], batch_size=32, optimizer=torch.optim.Adam, train_split=CVSplit(cv=0.1, stratified=True), device='cuda' if torch.cuda.is_available() else 'cpu') print("Cross Validation: Started") #scoring metrics can be modified. Predefined metrics: https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter scorer = MultiScorer({ 'accuracy': (accuracy_score, {}), 'precision': (precision_score, {}), 'recall': (recall_score, {}), 'mcc': (matthews_corrcoef, {}) }) cross_validate(net, ds, y_train, scoring=scorer, cv=2, verbose=1)
def evaluator( val_split, metric, identifier, forcetraining, ): callbacks = [] isval = val_split != 0 if isval: cp_on = "val" else: cp_on = "train" if metric == "mae": energy_score = mae_energy_score forces_score = mae_forces_score elif metric == "mse": energy_score = mse_energy_score forces_score = mse_forces_score else: raise NotImplementedError(f"{metric} metric not available!") callbacks.append( EpochScoring( energy_score, on_train=True, use_caching=True, name="train_energy_{}".format(metric), target_extractor=target_extractor, )) if isval: callbacks.append( EpochScoring( energy_score, on_train=False, use_caching=True, name="val_energy_{}".format(metric), target_extractor=target_extractor, )) callbacks.append( Checkpoint( monitor="{}_energy_{}_best".format(cp_on, metric), fn_prefix="checkpoints/{}/".format(identifier), )) if forcetraining: callbacks.append( EpochScoring( forces_score, on_train=True, use_caching=True, name="train_forces_{}".format(metric), target_extractor=target_extractor, )) if isval: callbacks.append( EpochScoring( forces_score, on_train=False, use_caching=True, name="val_forces_{}".format(metric), target_extractor=target_extractor, )) callbacks.append( Checkpoint( monitor="{}_forces_{}_best".format(cp_on, metric), fn_prefix="checkpoints/{}/".format(identifier), )) return callbacks
transforms.ToTensor(), normalize ]) train_ds = datasets.ImageFolder(os.path.join(data_dir, 'training'), train_transforms) valid_ds = datasets.ImageFolder(os.path.join(data_dir, 'validation'), val_transforms) model_num = 0 for rate in [0.0001]: for arch in ['wide_resnet50_2']: model_num += 1 checkpoint = Checkpoint(f_params=f'best_model_{model_num}.pt', monitor='valid_acc_best') freezer = Freezer(lambda x: not x.startswith('model.fc')) CNN = NeuralNetClassifier( NewResNet, max_epochs=115, lr=rate, criterion=nn.CrossEntropyLoss, device=device, optimizer=torch.optim.Adam, train_split=predefined_split(valid_ds), batch_size=64, callbacks=[checkpoint, clear_cache, lrscheduler], iterator_train__shuffle=True, iterator_valid__shuffle=True, iterator_train__num_workers=4,
lower_is_better=False, on_train=False, name='tp_valid') calc_fp_train = EpochScoring(calc_fp, lower_is_better=False, on_train=True, name='fp_train') calc_fp_valid = EpochScoring(calc_fp, lower_is_better=False, on_train=False, name='fp_valid') cp = Checkpoint(monitor='precision_valid', f_history=os.path.join( args.output_dir, args.output_filename_prefix + '.json')) train_end_cp = TrainEndCheckpoint(dirname=args.output_dir, fn_prefix=args.output_filename_prefix) lr_scheduler = LRScheduler(policy=ReduceLROnPlateau, mode='max', factor=0.1, patience=3, min_lr=1e-04, verbose=True) # n_cv_folds = int(np.floor(1/args.validation_size)) ## start training fixed_precision = 0.2
def main(): parser = argparse.ArgumentParser( description='PyTorch RNN with variable-length numeric sequences wrapper' ) parser.add_argument('--outcome_col_name', type=str, required=True) parser.add_argument('--train_csv_files', type=str, required=True) parser.add_argument('--valid_csv_files', type=str, required=True) parser.add_argument('--test_csv_files', type=str, required=True) parser.add_argument('--data_dict_files', type=str, required=True) parser.add_argument('--batch_size', type=int, default=1024, help='Number of sequences per minibatch') parser.add_argument('--epochs', type=int, default=50, help='Number of epochs') parser.add_argument('--hidden_units', type=int, default=32, help='Number of hidden units') parser.add_argument('--hidden_layers', type=int, default=1, help='Number of hidden layers') parser.add_argument('--lr', type=float, default=0.0005, help='Learning rate for the optimizer') parser.add_argument('--dropout', type=float, default=0, help='dropout for optimizer') parser.add_argument('--weight_decay', type=float, default=0.0001, help='weight decay for optimizer') parser.add_argument('--seed', type=int, default=1111, help='random seed') parser.add_argument('--validation_size', type=float, default=0.15, help='validation split size') parser.add_argument( '--is_data_simulated', type=bool, default=False, help='boolean to check if data is simulated or from mimic') parser.add_argument( '--output_dir', type=str, default=None, help= 'directory where trained model and loss curves over epochs are saved') parser.add_argument( '--output_filename_prefix', type=str, default=None, help='prefix for the training history jsons and trained classifier') args = parser.parse_args() torch.manual_seed(args.seed) device = 'cpu' x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split( ',') x_valid_csv_filename, y_valid_csv_filename = args.valid_csv_files.split( ',') x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',') x_dict, y_dict = args.data_dict_files.split(',') x_data_dict = load_data_dict_json(x_dict) # get the id and feature columns id_cols = parse_id_cols(x_data_dict) feature_cols = parse_feature_cols(x_data_dict) # extract data train_vitals = TidySequentialDataCSVLoader( x_csv_path=x_train_csv_filename, y_csv_path=y_train_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_tstep') valid_vitals = TidySequentialDataCSVLoader( x_csv_path=x_valid_csv_filename, y_csv_path=y_valid_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_tstep') test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename, y_csv_path=y_test_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_tstep') X_train, y_train = train_vitals.get_batch_data(batch_id=0) X_valid, y_valid = valid_vitals.get_batch_data(batch_id=0) X_test, y_test = test_vitals.get_batch_data(batch_id=0) N, T, F = X_train.shape # from IPython import embed; embed() # X_train = (X_train - np.min(X_train))/(np.max(X_train)-np.min(X_train)) # X_valid = (X_valid - np.min(X_train))/(np.max(X_train)-np.min(X_train)) # X_test = (X_test - np.min(X_train))/(np.max(X_train)-np.min(X_train)) valid_ds = Dataset(X_valid, y_valid) print('number of time points : %s\nnumber of features : %s\n' % (T, F)) # set class weights as 1/(number of samples in class) for each class to handle class imbalance class_weights = torch.tensor( [1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).float() print('Number of training sequences : %s' % N) print('Number of test sequences : %s' % X_test.shape[0]) print('Ratio positive in train : %.2f' % ((y_train == 1).sum() / len(y_train))) print('Ratio positive in test : %.2f' % ((y_test == 1).sum() / len(y_test))) # callback to compute gradient norm compute_grad_norm = ComputeGradientNorm(norm_type=2) # LSTM if args.output_filename_prefix == None: output_filename_prefix = ( 'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' % (args.hidden_units, args.hidden_layers, args.lr, args.dropout, args.weight_decay)) else: output_filename_prefix = args.output_filename_prefix print('RNN parameters : ' + output_filename_prefix) loss_early_stopping_cp = EarlyStopping(monitor='valid_loss', patience=15, threshold=0.002, threshold_mode='rel', lower_is_better=True) rnn = RNNPerTStepBinaryClassifier( max_epochs=250, batch_size=args.batch_size, device=device, lr=args.lr, callbacks=[ EpochScoring(calc_auprc, lower_is_better=False, on_train=True, name='auprc_train'), EpochScoring(calc_auprc, lower_is_better=False, on_train=False, name='auprc_valid'), EpochScoring(calc_auroc, lower_is_better=False, on_train=True, name='auroc_train'), EpochScoring(calc_auroc, lower_is_better=False, on_train=False, name='auroc_valid'), # EpochScoring(calc_precision, lower_is_better=False, on_train=True, name='precision_train'), # EpochScoring(calc_precision, lower_is_better=False, on_train=False, name='precision_valid'), # EpochScoring(calc_recall, lower_is_better=False, on_train=True, name='recall_train'), # EpochScoring(calc_recall, lower_is_better=False, on_train=False, name='recall_valid'), # EpochScoring('roc_auc', lower_is_better=False, on_train=True, name='aucroc_score_train'), # EpochScoring('roc_auc', lower_is_better=False, on_train=False, name='aucroc_score_valid'), # EarlyStopping(monitor='auprc_valid', patience=5, threshold=0.002, threshold_mode='rel', # lower_is_better=False), # LRScheduler(policy=ReduceLROnPlateau, mode='max', monitor='aucroc_score_valid', patience=10), # compute_grad_norm, # GradientNormClipping(gradient_clip_value=0.5, gradient_clip_norm_type=2), loss_early_stopping_cp, Checkpoint(monitor='auprc_valid', f_history=os.path.join( args.output_dir, output_filename_prefix + '.json')), TrainEndCheckpoint(dirname=args.output_dir, fn_prefix=output_filename_prefix), ], # criterion=torch.nn.CrossEntropyLoss, # criterion__weight=class_weights, train_split=predefined_split(valid_ds), module__rnn_type='GRU', module__n_layers=args.hidden_layers, module__n_hiddens=args.hidden_units, module__n_inputs=X_train.shape[-1], module__dropout_proba=args.dropout, optimizer=torch.optim.Adam, optimizer__weight_decay=args.weight_decay) # N=len(X_train) # X_train = X_train[:N] # y_train = y_train[:N] clf = rnn.fit(X_train, y_train) # get threshold with max recall at fixed precision fixed_precision = 0.1 # get predict probas for y=1 on validation set keep_inds_va = torch.logical_not( torch.all(torch.isnan(torch.FloatTensor(X_valid)), dim=-1)) y_va_pred_proba = clf.predict_proba( X_valid)[keep_inds_va][:, 1].detach().numpy() unique_probas = np.unique(y_va_pred_proba) thr_grid_G = np.linspace(np.percentile(unique_probas, 1), max(unique_probas), 100) precision_scores_G, recall_scores_G = [ np.zeros(thr_grid_G.size), np.zeros(thr_grid_G.size) ] for gg, thr in enumerate(thr_grid_G): # logistic_clf.module_.linear_transform_layer.bias.data = torch.tensor(thr_grid[gg]).double() curr_thr_y_preds = clf.predict_proba( torch.FloatTensor(X_valid))[keep_inds_va][:, 1] >= thr_grid_G[gg] precision_scores_G[gg] = precision_score(y_valid[keep_inds_va], curr_thr_y_preds) recall_scores_G[gg] = recall_score(y_valid[keep_inds_va], curr_thr_y_preds) keep_inds = precision_scores_G >= fixed_precision if keep_inds.sum() > 0: print('Choosing threshold with precision >= %.3f' % fixed_precision) else: fixed_precision_old = fixed_precision fixed_precision = np.percentile(precision_scores_G, 99) keep_inds = precision_scores_G >= fixed_precision print( 'Could not find threshold with precision >= %.3f \n Choosing threshold to maximize recall at precision %.3f' % (fixed_precision_old, fixed_precision)) thr_grid_G = thr_grid_G[keep_inds] precision_scores_G = precision_scores_G[keep_inds] recall_scores_G = recall_scores_G[keep_inds] thr_perf_df = pd.DataFrame( np.vstack([ thr_grid_G[np.newaxis, :], precision_scores_G[np.newaxis, :], recall_scores_G[np.newaxis, :] ]).T, columns=['thr', 'precision_score', 'recall_score']) print(thr_perf_df) best_ind = np.argmax(recall_scores_G) best_thr = thr_grid_G[best_ind] print('chosen threshold : %.3f' % best_thr) splits = ['train', 'valid', 'test'] # data_splits = ((x_tr, y_tr), (x_va, y_va), (X_test, y_test)) auroc_per_split, auprc_per_split, precisions_per_split, recalls_per_split = [ np.zeros(len(splits)), np.zeros(len(splits)), np.zeros(len(splits)), np.zeros(len(splits)) ] for ii, (X, y) in enumerate([(X_train, y_train), (X_valid, y_valid), (X_test, y_test)]): keep_inds = torch.logical_not( torch.all(torch.isnan(torch.FloatTensor(X)), dim=-1)) y_pred_proba_pos = clf.predict_proba(X)[keep_inds][:, 1].detach().numpy() # y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba) auroc_per_split[ii] = roc_auc_score(y[keep_inds], y_pred_proba_pos) # y_pred_proba_pos = np.asarray(y_pred_proba_pos) auprc_per_split[ii] = average_precision_score(y[keep_inds], y_pred_proba_pos) y_pred = y_pred_proba_pos >= best_thr precisions_per_split[ii] = precision_score(y[keep_inds], y_pred) recalls_per_split[ii] = recall_score(y[keep_inds], y_pred) auroc_train, auroc_valid, auroc_test = auroc_per_split auprc_train, auprc_valid, auprc_test = auprc_per_split precision_train, precision_valid, precision_test = precisions_per_split recall_train, recall_valid, recall_test = recalls_per_split # save performance perf_dict = { 'auroc_train': auroc_train, 'auroc_valid': auroc_valid, 'auroc_test': auroc_test, 'auprc_train': auprc_train, 'auprc_valid': auprc_valid, 'auprc_test': auprc_test, 'precision_train': precision_train, 'precision_valid': precision_valid, 'precision_test': precision_test, 'recall_train': recall_train, 'recall_valid': recall_valid, 'recall_test': recall_test, 'threshold': best_thr } perf_df = pd.DataFrame([perf_dict]) perf_csv = os.path.join(args.output_dir, output_filename_prefix + '.csv') print('Final performance on train, valid and test :\n') print(perf_df) print('Final performance saved to %s' % perf_csv) perf_df.to_csv(perf_csv, index=False)
from skorch.callbacks import LRScheduler lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1) from skorch.callbacks import Checkpoint from skorch.callbacks import Freezer from skorch.callbacks import EpochScoring #freezer = Freezer(lambda x: not x.startswith('model.fc')) auc = EpochScoring(scoring='roc_auc', lower_is_better=False) f1 = EpochScoring(scoring='f1', lower_is_better=False) for i in range(1): checkpoint = Checkpoint(f_params='./' + str(i + 1) + 'best_model.pt', monitor='valid_acc_best') print("data folder: " + str('./combined_HE')) data = load_data.load('./combined_he') train_ds, valid_ds = data[0], data[1] #data[2] net = NeuralNetClassifier( Inception3, criterion=nn.CrossEntropyLoss, lr=0.002, batch_size=10, max_epochs=40, #module__output_features=2, #module__dropout = 0.5, optimizer=optim.SGD, optimizer__momentum=0.9, iterator_train__shuffle=True, iterator_train__num_workers=4,
def train(data, dilation, kernel_size, lr, patient_index, model_string, correlation_monitor, output_dir, max_train_epochs=300, split=None, cropped=True, padding=False): """ Creates and fits a model with the specified parameters onto the specified data :param data: dataset on which the model is to be trained :param dilation: dilation parameters of the model max-pool layers :param kernel_size: kernel sizes of the model's max-pool layers :param lr: learning rate :param patient_index: index of the patient on whose data the model is trained :param model_string: string specifying the setting of the data :param correlation_monitor: correlation monitor object calculating the correlations while fitting :param output_dir: where the trained model should be saved :param max_train_epochs: number of epochs for which to train the model :param split: the fold from cross-validation for which we are currently trainig the model :param cropped: if the decoding is cropped, alwasy True in thesis experiments :param padding: if padding should be added, always False in thesis experiments :return: """ model, changed_model, model_name = get_model(data.in_channels, input_time_length, dilations=dilation, kernel_sizes=kernel_size, padding=padding) if cuda: device = 'cuda' model.model = changed_model.cuda() else: model.model = changed_model device = 'cpu' if not padding: n_preds_per_input = get_output_shape(model.model, model.input_channels, model.input_time_length)[1] else: n_preds_per_input = 1 Path(home + f'/models/saved_models/{output_dir}/').mkdir(parents=True, exist_ok=True) # cutting the input into batches compatible with model # if data.num_of_folds != -1, then also pre-whitening or filtering takes place # as part of the cut_input method data.cut_input(input_time_length=input_time_length, n_preds_per_input=n_preds_per_input, shuffle=False) print( f'starting cv epoch {split} out of {data.num_of_folds} for model: {model_string}_{model_name}' ) correlation_monitor.step_number = 0 if split is not None: correlation_monitor.split = split monitor = 'validation_correlation_best' monitors = [ ('correlation monitor', correlation_monitor), ('checkpoint', Checkpoint( monitor=monitor, f_history=home + f'/logs/model_{model_name}/histories/{model_string}_k_{model_name}_p_{patient_index}.json', )), ] # cropped=False print('cropped:', cropped) # object EEGRegressor from the braindecode library suited for fitting models for regression tasks regressor = EEGRegressor(cropped=cropped, module=model.model, criterion=model.loss_function, optimizer=model.optimizer, max_epochs=max_train_epochs, verbose=1, train_split=data.cv_split, callbacks=monitors, lr=lr, device=device, batch_size=32).initialize() torch.save( model.model, home + f'/models/saved_models/{output_dir}/initial_{model_string}_{model_name}_p_{patient_index}' ) regressor.max_correlation = -1000 if padding: regressor.fit(data.train_set[0], data.train_set[1]) regressor.fit(np.stack(data.train_set.X), np.stack(data.train_set.y)) # best_model = load_model( # f'/models/saved_models/{output_dir}/best_model_split_0') torch.save(model.model, home + f'/models/saved_models/{output_dir}/last_model_{split}') if cuda: best_corr = get_corr_coef(correlation_monitor.validation_set, model.model.cuda(device=device)) else: best_corr = get_corr_coef(correlation_monitor.validation_set, model.model) print(patient_index, best_corr) return best_corr
import torch from torch.nn import init from ase import Atoms from ase.calculators.emt import EMT from ase.io import read class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params("valid_best_params.pt") LR_schedule = LRScheduler("CosineAnnealingLR", T_max=5) # saves best validation loss cp = Checkpoint(monitor="valid_loss_best", fn_prefix="valid_best_") # loads best validation loss at the end of training load_best_valid_loss = train_end_load_best_valid_loss() distances = np.linspace(2, 5, 10) label = "delta_ml_example" images = [] for l in distances: image = Atoms( "CuCO", [ (-l * np.sin(0.65), l * np.cos(0.65), 0), (0, 0, 0), (l * np.sin(0.65), l * np.cos(0.65), 0), ], )
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params('./histories/%i_valid_best_params.pt' % k) nets = [] # Fold the CV data k_folder = KFold(n_splits=5) for k, (indices_train, _) in enumerate(k_folder.split(sdts_train)): stds_train_ = [sdts_train[index] for index in indices_train] targets_train_ = np.array( [targets_train[index] for index in indices_train]) # Define various callbacks and checkpointers for this network LR_schedule = LRScheduler('MultiStepLR', milestones=[75], gamma=0.1) cp = Checkpoint(monitor='valid_loss_best', fn_prefix='./histories/%i_valid_best_' % k) load_best_valid_loss = train_end_load_best_valid_loss() # Train this fold's network net = NeuralNetRegressor(CrystalGraphConvNet, module__orig_atom_fea_len=orig_atom_fea_len, module__nbr_fea_len=nbr_fea_len, batch_size=214, module__classification=False, lr=0.0056, max_epochs=100, module__atom_fea_len=46, module__h_fea_len=83, module__n_conv=8, module__n_h=4, optimizer=Adam,
print("Progress: {:.2%}".format(cnt / len(settings))) print(setting) # Prepare neural network for training hidden_layer_sizes = tuple(x_train.shape[1] // (i + 1)**2 for i in range(setting['num_hidden_layers'])) # # LR scheduler used with SGD. # schedule = LRScheduler( # policy=lr_scheduler.CosineAnnealingLR, # T_max=epochs * (5 / 4), # lr = 0.04 for max learning with audio_text # eta_min=setting['eta_min'], # ) model_id = start_time.strftime('%Y%m%d%H%M%S') base_checkpoint_dir = "/home/cgn/skorch_checkpoints/" cp = Checkpoint( dirname=base_checkpoint_dir + model_id, f_params="nn.pt", # Name of checkpoint parameters saved. ) # Set-up neural network for training. model = NeuralNetwork( # Skorch NeuralNetClassifer module=NNClassifier, callbacks=[ cp, # ('lr_scheduler', schedule), # Use with SGD optimizer ], lr=setting['initial_lr'], module__input_size=x_train.shape[1], module__hidden_layer_sizes=hidden_layer_sizes, module__dropout=setting['dropout'], module__output_size=output_size, criterion=criterion, optimizer=torch.optim.Adam, # torch.optim.SGD,
MODELS, BATCH_SIZE, SAVE_HIST, SAVE_MODELS, SAVE_OPT, MAX_EPs): if model == "alexnet": # Alexnet requires lower lr lr = 0.001 else: lr = 0.01 print("Training with model ", model) model, input_size = get_pretrained_models(model) dataset_train = Alzheimer_Dataset(X_train, y_train, transform=data_transforms['train']) dataset_val = Alzheimer_Dataset(X_val, y_val, transform=data_transforms['train']) checkpoint = Checkpoint(f_params=save_model, monitor='valid_acc_best', f_optimizer=save_opt, f_history=hist) seed_everything = FixRandomSeed() net = NeuralNetClassifier(model, criterion=nn.CrossEntropyLoss, optimizer=optim.SGD, lr=lr, batch_size=batch_size, max_epochs=ep, optimizer__momentum=0.90, iterator_train__shuffle=True, iterator_train__num_workers=8, iterator_valid__shuffle=True, iterator_valid__num_workers=8, train_split=predefined_split(dataset_val), callbacks=[
net = NeuralNetClassifier( MyModule, max_epochs=100, lr=0.001, batch_size=1024, optimizer=Adam, iterator_train__shuffle=True, iterator_train__num_workers=4, iterator_train__pin_memory=True, train_split=predefined_split(val), callbacks=[LRScheduler(policy=CosineAnnealingLR, T_max=64), EpochScoring(macrof1, use_caching=True, lower_is_better=False), EpochScoring(microf1, use_caching=True, lower_is_better=False), Checkpoint(monitor='macrof1_best', dirname='model')], device='cuda', verbose=1 ) print('start training') _ = net.fit(tra, y=None) # net.initialize() net.load_params(f_params='model/params.pt', f_optimizer='model/optimizer.pt', f_history='model/history.json') # In[ ]: submission = pd.read_csv('../data/submission.csv') ann_prob = np.zeros(len(submission)) ann_labels = []
nn.MaxUnpool2d(self.encoder.pooling_3.size()), nn.Conv2d(4, 4, 3, stride=2), nn.ReLU(True), nn.MaxUnpool2d(self.encoder.pooling_2.size()), nn.Conv2d(16, 4, 3, padding=1), nn.ReLU(True), nn.MaxUnpool2d(self.encoder.pooling_1.size()), nn.Conv2d(3, 16, 4, padding=1) ) def forward(self, x): return self.encoder(x) if __name__ == "__main__": cp = Checkpoint(dirname='segnet_mse_no_sigmoid_sgd_150ep_b8_lr_0.01_30enc/checkpoints') train_end_cp = TrainEndCheckpoint(dirname='segnet_mse_no_sigmoid_sgd_150ep_b8_lr_0.01_30enc/checkpoints') net = NeuralNetRegressor( SegNet, module__encoding_size=30, device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu"), max_epochs=150, batch_size=8, criterion=MSELoss, lr=0.01, iterator_train__shuffle=True, optimizer=torch.optim.SGD, optimizer__momentum=.9, callbacks=[cp, train_end_cp] ) net.initialize()
def test_skorch_delta(): from amptorch.skorch_model import AMP cp = Checkpoint(monitor="valid_loss_best", fn_prefix="valid_best_") distances = np.linspace(2, 5, 10) label = "skorch_example" images = [] energies = [] forces = [] for l in distances: image = Atoms( "CuCO", [ (-l * np.sin(0.65), l * np.cos(0.65), 0), (0, 0, 0), (l * np.sin(0.65), l * np.cos(0.65), 0), ], ) image.set_cell([10, 10, 10]) image.wrap(pbc=True) image.set_calculator(EMT()) images.append(image) energies.append(image.get_potential_energy()) forces.append(image.get_forces()) image = Atoms("CuC", [(-1, 1, 0), (1, 1, 0)]) image.set_cell([10, 10, 10]) image.wrap(pbc=True) image.set_calculator(EMT()) images.append(image) energies.append(image.get_potential_energy()) forces.append(image.get_forces()) energies = np.array(energies) forces = np.concatenate(np.array(forces)) Gs = {} Gs["G2_etas"] = np.logspace(np.log10(0.05), np.log10(5.0), num=2) Gs["G2_rs_s"] = [0] * 2 Gs["G4_etas"] = [0.005] Gs["G4_zetas"] = [1.0] Gs["G4_gammas"] = [+1.0, -1] Gs["cutoff"] = 6.5 params = { "C": {"re": 0.972, "D": 6.379, "sig": 0.477}, "O": {"re": 1.09, "D": 8.575, "sig": 0.603}, "Cu": {"re": 2.168, "D": 3.8386, "sig": 1.696}, } morse_model = morse_potential(images, params, Gs["cutoff"], label) morse_energies, morse_forces, num_atoms = morse_model.morse_pred( images, params) morse_data = [morse_energies, morse_forces, num_atoms, params, morse_model] forcetraining = True training_data = AtomsDataset( images, SNN_Gaussian, Gs, forcetraining=forcetraining, label=label, cores=1, delta_data=morse_data, ) batch_size = len(training_data) unique_atoms = training_data.elements fp_length = training_data.fp_length device = "cpu" net = NeuralNetRegressor( module=FullNN( unique_atoms, [fp_length, 2, 2], device, forcetraining=forcetraining ), criterion=CustomMSELoss, criterion__force_coefficient=0.3, optimizer=torch.optim.LBFGS, optimizer__line_search_fn="strong_wolfe", lr=1e-2, batch_size=batch_size, max_epochs=100, iterator_train__collate_fn=collate_amp, iterator_train__shuffle=False, iterator_valid__collate_fn=collate_amp, device=device, train_split=0, verbose=0, callbacks=[ EpochScoring( forces_score, on_train=True, use_caching=True, target_extractor=target_extractor, ), EpochScoring( energy_score, on_train=True, use_caching=True, target_extractor=target_extractor, ), ], ) calc = AMP(training_data, net, "test") calc.train(overwrite=True) num_of_atoms = 3 calculated_energies = np.array( [calc.get_potential_energy(image) for idx, image in enumerate(images)] ) energy_rmse = np.sqrt( (((calculated_energies - energies) / num_of_atoms) ** 2).sum() / len(images) ) last_energy_score = net.history[-1]["energy_score"] assert round(energy_rmse, 4) == round( last_energy_score, 4 ), "Energy errors incorrect!" last_forces_score = net.history[-1]["forces_score"] calculated_forces = np.concatenate( np.array([calc.get_forces(image) for image in images]) ) force_rmse = np.sqrt( (((calculated_forces - forces)) ** 2).sum() / (3 * num_of_atoms * len(images)) ) assert round(force_rmse, 4) == round( last_forces_score, 4 ), "Force errors incorrect!"
def main(): parser = argparse.ArgumentParser( description='PyTorch RNN with variable-length numeric sequences wrapper' ) parser.add_argument('--outcome_col_name', type=str, required=True) parser.add_argument('--train_csv_files', type=str, required=True) parser.add_argument('--test_csv_files', type=str, required=True) parser.add_argument('--data_dict_files', type=str, required=True) parser.add_argument('--batch_size', type=int, default=1024, help='Number of sequences per minibatch') parser.add_argument('--epochs', type=int, default=50, help='Number of epochs') parser.add_argument('--hidden_units', type=int, default=32, help='Number of hidden units') parser.add_argument('--hidden_layers', type=int, default=1, help='Number of hidden layers') parser.add_argument('--lr', type=float, default=0.0005, help='Learning rate for the optimizer') parser.add_argument('--dropout', type=float, default=0, help='dropout for optimizer') parser.add_argument('--weight_decay', type=float, default=0.0001, help='weight decay for optimizer') parser.add_argument('--seed', type=int, default=1111, help='random seed') parser.add_argument('--validation_size', type=float, default=0.15, help='validation split size') parser.add_argument( '--is_data_simulated', type=bool, default=False, help='boolean to check if data is simulated or from mimic') parser.add_argument( '--simulated_data_dir', type=str, default='simulated_data/2-state/', help= 'dir in which to simulated data is saved.Must be provide if is_data_simulated = True' ) parser.add_argument( '--output_dir', type=str, default=None, help= 'directory where trained model and loss curves over epochs are saved') parser.add_argument( '--output_filename_prefix', type=str, default=None, help='prefix for the training history jsons and trained classifier') args = parser.parse_args() torch.manual_seed(args.seed) device = 'cpu' x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split( ',') x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',') x_dict, y_dict = args.data_dict_files.split(',') x_data_dict = load_data_dict_json(x_dict) # get the id and feature columns id_cols = parse_id_cols(x_data_dict) feature_cols = parse_feature_cols(x_data_dict) # extract data train_vitals = TidySequentialDataCSVLoader( x_csv_path=x_train_csv_filename, y_csv_path=y_train_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_sequence') test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename, y_csv_path=y_test_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_sequence') X_train, y_train = train_vitals.get_batch_data(batch_id=0) X_test, y_test = test_vitals.get_batch_data(batch_id=0) _, T, F = X_train.shape print('number of time points : %s\n number of features : %s\n' % (T, F)) # set class weights as 1/(number of samples in class) for each class to handle class imbalance class_weights = torch.tensor( [1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).double() # scale features # X_train = standard_scaler_3d(X_train) # X_test = standard_scaler_3d(X_test) # callback to compute gradient norm compute_grad_norm = ComputeGradientNorm(norm_type=2) # LSTM if args.output_filename_prefix == None: output_filename_prefix = ( 'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' % (args.hidden_units, args.hidden_layers, args.lr, args.dropout, args.weight_decay)) else: output_filename_prefix = args.output_filename_prefix print('RNN parameters : ' + output_filename_prefix) # # from IPython import embed; embed() rnn = RNNBinaryClassifier( max_epochs=50, batch_size=args.batch_size, device=device, lr=args.lr, callbacks=[ EpochScoring('roc_auc', lower_is_better=False, on_train=True, name='aucroc_score_train'), EpochScoring('roc_auc', lower_is_better=False, on_train=False, name='aucroc_score_valid'), EarlyStopping(monitor='aucroc_score_valid', patience=20, threshold=0.002, threshold_mode='rel', lower_is_better=False), LRScheduler(policy=ReduceLROnPlateau, mode='max', monitor='aucroc_score_valid', patience=10), compute_grad_norm, GradientNormClipping(gradient_clip_value=0.3, gradient_clip_norm_type=2), Checkpoint(monitor='aucroc_score_valid', f_history=os.path.join( args.output_dir, output_filename_prefix + '.json')), TrainEndCheckpoint(dirname=args.output_dir, fn_prefix=output_filename_prefix), ], criterion=torch.nn.CrossEntropyLoss, criterion__weight=class_weights, train_split=skorch.dataset.CVSplit(args.validation_size), module__rnn_type='LSTM', module__n_layers=args.hidden_layers, module__n_hiddens=args.hidden_units, module__n_inputs=X_train.shape[-1], module__dropout_proba=args.dropout, optimizer=torch.optim.Adam, optimizer__weight_decay=args.weight_decay) clf = rnn.fit(X_train, y_train) y_pred_proba = clf.predict_proba(X_train) y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba) auroc_train_final = roc_auc_score(y_train, y_pred_proba_pos) print('AUROC with LSTM (Train) : %.2f' % auroc_train_final) y_pred_proba = clf.predict_proba(X_test) y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba) auroc_test_final = roc_auc_score(y_test, y_pred_proba_pos) print('AUROC with LSTM (Test) : %.2f' % auroc_test_final)
from torch.nn import init from skorch.utils import to_numpy import numpy as np from ase import Atoms from ase.calculators.emt import EMT from ase.io import read class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params('valid_best_params.pt') LR_schedule = LRScheduler('CosineAnnealingLR', T_max=5) # saves best validation loss cp = Checkpoint(monitor='valid_loss_best', fn_prefix='valid_best_') # loads best validation loss at the end of training load_best_valid_loss = train_end_load_best_valid_loss() distances = np.linspace(2, 5, 100) label = "skorch_example" images = [] for l in distances: image = Atoms( "CuCO", [ (-l * np.sin(0.65), l * np.cos(0.65), 0), (0, 0, 0), (l * np.sin(0.65), l * np.cos(0.65), 0), ], )