def train(_run, layer_name, lr, mu, lambda_bar, batch_size, device, max_epochs, epochs_update, alphaupdate, optimizer): module = get_module() X, y = get_dataset() alphaupdate_kwags = flatten_dict(alphaupdate, preffix='callbacks__AlphaUpdate') checkpoint_tmpfile = '/tmp/weights.pt' callbacks = [LRScheduler(policy='StepLR', gamma=0.1, step_size=200)] net = LDMnet(module, criterion=torch.nn.CrossEntropyLoss, layer_name=layer_name, mu=mu, lambda_bar=lambda_bar, lr=lr, epochs_update=epochs_update, max_epochs=max_epochs, batch_size=batch_size, device=device, callbacks=callbacks, **alphaupdate_kwags, **flatten_dict(optimizer, 'optimizer')) net.fit(X, y=y) net.save_params(checkpoint_tmpfile) ex.add_artifact(checkpoint_tmpfile)
def __init__(self, max_epochs=100, lr=0.0001, optimizer=torch.optim.SGD, optimizer__momentum = 0.999, optimizer__weight_decay = 1e-4, optimizer__nesterov = True, module__dropout_depth = True, module__dropout_width = True, module__dropout_p = 0.5, **kwds): args = [ResNetUNet, torch.nn.CrossEntropyLoss] kwds["module__nInputFeatures"] = self.number_of_features kwds["module__nClasses"] = self.number_of_classes kwds["criterion__weight"] = true_distribution kwds["iterator_train"] = MMDBDataset kwds["iterator_train_collate_fn"] = sparse_collate kwds["iterator_valid"] = MMBDDataset kwds["iterator_valid_collate_fn"] = sparse_collate scheduler = LRScheduler(policy="LambdaLR", lr_lambda=lambda epoch: math.exp((1 - epoch) * lr_decay)) if "callbacks" in kwds and isinstance(kwds["callbacks"], list): callbacks.append(scheduler) else: kwds["callbacks"] = [scheduler] super().__init__(*args, **kwargs)
def load_extras(self): callbacks = [] load_best_loss = train_end_load_best_loss(self.identifier) self.split = CVSplit(cv=self.val_split) if self.val_split != 0 else 0 metrics = evaluator( self.val_split, self.config["optim"].get("metric", "mae"), self.identifier, self.forcetraining, ) callbacks.extend(metrics) if not self.debug: callbacks.append(load_best_loss) scheduler = self.config["optim"].get("scheduler", None) if scheduler: scheduler = LRScheduler(scheduler, **self.config["optim"]["scheduler_params"]) callbacks.append(scheduler) if self.config["cmd"].get("logger", False): from skorch.callbacks import WandbLogger callbacks.append( WandbLogger( self.wandb_run, save_model=False, keys_ignored="dur", )) self.callbacks = callbacks
def build_estimator(hyperparams, train_data, test=False): device = 'cuda' if torch.cuda.is_available() else 'cpu' # Extract info from training data X, y, *_ = train_data in_features = X.shape[1] n_classes = len(np.unique(y)) n_samples = y.shape[0] bal_weights = torch.from_numpy( n_samples / (n_classes * np.bincount(y))).float().to(device) callbacks = [ ('f1_score_valid', EpochScoring('f1' if n_classes == 2 else 'f1_macro', name='valid_f1', lower_is_better=False)), ('early_stopping', EarlyStopping(monitor='valid_loss', patience=5, lower_is_better=True)), ( 'learning_rate_scheduler', LRScheduler( policy=lr_scheduler.ReduceLROnPlateau, monitor='valid_loss', # Following kargs are passed to the # lr scheduler constructor mode='min', min_lr=1e-5)), ] return NeuralNetClassifier( NNModule, criterion=nn.CrossEntropyLoss, optimizer=torch.optim.SGD, max_epochs=300, iterator_train__shuffle=True, # Shuffle training data on each epoch callbacks=callbacks, device=device, train_split=CVSplit(cv=5, stratified=True, random_state=RANDOM_STATE), lr=hyperparams['lr'], batch_size=hyperparams['batch_size'], module__in_features=in_features, module__n_classes=n_classes, module__n_layers=hyperparams['n_layers'], module__n_neuron_per_layer=hyperparams['n_neuron_per_layer'], module__activation=getattr(F, hyperparams['activation']), module__p_dropout=hyperparams['p_dropout'], criterion__weight=bal_weights if hyperparams['class_weight'] == 'balanced' else None, optimizer__momentum=hyperparams['momentum'], optimizer__weight_decay=hyperparams['weight_decay'], optimizer__nesterov=True, verbose=3, iterator_train__num_workers=4, iterator_valid__num_workers=4)
def build_estimator(cls, hyperparams, train_data, verbose=True, test=False): # change default verbose to false later early_stopping_val_percent = 10 n_training_examples = len( train_data[0]) * (1 - (early_stopping_val_percent / 100)) n_iter_per_epoch = n_training_examples / hyperparams['batch_size'] n_iter_btw_restarts = int(hyperparams['epochs_btw_restarts'] * n_iter_per_epoch) callbacks = [ ('fix_seed', cls.FixRandomSeed(RANDOM_STATE)), ('lr_monitor', cls.LRMonitor()), ('accuracy_score_valid', EpochScoring('accuracy', lower_is_better=False, on_train=True)), ('early_stopping', EarlyStopping(monitor='valid_acc', lower_is_better=False, patience=100)), ('learning_rate_scheduler', LRScheduler(policy=cls.SkorchCosineAnnealingWarmRestarts, T_0=n_iter_btw_restarts, T_mult=hyperparams['epochs_btw_restarts_mult'])) ] def validation_split(X, y): """ Custom split is used to apply augmentation to the training set only """ splitter = CVSplit(cv=int(100 / early_stopping_val_percent), random_state=RANDOM_STATE) dataset_train, dataset_valid = splitter(X) dataset_train = cls.AugmentedDataset(dataset_train) return dataset_train, dataset_valid return NeuralNetClassifier( cls.CifarCustomNet, criterion=nn.CrossEntropyLoss, optimizer=torch.optim.SGD, max_epochs=hyperparams['max_epochs'] if not test else 1, iterator_train__shuffle=True, iterator_train__num_workers=4, iterator_valid__num_workers=4, dataset=cls.NormalizedDataset, callbacks=callbacks, device=cls.device, train_split=validation_split, lr=hyperparams['learning_rate'], batch_size=hyperparams['batch_size'], optimizer__momentum=hyperparams['momentum'], optimizer__weight_decay=hyperparams['weight_decay'], optimizer__nesterov=hyperparams['nesterov'], module__conv_dropout=hyperparams['conv_dropout'], module__fc_dropout=hyperparams['fc_dropout'], verbose=3 if verbose else 0)
def get_deep_learning_model(model_args, valid_dataset): cuda = torch.cuda.is_available() device = model_args["device"] if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = model_args["seed"] # = 20200220 random seed to make results reproducible # Set random seed to be able to reproduce results if seed: set_random_seeds(seed=seed, cuda=cuda) if model_args["model_type"] == "ShallowFBCSPNet": model = ShallowFBCSPNet( model_args["n_chans"], model_args["n_classes"] + 1, input_window_samples=model_args["input_window_samples"], final_conv_length='auto', ) elif model_args["model_type"] == "SleepStager": model = model = SleepStager( n_channels=model_args["n_chans"], sfreq=model_args["sfreq"], n_classes=model_args["n_classes"] + 1, input_size_s=model_args["input_window_samples"] / model_args["sfreq"], ) else: raise ValueError("Boom !") if cuda: model.cuda() clf = EEGClassifier( model, criterion=model_args["criterion"], optimizer=torch.optim.AdamW, # using test_sample for validation train_split=predefined_split(valid_dataset), optimizer__lr=model_args["lr"], optimizer__weight_decay=model_args["weight_decay"], batch_size=model_args["batch_size"], callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=model_args["n_epochs"] - 1)), ("early_stopping", EarlyStopping(monitor='valid_loss', patience=model_args["patience"])) ], device=device, iterator_train__num_workers=20, iterator_train__pin_memory=True) # torch.in torch.out return clf
def build_model( self, network=MVRegressor, device: str = "cpu", scale_data: bool = False, num_layers: int = 10, num_units: int = 50, dropout: float = 0.5, num_epochs: int = 10, batch_size: int = 128, ): self.scale_data = scale_data self.num_layers = num_layers self.num_units = num_units self.dropout = dropout self.num_epochs = num_epochs self.batch_size = batch_size if not all([hasattr(self, "input_dim"), hasattr(self, "output_dim")]): raise ValueError( "Please load dataset first to obtain proper sizes") if device == "cpu": self.device = device else: use_cuda = torch.cuda.is_available() self.device = torch.device("cuda" if use_cuda else "cpu") self.model = NeuralNetRegressor( network, device=self.device, module__input_dim=self.input_dim, module__output_dim=self.output_dim, module__n_layers=self.num_layers, module__num_units=self.num_units, module__p_dropout=self.dropout, max_epochs=self.num_epochs, criterion=nn.MSELoss, batch_size=self.batch_size, # Shuffle training data on each epoch iterator_train__shuffle=True, callbacks=[ ( "lr_scheduler", LRScheduler(policy=CyclicLR, base_lr=0.001, max_lr=0.01, step_every="batch"), ), ], )
def main(): sampling_rate = 360 wavelet = "mexh" # mexh, morl, gaus8, gaus4 scales = pywt.central_frequency(wavelet) * sampling_rate / np.arange( 1, 101, 1) (x1_train, x2_train, y_train, groups_train), (x1_test, x2_test, y_test, groups_test) = load_data(wavelet=wavelet, scales=scales, sampling_rate=sampling_rate) print("Data loaded successfully!") log_dir = "./logs/{}".format(wavelet) shutil.rmtree(log_dir, ignore_errors=True) callbacks = [ Initializer("[conv|fc]*.weight", fn=torch.nn.init.kaiming_normal_), Initializer("[conv|fc]*.bias", fn=partial(torch.nn.init.constant_, val=0.0)), LRScheduler(policy=StepLR, step_size=5, gamma=0.1), EpochScoring(scoring=make_scorer(f1_score, average="macro"), lower_is_better=False, name="valid_f1"), TensorBoard(SummaryWriter(log_dir)) ] net = NeuralNetClassifier( # skorch is extensive package of pytorch for compatible with scikit-learn MyModule, criterion=torch.nn.CrossEntropyLoss, optimizer=torch.optim.Adam, lr=0.001, max_epochs=30, batch_size=1024, train_split=predefined_split( Dataset({ "x1": x1_test, "x2": x2_test }, y_test)), verbose=1, device="cuda", callbacks=callbacks, iterator_train__shuffle=True, optimizer__weight_decay=0, ) net.fit({"x1": x1_train, "x2": x2_train}, y_train) y_true, y_pred = y_test, net.predict({"x1": x1_test, "x2": x2_test}) print(confusion_matrix(y_true, y_pred)) print(classification_report(y_true, y_pred, digits=4)) net.save_params(f_params="./models/model_{}.pkl".format(wavelet))
def build_estimator(hyperparams, train_data, test=False): device = "cuda" if torch.cuda.is_available() else "cpu" # Extract info from training data X, y, *_ = train_data in_features = X.shape[1] callbacks = [ ("r2_score_valid", EpochScoring("r2", lower_is_better=False)), ( "early_stopping", EarlyStopping(monitor="valid_loss", patience=5, lower_is_better=True), ), ( "learning_rate_scheduler", LRScheduler( policy=lr_scheduler.ReduceLROnPlateau, monitor="valid_loss", # Following kargs are passed to the # lr scheduler constructor mode="min", min_lr=1e-5, ), ), ] return NeuralNetRegressor( NNModule, criterion=nn.MSELoss, optimizer=torch.optim.SGD, max_epochs=300, iterator_train__shuffle=True, # Shuffle training data on each epoch callbacks=callbacks, device=device, train_split=CVSplit(cv=5, random_state=RANDOM_STATE), lr=hyperparams["lr"], batch_size=hyperparams["batch_size"], module__in_features=in_features, module__n_layers=hyperparams["n_layers"], module__n_neuron_per_layer=hyperparams["n_neuron_per_layer"], module__activation=getattr(F, hyperparams["activation"]), module__p_dropout=hyperparams["p_dropout"], optimizer__momentum=hyperparams["momentum"], optimizer__weight_decay=hyperparams["weight_decay"], optimizer__nesterov=True, verbose=3, iterator_train__num_workers=4, iterator_valid__num_workers=4, )
def build_estimator(hyperparams, train_data, test=False): device = 'cuda' if torch.cuda.is_available() else 'cpu' # Extract info from training data X, y, *_ = train_data in_features = X.shape[1] callbacks = [ ('r2_score_valid', EpochScoring('r2', lower_is_better=False)), ('early_stopping', EarlyStopping(monitor='valid_loss', patience=5, lower_is_better=True)), ('learning_rate_scheduler', LRScheduler(policy=lr_scheduler.ReduceLROnPlateau, monitor='valid_loss', # Following kargs are passed to the # lr scheduler constructor mode='min', min_lr=1e-5 )), ] return NeuralNetRegressor( NNModule, criterion=nn.MSELoss, optimizer=torch.optim.SGD, max_epochs=300, iterator_train__shuffle=True, # Shuffle training data on each epoch callbacks=callbacks, device=device, train_split=CVSplit(cv=5, random_state=RANDOM_STATE), lr=hyperparams['lr'], batch_size=hyperparams['batch_size'], module__in_features=in_features, module__n_layers=hyperparams['n_layers'], module__n_neuron_per_layer=hyperparams['n_neuron_per_layer'], module__activation=getattr(F, hyperparams['activation']), module__p_dropout=hyperparams['p_dropout'], optimizer__momentum=hyperparams['momentum'], optimizer__weight_decay=hyperparams['weight_decay'], optimizer__nesterov=True, verbose=3, iterator_train__num_workers=4, iterator_valid__num_workers=4 )
def main(args): # Step 1: Fetch Datasets for training/validation/test datasets train_set, val_set, test_set = get_datasets(args) # Step 2: Initialize callbacks for NeuralNetClassifier lrscheduler = LRScheduler( policy='StepLR', step_size=7, gamma=0.1) checkpoint = Checkpoint( f_params='best_model.pt', monitor='valid_acc_best') freezer = Freezer(lambda x: not x.startswith('model.classifier.1')) net = NeuralNetClassifier( TwoClassSqueezeNet, criterion=nn.CrossEntropyLoss, batch_size=args.batch_size, max_epochs=args.num_epochs, module__num_classes=args.num_classes, optimizer=optim.SGD, iterator_train__shuffle=True, iterator_train__num_workers=args.num_workers, iterator_valid__shuffle=True, iterator_valid__num_workers=args.num_workers, # train_split fixes bug in skorch library, see: # https://github.com/skorch-dev/skorch/issues/599 train_split=None, device='cuda' # comment to train on cpu ) params = { 'optimizer__lr': [1e-5, 1e-4, 1e-3], 'optimizer__momentum': [0.5, 0.9, 0.99, 0.999], 'optimizer__nesterov': [True, False] } gs = GridSearchCV(net, params, refit=False, cv=3, scoring='accuracy', verbose=10) X_sl = SliceDataset(train_set, idx=0) # idx=0 is the default y_sl = SliceDataset(train_set, idx=1) # net.fit(train_set, y=None) gs.fit(X_sl, y_sl) print(gs.best_score_, gs.best_params_)
def calorie_model(val_ds): lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1) checkpoint = Checkpoint(f_params='models/calorie_net.pt', monitor='valid_acc_best') return NeuralNet(CalorieNet, criterion=nn.MSELoss(), lr=0.001, batch_size=64, max_epochs=25, optimizer=optim.SGD, optimizer__momentum=0.9, iterator_train__shuffle=True, iterator_train__num_workers=4, iterator_valid__shuffle=True, iterator_valid__num_workers=4, train_split=predefined_split(val_ds), callbacks=[lrscheduler, checkpoint], device='cuda')
def test_eeg_classifier_clonable(): preds = np.array([ [[0.2, 0.1, 0.1, 0.1], [0.8, 0.9, 0.9, 0.9]], [[1.0, 1.0, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0]], [[1.0, 1.0, 1.0, 0.2], [0.0, 0.0, 0.0, 0.8]], [[0.9, 0.8, 0.9, 1.0], [0.1, 0.2, 0.1, 0.0]], ]) clf = EEGClassifier(MockModule(preds), cropped=False, callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=1)) ], criterion=CroppedLoss, criterion__loss_function=nll_loss, optimizer=optim.Adam, batch_size=32) clone(clf) clf.initialize() clone(clf)
def prepare_learnt_model(model_args, path_tfms, is_meta, verbose=2): """Model builder if learnt transforms are involved. The key difference (as explained in prepare_non_learnt_model) between this function and prepare_non_learnt_model is that the Args: model_args (dict): Experiment model args as defined in the main experiment function. path_tfms (Pipeline): An sklearn pipeline of path transformations to be applied before model training. is_meta (bool): Set True for a dyadic meta model. verbose (int): Output verbosity level. Returns: """ # Initialise the signature string class. model_args['is_meta'] = is_meta module = SignatureStringModel(**model_args) model = NeuralNetClassifier( module=module, criterion=nn.BCEWithLogitsLoss if model_args['out_channels'] == 1 else nn.CrossEntropyLoss, batch_size=64, verbose=verbose, iterator_train__drop_last=True, callbacks=[ ('scheduler', LRScheduler(policy='ReduceLROnPlateau')), ('val_stopping', EarlyStopping(monitor='valid_loss', patience=30)), ('checkpoint', CustomCheckpoint(monitor='valid_loss_best')), ('scorer', EpochScoring(custom_scorer, lower_is_better=False, name='true_acc')) ], train_split=CVSplit(cv=5, random_state=1, stratified=True), device=device if model_args['gpu'] else 'cpu', ) pipeline = Pipeline([ *path_tfms, ('classifier', model) ]) return pipeline
def prepare_NN(self, num_classes): class ThreeLayerNN(nn.Module): def __init__(self, hidden, num_classes, activation): super(ThreeLayerNN, self).__init__() self.hidden = hidden self.num_classes = num_classes self.activation = activation self.Layer1 = nn.Linear(768, self.hidden, bias=True) self.Layer2 = nn.Linear(self.hidden, self.hidden, bias=True) self.Output = nn.Linear(self.hidden, self.num_classes, bias=True) def forward(self, x): z1 = self.Layer1(x) a1 = self.activation(z1) z2 = self.Layer2(a1) a2 = self.activation(z2) y_hat = F.softmax(a2) return(y_hat) monitor = lambda net: all(net.history[-1, ("train_loss_best", "valid_loss_best")]) net = NeuralNetClassifier(module=ThreeLayerNN, module__hidden=self.hidden, module__num_classes=num_classes, module__activation=self.activation, max_epochs=1000, lr=0.1, iterator_train__shuffle=True, callbacks=[Checkpoint(monitor=monitor), EarlyStopping(patience=5, threshold=0.0001), LRScheduler()], verbose=0, device=self.device, batch_size=32) return(net)
def sim_fit_model(args, X, y): auc = EpochScoring(scoring='roc_auc', lower_is_better=False) apr = EpochScoring(scoring='average_precision', lower_is_better=False) lrs = LRScheduler(policy='StepLR', step_size=10, gamma=0.5) if args.model == 'standard': net = NeuralNetClassifier(models.MpraDense, batch_size=256, optimizer=torch.optim.Adam, optimizer__weight_decay=2e-6, lr=1e-4, max_epochs=20, module__n_input=1079, module__n_units=(400, 250), module__dropout=0.3, callbacks=[auc, apr], iterator_train__shuffle=True, train_split=None) elif args.model == 'neighbors': net = NeuralNetClassifier(models.MpraFullCNN, batch_size=256, optimizer=torch.optim.Adam, optimizer__weight_decay=1e-2, lr=5e-5, max_epochs=20, callbacks=[auc, apr], iterator_train__shuffle=True, train_split=None) # generate predictions torch.manual_seed(1000) net.fit(X, y) return net
def main(): parser = argparse.ArgumentParser( description='PyTorch RNN with variable-length numeric sequences wrapper' ) parser.add_argument('--outcome_col_name', type=str, required=True) parser.add_argument('--train_csv_files', type=str, required=True) parser.add_argument('--test_csv_files', type=str, required=True) parser.add_argument('--data_dict_files', type=str, required=True) parser.add_argument('--batch_size', type=int, default=1024, help='Number of sequences per minibatch') parser.add_argument('--epochs', type=int, default=50, help='Number of epochs') parser.add_argument('--hidden_units', type=int, default=32, help='Number of hidden units') parser.add_argument('--hidden_layers', type=int, default=1, help='Number of hidden layers') parser.add_argument('--lr', type=float, default=0.0005, help='Learning rate for the optimizer') parser.add_argument('--dropout', type=float, default=0, help='dropout for optimizer') parser.add_argument('--weight_decay', type=float, default=0.0001, help='weight decay for optimizer') parser.add_argument('--seed', type=int, default=1111, help='random seed') parser.add_argument('--validation_size', type=float, default=0.15, help='validation split size') parser.add_argument( '--is_data_simulated', type=bool, default=False, help='boolean to check if data is simulated or from mimic') parser.add_argument( '--simulated_data_dir', type=str, default='simulated_data/2-state/', help= 'dir in which to simulated data is saved.Must be provide if is_data_simulated = True' ) parser.add_argument( '--output_dir', type=str, default=None, help= 'directory where trained model and loss curves over epochs are saved') parser.add_argument( '--output_filename_prefix', type=str, default=None, help='prefix for the training history jsons and trained classifier') args = parser.parse_args() torch.manual_seed(args.seed) device = 'cpu' x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split( ',') x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',') x_dict, y_dict = args.data_dict_files.split(',') x_data_dict = load_data_dict_json(x_dict) # get the id and feature columns id_cols = parse_id_cols(x_data_dict) feature_cols = parse_feature_cols(x_data_dict) # extract data train_vitals = TidySequentialDataCSVLoader( x_csv_path=x_train_csv_filename, y_csv_path=y_train_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_sequence') test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename, y_csv_path=y_test_csv_filename, x_col_names=feature_cols, idx_col_names=id_cols, y_col_name=args.outcome_col_name, y_label_type='per_sequence') X_train, y_train = train_vitals.get_batch_data(batch_id=0) X_test, y_test = test_vitals.get_batch_data(batch_id=0) _, T, F = X_train.shape print('number of time points : %s\n number of features : %s\n' % (T, F)) # set class weights as 1/(number of samples in class) for each class to handle class imbalance class_weights = torch.tensor( [1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).double() # scale features # X_train = standard_scaler_3d(X_train) # X_test = standard_scaler_3d(X_test) # callback to compute gradient norm compute_grad_norm = ComputeGradientNorm(norm_type=2) # LSTM if args.output_filename_prefix == None: output_filename_prefix = ( 'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' % (args.hidden_units, args.hidden_layers, args.lr, args.dropout, args.weight_decay)) else: output_filename_prefix = args.output_filename_prefix print('RNN parameters : ' + output_filename_prefix) # # from IPython import embed; embed() rnn = RNNBinaryClassifier( max_epochs=50, batch_size=args.batch_size, device=device, lr=args.lr, callbacks=[ EpochScoring('roc_auc', lower_is_better=False, on_train=True, name='aucroc_score_train'), EpochScoring('roc_auc', lower_is_better=False, on_train=False, name='aucroc_score_valid'), EarlyStopping(monitor='aucroc_score_valid', patience=20, threshold=0.002, threshold_mode='rel', lower_is_better=False), LRScheduler(policy=ReduceLROnPlateau, mode='max', monitor='aucroc_score_valid', patience=10), compute_grad_norm, GradientNormClipping(gradient_clip_value=0.3, gradient_clip_norm_type=2), Checkpoint(monitor='aucroc_score_valid', f_history=os.path.join( args.output_dir, output_filename_prefix + '.json')), TrainEndCheckpoint(dirname=args.output_dir, fn_prefix=output_filename_prefix), ], criterion=torch.nn.CrossEntropyLoss, criterion__weight=class_weights, train_split=skorch.dataset.CVSplit(args.validation_size), module__rnn_type='LSTM', module__n_layers=args.hidden_layers, module__n_hiddens=args.hidden_units, module__n_inputs=X_train.shape[-1], module__dropout_proba=args.dropout, optimizer=torch.optim.Adam, optimizer__weight_decay=args.weight_decay) clf = rnn.fit(X_train, y_train) y_pred_proba = clf.predict_proba(X_train) y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba) auroc_train_final = roc_auc_score(y_train, y_pred_proba_pos) print('AUROC with LSTM (Train) : %.2f' % auroc_train_final) y_pred_proba = clf.predict_proba(X_test) y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba) auroc_test_final = roc_auc_score(y_test, y_pred_proba_pos) print('AUROC with LSTM (Test) : %.2f' % auroc_test_final)
# In[28]: net = NeuralNetClassifier( MyModule, max_epochs=100, lr=0.001, batch_size=1024, optimizer=Adam, iterator_train__shuffle=True, iterator_train__num_workers=4, iterator_train__pin_memory=True, train_split=predefined_split(val), callbacks=[LRScheduler(policy=CosineAnnealingLR, T_max=64), EpochScoring(macrof1, use_caching=True, lower_is_better=False), EpochScoring(microf1, use_caching=True, lower_is_better=False), Checkpoint(monitor='macrof1_best', dirname='model')], device='cuda', verbose=1 ) print('start training') _ = net.fit(tra, y=None) # net.initialize() net.load_params(f_params='model/params.pt', f_optimizer='model/optimizer.pt', f_history='model/history.json') # In[ ]:
num_units=512, drop=0.5, num_units1=256, drop1=0.2): super().init() model = torchvision.models.mobilenet_v2(pretrained=False) n_inputs = model.classifier[1].in_features model.classifier = nn.Sequential(nn.Dropout(p=drop1), nn.Linear(n_inputs, output_features)) self.model = model def forward(self, x): return self.model(x) lr_scheduler_mobilenet = LRScheduler(policy='StepLR', step_size=8, gamma=0.2) # callback for saving the best on validation accuracy model checkpoint_mobilenet = Checkpoint( f_params='/content/drive/MyDrive/chess_weights/best_model_mobilenet.pkl', monitor='valid_acc_best') # callback for freezing all layer of the model except the last layer #freezer_vgg = Freezer(lambda x: not x.startswith('model.classifier')) # callback for early stopping early_stopping_mobilenet = EarlyStopping(patience=10) mobilenet = NeuralNetClassifier( # pretrained ResNet50 + custom classifier module=MobileNet, # fine tuning model's inner parameters module__output_features=13, module__num_units=512, module__drop=0.5,
net = NeuralNetBinaryClassifier( module=AudioNet, criterion=nn.BCEWithLogitsLoss, max_epochs=5000, lr=0.01, optimizer=optim.SGD, optimizer__momentum=0.9, batch_size=160, device=device, callbacks=[ ('tensorboard', TensorBoard(writer)), ('cp', cp), ('train_end_cp', train_end_cp), # ("load_state", load_state), ('early_stoping', EarlyStopping(patience=5)), ('lr_scheduler', LRScheduler( policy=ReduceLROnPlateau, monitor='valid_loss')), ], ) print("Begin training") try: y_train = np.concatenate(( np.zeros((100, )), np.ones((100, )))).astype('float32') net.fit(train_dataset, y_train) except KeyboardInterrupt: net.save_params(f_params=run+'.pkl') net.save_params(f_params=run + '.pkl') print("Finish training")
from skorch.callbacks import Checkpoint, EarlyStopping, EpochScoring early_stopping = EarlyStopping(patience=30) cp = Checkpoint(dirname='', f_criterion=None, f_optimizer=None, f_history=None) clf = EEGClassifier( model, # criterion=torch.nn.NLLLoss, criterion=torch.nn.CrossEntropyLoss, optimizer=torch.optim.AdamW, train_split=predefined_split(valid_set), # using valid_set for validation optimizer__lr=lr, optimizer__weight_decay=weight_decay, batch_size=batch_size, callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)),('cp', cp),('patience', early_stopping) ], device=device, ) # Model training for a specified number of epochs. `y` is None as it is already supplied # in the dataset. clf.fit(train_set, y=None, epochs=n_epochs) clf.load_params(checkpoint=cp) # Load the model with the lowest valid_loss import os os.remove('./params.pt') # Delete parameters file ###################################################################### # Save the feature extractor model # ------------
def exp(subject_id): import torch test_subj = np.r_[subject_id] print('test subj:' + str(test_subj)) # train_subj = np.setdiff1d(np.r_[1:10], test_subj) train_subj = np.setdiff1d(np.r_[1, 3, 7, 8], test_subj) tr = [] val = [] for ids in train_subj: train_size = int(0.99 * len(splitted[ids])) test_size = len(splitted[ids]) - train_size tr_i, val_i = torch.utils.data.random_split(splitted[ids], [train_size, test_size]) tr.append(tr_i) val.append(val_i) train_set = torch.utils.data.ConcatDataset(tr) valid_set = torch.utils.data.ConcatDataset(val) valid_set = BaseConcatDataset([splitted[ids] for ids in test_subj]) ###################################################################### # Create model # ------------ # ###################################################################### # Now we create the deep learning model! Braindecode comes with some # predefined convolutional neural network architectures for raw # time-domain EEG. Here, we use the shallow ConvNet model from `Deep # learning with convolutional neural networks for EEG decoding and # visualization <https://arxiv.org/abs/1703.05051>`__. These models are # pure `PyTorch <https://pytorch.org>`__ deep learning models, therefore # to use your own model, it just has to be a normal PyTorch # `nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__. # import torch from braindecode.util import set_random_seeds from braindecode.models import ShallowFBCSPNet, Deep4Net cuda = torch.cuda.is_available( ) # check if GPU is available, if True chooses to use it device = 'cuda:0' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = 20200220 # random seed to make results reproducible # Set random seed to be able to reproduce results set_random_seeds(seed=seed, cuda=cuda) n_classes = 3 # Extract number of chans and time steps from dataset n_chans = train_set[0][0].shape[0] input_window_samples = train_set[0][0].shape[1] # # model = ShallowFBCSPNet( # n_chans, # n_classes, # input_window_samples=input_window_samples, # final_conv_length='auto', # ) from mynetworks import Deep4Net_origin, ConvClfNet, FcClfNet model = Deep4Net( n_chans, n_classes, input_window_samples=input_window_samples, final_conv_length="auto", ) # # embedding_net = Deep4Net_origin(4, 22, input_window_samples) # model = FcClfNet(embedding_net) # # print(model) # Send model to GPU if cuda: model.cuda() ###################################################################### # Training # -------- # ###################################################################### # Now we train the network! EEGClassifier is a Braindecode object # responsible for managing the training of neural networks. It inherits # from skorch.NeuralNetClassifier, so the training logic is the same as in # `Skorch <https://skorch.readthedocs.io/en/stable/>`__. # ###################################################################### # **Note**: In this tutorial, we use some default parameters that we # have found to work well for motor decoding, however we strongly # encourage you to perform your own hyperparameter optimization using # cross validation on your training data. # from skorch.callbacks import LRScheduler from skorch.helper import predefined_split from braindecode import EEGClassifier # # These values we found good for shallow network: lr = 0.0625 * 0.01 weight_decay = 0 # For deep4 they should be: # lr = 1 * 0.01 # weight_decay = 0.5 * 0.001 batch_size = 8 n_epochs = 100 clf = EEGClassifier( model, criterion=torch.nn.NLLLoss, optimizer=torch.optim.AdamW, train_split=predefined_split( valid_set), # using valid_set for validation optimizer__lr=lr, optimizer__weight_decay=weight_decay, batch_size=batch_size, callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)), ], device=device, ) # Model training for a specified number of epochs. `y` is None as it is already supplied # in the dataset. clf.fit(train_set, y=None, epochs=n_epochs) ###################################################################### # Plot Results # ------------ # ###################################################################### # Now we use the history stored by Skorch throughout training to plot # accuracy and loss curves. # import matplotlib.pyplot as plt from matplotlib.lines import Line2D import pandas as pd # Extract loss and accuracy values for plotting from history object results_columns = [ 'train_loss', 'valid_loss', 'train_accuracy', 'valid_accuracy' ] df = pd.DataFrame(clf.history[:, results_columns], columns=results_columns, index=clf.history[:, 'epoch']) # get percent of misclass for better visual comparison to loss df = df.assign(train_misclass=100 - 100 * df.train_accuracy, valid_misclass=100 - 100 * df.valid_accuracy) plt.style.use('seaborn') fig, ax1 = plt.subplots(figsize=(8, 3)) df.loc[:, ['train_loss', 'valid_loss']].plot(ax=ax1, style=['-', ':'], marker='o', color='tab:blue', legend=False, fontsize=14) ax1.tick_params(axis='y', labelcolor='tab:blue', labelsize=14) ax1.set_ylabel("Loss", color='tab:blue', fontsize=14) ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis df.loc[:, ['train_misclass', 'valid_misclass']].plot(ax=ax2, style=['-', ':'], marker='o', color='tab:red', legend=False) ax2.tick_params(axis='y', labelcolor='tab:red', labelsize=14) ax2.set_ylabel("Misclassification Rate [%]", color='tab:red', fontsize=14) ax2.set_ylim(ax2.get_ylim()[0], 85) # make some room for legend ax1.set_xlabel("Epoch", fontsize=14) # where some data has already been plotted to ax handles = [] handles.append( Line2D([0], [0], color='black', linewidth=1, linestyle='-', label='Train')) handles.append( Line2D([0], [0], color='black', linewidth=1, linestyle=':', label='Valid')) plt.legend(handles, [h.get_label() for h in handles], fontsize=14) plt.tight_layout() # plt.show() return df
import os def setup_dirs(): for dir in [ "models", "optimizers", "outputs", "train_histories", "feature_vectors" ]: if not os.path.exists(dir): os.makedirs(dir) if __name__ == "__main__": setup_dirs() device = torch.device("cuda:0" if torch.cuda.is_available else "cpu") lrscheduler = LRScheduler(policy='StepLR', step_size=10, gamma=0.1) early_stopping = EarlyStopping(monitor="valid_loss", patience=5, threshold=0.01) csv_path = "./dataset.csv" train_df = pd.read_csv(csv_path) train_df['num_label'], _ = pd.factorize( train_df['label']) # turn labels (string) into numbers train_df = train_df[train_df['type'] == "train"] img_path = train_df['path'].values labels = train_df['num_label'].values X_train, X_val, y_train, y_val = train_test_split(img_path, labels, test_size=0.1, stratify=labels, random_state=42)
def fit(self, X, y, *args, w=None, **kwargs): # Determine optional parameters if self.claim_count_names is None: self.claim_count_names = [ "claim_count_{}".format(x) for x in range(0, int(y.shape[1] / 2)) ] if self.claim_paid_names is None: self.claim_paid_names = [ "claim_paid_{}".format(x) for x in range(0, int(y.shape[1] / 2)) ] if self.feature_dimension is None: self.feature_dimension = X.shape[1] if self.output_dimension is None: self.output_dimension = len(self.claim_paid_names) if self.categorical_dimensions is None: self.categorical_dimensions = [] # TODO: This is a bit slow and unstable, is there a better way? for i in range(X.shape[1]): X_int = X[:, i].astype(int) if np.all((X_int - X[:, i]) == 0): self.categorical_dimensions += [(i, np.max(X_int))] print( "Auto detect categorical dimensions to be: {}".format( self.categorical_dimensions ) ) # Standardize outputs # self.X_mean = np.mean(X, axis=0) # self.X_std = np.std(X, axis=0) # Except categoricals # for i, j in self.categorical_dimensions: # self.X_mean[i] = 0 # self.X_std[i] = 1 # X = (X - self.X_mean) / self.X_std # Shuffle X, y X, y = shuffle(X, y, random_state=0) if w is None: w = np.where(np.isnan(y), 0.0, 1.0) else: w = w * np.where(np.isnan(y), 0.0, 1.0) X = np.hstack([X, w]).astype(np.float32) y_mean = np.nanmean(y, axis=0) y = np.hstack([y, y, y]) y = np.where(np.isnan(y), 0, np.maximum(EPSILON, y)).astype(np.float32) earlystop = EarlyStopping(patience=self.patience, threshold=0.0) gradclip = GradientNormClipping(gradient_clip_value=self.clipnorm) if X.shape[0] < self.batch_size: print("NOTE: Data size is small, outcomes may be odd.") batch_size = X.shape[0] else: batch_size = self.batch_size # One cycle policy (with Adam) # Step 1: LR Range Finder # Test which values fit # Use earlystop to get an idea of epochs to 1 cycle policy as well. for lr in self.lr_range: super(PUNPPCIClaimRegressor, self).__init__( PUNPPCIClaimModule( feature_dim=self.feature_dimension, output_dim=self.output_dimension, cat_dim=self.categorical_dimensions, y_mean=y_mean, layer_size=self.layer_size, device=self.device, ), *args, **kwargs, max_epochs=self.max_epochs, lr=lr, device=self.device, optimizer=self.optimizer, # optimizer__momentum=self.momentum, optimizer__param_groups=[ ("embeddings_linear*", {"weight_decay": self.l1_l2_linear}), ( "embeddings_residual*", {"weight_decay": self.l2_weights_residual}, ), ("dense_pricing*", {"weight_decay": self.l2_weights_residual}), ("count_linear_0.weight", {"weight_decay": self.l1_l2_linear}), ("paid_linear_0.weight", {"weight_decay": self.l1_l2_linear}), ( "count_residual_spread.bias", {"weight_decay": self.l2_bias_residual}, ), ( "paid_residual_spread.bias", {"weight_decay": self.l2_bias_residual}, ), ("count_residual_0.bias", {"weight_decay": self.l2_bias_residual}), ("paid_residual_0.bias", {"weight_decay": self.l2_bias_residual}), ], batch_size=batch_size, criterion=nn.MSELoss, callbacks=[gradclip, earlystop], verbose=0 ) self.initialize_module() super(PUNPPCIClaimRegressor, self).fit(X, y) if not np.isnan(self.history[-1]["valid_loss"]): self.lr_min = self.lr_range[-1] self.lr_max = lr break # Still broke? if np.isnan(self.history[-1]["valid_loss"]): warn( "This model may fail to converge on the data. Please review data and parameters." ) self.lr_min = self.lr_range[-1] self.lr_max = 0.001 print("Setting maximum learn rate to {}.".format(self.lr_max)) # Step 2: Cyclic LR with expected epoch count... valid_losses = [x["valid_loss"] for x in self.history] expected_epoch_count = valid_losses.index(min(valid_losses)) + 1 expected_epoch_count = int(np.ceil(expected_epoch_count / 2) * 2) expected_epoch_count = 4 if expected_epoch_count < 4 else expected_epoch_count print("Setting epochs for training model to {}".format(expected_epoch_count)) cyclic_lr = LRScheduler( policy=CyclicLR, base_lr=self.lr_min, max_lr=self.lr_max, step_size_up=expected_epoch_count / 2, step_size_down=expected_epoch_count / 2, ) # ... but still keep training for as many epochs as required. super(PUNPPCIClaimRegressor, self).__init__( PUNPPCIClaimModule( feature_dim=self.feature_dimension, output_dim=self.output_dimension, cat_dim=self.categorical_dimensions, y_mean=y_mean, layer_size=self.layer_size, device=self.device, ), max_epochs=expected_epoch_count, lr=self.lr_min, device=self.device, optimizer=self.optimizer, # optimizer__momentum=self.momentum, optimizer__param_groups=[ ("embeddings_linear*", {"weight_decay": self.l1_l2_linear}), ("embeddings_residual*", {"weight_decay": self.l2_weights_residual}), ("dense_pricing*", {"weight_decay": self.l2_weights_residual}), ("count_linear_0.weight", {"weight_decay": self.l1_l2_linear}), ("paid_linear_0.weight", {"weight_decay": self.l1_l2_linear}), ("count_residual_spread.bias", {"weight_decay": self.l2_bias_residual}), ("paid_residual_spread.bias", {"weight_decay": self.l2_bias_residual}), ("count_residual_0.bias", {"weight_decay": self.l2_bias_residual}), ("paid_residual_0.bias", {"weight_decay": self.l2_bias_residual}), ], batch_size=batch_size, criterion=nn.MSELoss, callbacks=[ CheckNaN(), # CheckMean(X, self.output_dimension, 1), # expected_epoch_count cyclic_lr, gradclip, # earlystop, ], ) self.initialize_module() super(PUNPPCIClaimRegressor, self).fit(X, y) # Finished fitting! self.is_fitted_ = True
def prepare_non_learnt_model(model_args, path_tfms, is_meta=False, verbose=2): """Prepares a model that does not use learnt projections. We have a separate function for learnt and non-learnt transforms since the learnt transform requires signatures to be computed each time we evaluate over a batch. If the model is non-learnt then all signatures can be computed in advance which significantly reduces the training time. Args: model_args (dict): Experiment model arguments. See the main e path_tfms (Pipeline): A pipeline of path transformations. is_meta (bool): Set true if the model is a dyadic-meta-model. verbose (int): Level of output verbosity. Returns: sklearn transformer: Trained model. """ # Check if sklearn is_sklearn = check_sklearn(model_args['clf']) # Window arguments arg_names = ['ds_length', 'window_name', 'window_kwargs', 'depth', 'sig_tfm', 'rescaling'] window_args = { k: v for k, v in model_args.items() if k in arg_names } # Setup the disintegrator darg = model_args['disintegrations'] disintegrator = Disintegrator(size=darg) if isinstance(darg, int) else NullTransformer() # Random projector proj_args = model_args['num_projections'], model_args['projection_channels'] proj_bool = all([x is not None for x in proj_args]) if proj_bool: projector = Sklearnify(RandomProjections(model_args['in_channels'], proj_args[0], proj_args[1])) else: projector = NullTransformer() # Signature computation window = SklearnComputeWindowSignature(**window_args) if is_sklearn else TrickSignature(ComputeWindowSignature(**window_args)) # Set classifier classifier = CLASSIFIERS[model_args['clf']] if not is_sklearn: # We initialise with some defaults if not specified. init_clf_params = DEFAULT_CLF_PARAMS[model_args['clf']] clf_args = inspect.getfullargspec(classifier).args if 'length' in clf_args: init_clf_params['length'] = window.module.num_windows(model_args['ds_length']) # Update if Dyadic module = classifier(in_channels=model_args['in_channels_clf'], out_channels=model_args['out_channels'], **init_clf_params) if is_meta: module = DyadicModelIndividual( out_channels=model_args['out_channels'], dyadic_depth=window_args['depth'], hidden_channels=100, model=module ) classifier = NeuralNetClassifier( module=module, criterion=nn.BCEWithLogitsLoss if model_args['out_channels'] == 1 else nn.CrossEntropyLoss, batch_size=64, verbose=verbose, optimizer=optim.Adam, iterator_train__drop_last=True, callbacks=[ ('scheduler', LRScheduler(policy='ReduceLROnPlateau')), ('val_stopping', EarlyStopping(monitor='valid_loss', patience=30)), ('checkpoint', CustomCheckpoint(monitor='valid_loss_best')), ('scorer', EpochScoring(custom_scorer, lower_is_better=False, name='true_acc')) ], train_split=CVSplit(cv=5, random_state=1, stratified=True), device=device if model_args['gpu'] else 'cpu', ) model = Pipeline([ *path_tfms, ('disintegrator', disintegrator), ('random_projections', projector), ('signatures', window), ('classifier', classifier) ]) return model
out = self.model(x) return out class ClearCache(Callback): def on_batch_end(self, net, X=None, y=None, training=None, **kwargs): torch.cuda.empty_cache() def on_train_begin(self, net, X=None, y=None, **kwargs): print(f'\nModel {model_num}') clear_cache = ClearCache() lrscheduler = LRScheduler(policy='StepLR', step_size=25, gamma=0.25) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(device) data_dir = 'data' normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) train_transforms = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomGrayscale(p=0.35), transforms.RandomRotation((-11, 11)), transforms.RandomHorizontalFlip(p=0.3), transforms.ToTensor(), normalize ])
# drop_bad_windows=True, # ) # splits = dataset.split("session") # train_set = splits["train"] # valid_set = splits["eval"] regressor = EEGRegressor( model, cropped=True, criterion=CroppedLoss, criterion__loss_function=torch.nn.functional.mse_loss, optimizer=torch.optim.AdamW, train_split=predefined_split(valid_set), optimizer__lr=optimizer_lr, optimizer__weight_decay=optimizer_weight_decay, iterator_train__shuffle=True, batch_size=batch_size, callbacks=[ "neg_root_mean_squared_error", # seems n_epochs -1 leads to desired behavior of lr=0 after end of data? ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)), ], device=device, ) regressor.fit(train_set, y=None, epochs=n_epochs) if __name__ == '__main__': create_compatible_dataset('./data/BCICIV_4_mat/sub1_comp.mat')
def fit_model(args, X, y): print(f'Fitting model for {args.model}:') auc = EpochScoring(scoring='roc_auc', lower_is_better=False) apr = EpochScoring(scoring='average_precision', lower_is_better=False) lrs = LRScheduler(policy='StepLR', step_size=10, gamma=0.5) if args.model == 'glm': glm = LogitNet(alpha=0.5, n_lambda=50, n_jobs=-1) glm.fit(X, y) kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1111) net = LogitNet(alpha=0.5, n_lambda=1, lambda_path=[glm.lambda_best_]) if args.model == 'standard': net = NeuralNetClassifier( models.MpraDense, batch_size=256, optimizer=torch.optim.Adam, optimizer__weight_decay=2e-6, lr=1e-4, max_epochs=20, module__n_input=1079, module__n_units=(400, 250), module__dropout=0.3, callbacks=[auc, apr], iterator_train__shuffle=True, train_split=None ) elif args.model == 'neighbors': net = NeuralNetClassifier( models.MpraFullCNN, batch_size=256, optimizer=torch.optim.Adam, optimizer__weight_decay=1e-2, lr=5e-5, max_epochs=20, callbacks=[auc, apr], iterator_train__shuffle=True, train_split=None ) # generate CV predictions np.random.seed(1000) torch.manual_seed(1000) kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1000) cv_scores = cross_val_predict(net, X, y, cv=kf, method='predict_proba', n_jobs=-1) AUC = roc_auc_score(y, cv_scores[:, 1]) APR = average_precision_score(y, cv_scores[:, 1]) print('\tAUC ', np.round(AUC, 4)) print('\tAPR ', np.round(APR, 4)) save_scores(args, cv_scores[:, 1], y) # refit and store model on all data net.fit(X, y) save_model(net, args.project, args.model)
def build_estimator(hyperparams, train_data, test=False): device = "cuda" if torch.cuda.is_available() else "cpu" # Extract info from training data X, y, *_ = train_data in_features = X.shape[1] n_classes = len(np.unique(y)) n_samples = y.shape[0] bal_weights = (torch.from_numpy( n_samples / (n_classes * np.bincount(y))).float().to(device)) callbacks = [ ( "f1_score_valid", EpochScoring( "f1" if n_classes == 2 else "f1_macro", name="valid_f1", lower_is_better=False, ), ), ( "early_stopping", EarlyStopping(monitor="valid_loss", patience=5, lower_is_better=True), ), ( "learning_rate_scheduler", LRScheduler( policy=lr_scheduler.ReduceLROnPlateau, monitor="valid_loss", # Following kargs are passed to the # lr scheduler constructor mode="min", min_lr=1e-5, ), ), ] return NeuralNetClassifier( NNModule, criterion=nn.CrossEntropyLoss, optimizer=torch.optim.SGD, max_epochs=300, iterator_train__shuffle=True, # Shuffle training data on each epoch callbacks=callbacks, device=device, train_split=CVSplit(cv=5, stratified=True, random_state=RANDOM_STATE), lr=hyperparams["lr"], batch_size=hyperparams["batch_size"], module__in_features=in_features, module__n_classes=n_classes, module__n_layers=hyperparams["n_layers"], module__n_neuron_per_layer=hyperparams["n_neuron_per_layer"], module__activation=getattr(F, hyperparams["activation"]), module__p_dropout=hyperparams["p_dropout"], criterion__weight=bal_weights if hyperparams["class_weight"] == "balanced" else None, optimizer__momentum=hyperparams["momentum"], optimizer__weight_decay=hyperparams["weight_decay"], optimizer__nesterov=True, verbose=3, iterator_train__num_workers=4, iterator_valid__num_workers=4, )
calc_fp_valid = EpochScoring(calc_fp, lower_is_better=False, on_train=False, name='fp_valid') cp = Checkpoint(monitor='precision_valid', f_history=os.path.join( args.output_dir, args.output_filename_prefix + '.json')) train_end_cp = TrainEndCheckpoint(dirname=args.output_dir, fn_prefix=args.output_filename_prefix) lr_scheduler = LRScheduler(policy=ReduceLROnPlateau, mode='max', factor=0.1, patience=3, min_lr=1e-04, verbose=True) # n_cv_folds = int(np.floor(1/args.validation_size)) ## start training fixed_precision = 0.2 thr_list = [0.5] if args.scoring == 'cross_entropy_loss': mlp = SkorchMLP( n_features=len(feature_cols), l2_penalty_weights=args.weight_decay, lr=args.lr, batch_size=args.batch_size, max_epochs=max_epochs, train_split=predefined_split(valid_ds),