def train(_run, layer_name, lr, mu, lambda_bar, batch_size, device, max_epochs,
epochs_update, alphaupdate, optimizer):
module = get_module()
X, y = get_dataset()
alphaupdate_kwags = flatten_dict(alphaupdate,
preffix='callbacks__AlphaUpdate')
checkpoint_tmpfile = '/tmp/weights.pt'
callbacks = [LRScheduler(policy='StepLR', gamma=0.1, step_size=200)]
net = LDMnet(module,
criterion=torch.nn.CrossEntropyLoss,
layer_name=layer_name,
mu=mu,
lambda_bar=lambda_bar,
lr=lr,
epochs_update=epochs_update,
max_epochs=max_epochs,
batch_size=batch_size,
device=device,
callbacks=callbacks,
**alphaupdate_kwags,
**flatten_dict(optimizer, 'optimizer'))
net.fit(X, y=y)
net.save_params(checkpoint_tmpfile)
ex.add_artifact(checkpoint_tmpfile)
def __init__(self,
max_epochs=100,
lr=0.0001,
optimizer=torch.optim.SGD,
optimizer__momentum = 0.999,
optimizer__weight_decay = 1e-4,
optimizer__nesterov = True,
module__dropout_depth = True,
module__dropout_width = True,
module__dropout_p = 0.5,
**kwds):
args = [ResNetUNet, torch.nn.CrossEntropyLoss]
kwds["module__nInputFeatures"] = self.number_of_features
kwds["module__nClasses"] = self.number_of_classes
kwds["criterion__weight"] = true_distribution
kwds["iterator_train"] = MMDBDataset
kwds["iterator_train_collate_fn"] = sparse_collate
kwds["iterator_valid"] = MMBDDataset
kwds["iterator_valid_collate_fn"] = sparse_collate
scheduler = LRScheduler(policy="LambdaLR", lr_lambda=lambda epoch: math.exp((1 - epoch) * lr_decay))
if "callbacks" in kwds and isinstance(kwds["callbacks"], list):
callbacks.append(scheduler)
else:
kwds["callbacks"] = [scheduler]
super().__init__(*args, **kwargs)
def load_extras(self):
callbacks = []
load_best_loss = train_end_load_best_loss(self.identifier)
self.split = CVSplit(cv=self.val_split) if self.val_split != 0 else 0
metrics = evaluator(
self.val_split,
self.config["optim"].get("metric", "mae"),
self.identifier,
self.forcetraining,
)
callbacks.extend(metrics)
if not self.debug:
callbacks.append(load_best_loss)
scheduler = self.config["optim"].get("scheduler", None)
if scheduler:
scheduler = LRScheduler(scheduler,
**self.config["optim"]["scheduler_params"])
callbacks.append(scheduler)
if self.config["cmd"].get("logger", False):
from skorch.callbacks import WandbLogger
callbacks.append(
WandbLogger(
self.wandb_run,
save_model=False,
keys_ignored="dur",
))
self.callbacks = callbacks
def build_estimator(hyperparams, train_data, test=False):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
n_classes = len(np.unique(y))
n_samples = y.shape[0]
bal_weights = torch.from_numpy(
n_samples / (n_classes * np.bincount(y))).float().to(device)
callbacks = [
('f1_score_valid',
EpochScoring('f1' if n_classes == 2 else 'f1_macro',
name='valid_f1',
lower_is_better=False)),
('early_stopping',
EarlyStopping(monitor='valid_loss',
patience=5,
lower_is_better=True)),
(
'learning_rate_scheduler',
LRScheduler(
policy=lr_scheduler.ReduceLROnPlateau,
monitor='valid_loss',
# Following kargs are passed to the
# lr scheduler constructor
mode='min',
min_lr=1e-5)),
]
return NeuralNetClassifier(
NNModule,
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5,
stratified=True,
random_state=RANDOM_STATE),
lr=hyperparams['lr'],
batch_size=hyperparams['batch_size'],
module__in_features=in_features,
module__n_classes=n_classes,
module__n_layers=hyperparams['n_layers'],
module__n_neuron_per_layer=hyperparams['n_neuron_per_layer'],
module__activation=getattr(F, hyperparams['activation']),
module__p_dropout=hyperparams['p_dropout'],
criterion__weight=bal_weights
if hyperparams['class_weight'] == 'balanced' else None,
optimizer__momentum=hyperparams['momentum'],
optimizer__weight_decay=hyperparams['weight_decay'],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4)
def build_estimator(cls,
hyperparams,
train_data,
verbose=True,
test=False): # change default verbose to false later
early_stopping_val_percent = 10
n_training_examples = len(
train_data[0]) * (1 - (early_stopping_val_percent / 100))
n_iter_per_epoch = n_training_examples / hyperparams['batch_size']
n_iter_btw_restarts = int(hyperparams['epochs_btw_restarts'] *
n_iter_per_epoch)
callbacks = [
('fix_seed', cls.FixRandomSeed(RANDOM_STATE)),
('lr_monitor', cls.LRMonitor()),
('accuracy_score_valid',
EpochScoring('accuracy', lower_is_better=False, on_train=True)),
('early_stopping',
EarlyStopping(monitor='valid_acc',
lower_is_better=False,
patience=100)),
('learning_rate_scheduler',
LRScheduler(policy=cls.SkorchCosineAnnealingWarmRestarts,
T_0=n_iter_btw_restarts,
T_mult=hyperparams['epochs_btw_restarts_mult']))
]
def validation_split(X, y):
""" Custom split is used to apply augmentation to the training set only """
splitter = CVSplit(cv=int(100 / early_stopping_val_percent),
random_state=RANDOM_STATE)
dataset_train, dataset_valid = splitter(X)
dataset_train = cls.AugmentedDataset(dataset_train)
return dataset_train, dataset_valid
return NeuralNetClassifier(
cls.CifarCustomNet,
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.SGD,
max_epochs=hyperparams['max_epochs'] if not test else 1,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
iterator_valid__num_workers=4,
dataset=cls.NormalizedDataset,
callbacks=callbacks,
device=cls.device,
train_split=validation_split,
lr=hyperparams['learning_rate'],
batch_size=hyperparams['batch_size'],
optimizer__momentum=hyperparams['momentum'],
optimizer__weight_decay=hyperparams['weight_decay'],
optimizer__nesterov=hyperparams['nesterov'],
module__conv_dropout=hyperparams['conv_dropout'],
module__fc_dropout=hyperparams['fc_dropout'],
verbose=3 if verbose else 0)
def get_deep_learning_model(model_args, valid_dataset):
cuda = torch.cuda.is_available()
device = model_args["device"] if cuda else 'cpu'
if cuda:
torch.backends.cudnn.benchmark = True
seed = model_args["seed"]
# = 20200220 random seed to make results reproducible
# Set random seed to be able to reproduce results
if seed:
set_random_seeds(seed=seed, cuda=cuda)
if model_args["model_type"] == "ShallowFBCSPNet":
model = ShallowFBCSPNet(
model_args["n_chans"],
model_args["n_classes"] + 1,
input_window_samples=model_args["input_window_samples"],
final_conv_length='auto',
)
elif model_args["model_type"] == "SleepStager":
model = model = SleepStager(
n_channels=model_args["n_chans"],
sfreq=model_args["sfreq"],
n_classes=model_args["n_classes"] + 1,
input_size_s=model_args["input_window_samples"] /
model_args["sfreq"],
)
else:
raise ValueError("Boom !")
if cuda:
model.cuda()
clf = EEGClassifier(
model,
criterion=model_args["criterion"],
optimizer=torch.optim.AdamW,
# using test_sample for validation
train_split=predefined_split(valid_dataset),
optimizer__lr=model_args["lr"],
optimizer__weight_decay=model_args["weight_decay"],
batch_size=model_args["batch_size"],
callbacks=[
"accuracy",
("lr_scheduler",
LRScheduler('CosineAnnealingLR',
T_max=model_args["n_epochs"] - 1)),
("early_stopping",
EarlyStopping(monitor='valid_loss',
patience=model_args["patience"]))
],
device=device,
iterator_train__num_workers=20,
iterator_train__pin_memory=True) # torch.in torch.out
return clf
def build_model(
self,
network=MVRegressor,
device: str = "cpu",
scale_data: bool = False,
num_layers: int = 10,
num_units: int = 50,
dropout: float = 0.5,
num_epochs: int = 10,
batch_size: int = 128,
):
self.scale_data = scale_data
self.num_layers = num_layers
self.num_units = num_units
self.dropout = dropout
self.num_epochs = num_epochs
self.batch_size = batch_size
if not all([hasattr(self, "input_dim"), hasattr(self, "output_dim")]):
raise ValueError(
"Please load dataset first to obtain proper sizes")
if device == "cpu":
self.device = device
else:
use_cuda = torch.cuda.is_available()
self.device = torch.device("cuda" if use_cuda else "cpu")
self.model = NeuralNetRegressor(
network,
device=self.device,
module__input_dim=self.input_dim,
module__output_dim=self.output_dim,
module__n_layers=self.num_layers,
module__num_units=self.num_units,
module__p_dropout=self.dropout,
max_epochs=self.num_epochs,
criterion=nn.MSELoss,
batch_size=self.batch_size,
# Shuffle training data on each epoch
iterator_train__shuffle=True,
callbacks=[
(
"lr_scheduler",
LRScheduler(policy=CyclicLR,
base_lr=0.001,
max_lr=0.01,
step_every="batch"),
),
],
)
def main():
sampling_rate = 360
wavelet = "mexh" # mexh, morl, gaus8, gaus4
scales = pywt.central_frequency(wavelet) * sampling_rate / np.arange(
1, 101, 1)
(x1_train, x2_train, y_train,
groups_train), (x1_test, x2_test, y_test,
groups_test) = load_data(wavelet=wavelet,
scales=scales,
sampling_rate=sampling_rate)
print("Data loaded successfully!")
log_dir = "./logs/{}".format(wavelet)
shutil.rmtree(log_dir, ignore_errors=True)
callbacks = [
Initializer("[conv|fc]*.weight", fn=torch.nn.init.kaiming_normal_),
Initializer("[conv|fc]*.bias",
fn=partial(torch.nn.init.constant_, val=0.0)),
LRScheduler(policy=StepLR, step_size=5, gamma=0.1),
EpochScoring(scoring=make_scorer(f1_score, average="macro"),
lower_is_better=False,
name="valid_f1"),
TensorBoard(SummaryWriter(log_dir))
]
net = NeuralNetClassifier( # skorch is extensive package of pytorch for compatible with scikit-learn
MyModule,
criterion=torch.nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
lr=0.001,
max_epochs=30,
batch_size=1024,
train_split=predefined_split(
Dataset({
"x1": x1_test,
"x2": x2_test
}, y_test)),
verbose=1,
device="cuda",
callbacks=callbacks,
iterator_train__shuffle=True,
optimizer__weight_decay=0,
)
net.fit({"x1": x1_train, "x2": x2_train}, y_train)
y_true, y_pred = y_test, net.predict({"x1": x1_test, "x2": x2_test})
print(confusion_matrix(y_true, y_pred))
print(classification_report(y_true, y_pred, digits=4))
net.save_params(f_params="./models/model_{}.pkl".format(wavelet))
def build_estimator(hyperparams, train_data, test=False):
device = "cuda" if torch.cuda.is_available() else "cpu"
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
callbacks = [
("r2_score_valid", EpochScoring("r2", lower_is_better=False)),
(
"early_stopping",
EarlyStopping(monitor="valid_loss", patience=5, lower_is_better=True),
),
(
"learning_rate_scheduler",
LRScheduler(
policy=lr_scheduler.ReduceLROnPlateau,
monitor="valid_loss",
# Following kargs are passed to the
# lr scheduler constructor
mode="min",
min_lr=1e-5,
),
),
]
return NeuralNetRegressor(
NNModule,
criterion=nn.MSELoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5, random_state=RANDOM_STATE),
lr=hyperparams["lr"],
batch_size=hyperparams["batch_size"],
module__in_features=in_features,
module__n_layers=hyperparams["n_layers"],
module__n_neuron_per_layer=hyperparams["n_neuron_per_layer"],
module__activation=getattr(F, hyperparams["activation"]),
module__p_dropout=hyperparams["p_dropout"],
optimizer__momentum=hyperparams["momentum"],
optimizer__weight_decay=hyperparams["weight_decay"],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4,
)
def build_estimator(hyperparams, train_data, test=False):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
callbacks = [
('r2_score_valid', EpochScoring('r2',
lower_is_better=False)),
('early_stopping', EarlyStopping(monitor='valid_loss',
patience=5,
lower_is_better=True)),
('learning_rate_scheduler', LRScheduler(policy=lr_scheduler.ReduceLROnPlateau,
monitor='valid_loss',
# Following kargs are passed to the
# lr scheduler constructor
mode='min',
min_lr=1e-5
)),
]
return NeuralNetRegressor(
NNModule,
criterion=nn.MSELoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5, random_state=RANDOM_STATE),
lr=hyperparams['lr'],
batch_size=hyperparams['batch_size'],
module__in_features=in_features,
module__n_layers=hyperparams['n_layers'],
module__n_neuron_per_layer=hyperparams['n_neuron_per_layer'],
module__activation=getattr(F, hyperparams['activation']),
module__p_dropout=hyperparams['p_dropout'],
optimizer__momentum=hyperparams['momentum'],
optimizer__weight_decay=hyperparams['weight_decay'],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4
)
def main(args):
# Step 1: Fetch Datasets for training/validation/test datasets
train_set, val_set, test_set = get_datasets(args)
# Step 2: Initialize callbacks for NeuralNetClassifier
lrscheduler = LRScheduler(
policy='StepLR', step_size=7, gamma=0.1)
checkpoint = Checkpoint(
f_params='best_model.pt', monitor='valid_acc_best')
freezer = Freezer(lambda x: not x.startswith('model.classifier.1'))
net = NeuralNetClassifier(
TwoClassSqueezeNet,
criterion=nn.CrossEntropyLoss,
batch_size=args.batch_size,
max_epochs=args.num_epochs,
module__num_classes=args.num_classes,
optimizer=optim.SGD,
iterator_train__shuffle=True,
iterator_train__num_workers=args.num_workers,
iterator_valid__shuffle=True,
iterator_valid__num_workers=args.num_workers,
# train_split fixes bug in skorch library, see:
# https://github.com/skorch-dev/skorch/issues/599
train_split=None,
device='cuda' # comment to train on cpu
)
params = {
'optimizer__lr': [1e-5, 1e-4, 1e-3],
'optimizer__momentum': [0.5, 0.9, 0.99, 0.999],
'optimizer__nesterov': [True, False]
}
gs = GridSearchCV(net, params, refit=False, cv=3, scoring='accuracy',
verbose=10)
X_sl = SliceDataset(train_set, idx=0) # idx=0 is the default
y_sl = SliceDataset(train_set, idx=1)
# net.fit(train_set, y=None)
gs.fit(X_sl, y_sl)
print(gs.best_score_, gs.best_params_)
def calorie_model(val_ds):
lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1)
checkpoint = Checkpoint(f_params='models/calorie_net.pt',
monitor='valid_acc_best')
return NeuralNet(CalorieNet,
criterion=nn.MSELoss(),
lr=0.001,
batch_size=64,
max_epochs=25,
optimizer=optim.SGD,
optimizer__momentum=0.9,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
iterator_valid__shuffle=True,
iterator_valid__num_workers=4,
train_split=predefined_split(val_ds),
callbacks=[lrscheduler, checkpoint],
device='cuda')
def test_eeg_classifier_clonable():
preds = np.array([
[[0.2, 0.1, 0.1, 0.1], [0.8, 0.9, 0.9, 0.9]],
[[1.0, 1.0, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0]],
[[1.0, 1.0, 1.0, 0.2], [0.0, 0.0, 0.0, 0.8]],
[[0.9, 0.8, 0.9, 1.0], [0.1, 0.2, 0.1, 0.0]],
])
clf = EEGClassifier(MockModule(preds),
cropped=False,
callbacks=[
"accuracy",
("lr_scheduler",
LRScheduler('CosineAnnealingLR', T_max=1))
],
criterion=CroppedLoss,
criterion__loss_function=nll_loss,
optimizer=optim.Adam,
batch_size=32)
clone(clf)
clf.initialize()
clone(clf)
def prepare_learnt_model(model_args, path_tfms, is_meta, verbose=2):
"""Model builder if learnt transforms are involved.
The key difference (as explained in prepare_non_learnt_model) between this function and prepare_non_learnt_model
is that the
Args:
model_args (dict): Experiment model args as defined in the main experiment function.
path_tfms (Pipeline): An sklearn pipeline of path transformations to be applied before model training.
is_meta (bool): Set True for a dyadic meta model.
verbose (int): Output verbosity level.
Returns:
"""
# Initialise the signature string class.
model_args['is_meta'] = is_meta
module = SignatureStringModel(**model_args)
model = NeuralNetClassifier(
module=module,
criterion=nn.BCEWithLogitsLoss if model_args['out_channels'] == 1 else nn.CrossEntropyLoss,
batch_size=64,
verbose=verbose,
iterator_train__drop_last=True,
callbacks=[
('scheduler', LRScheduler(policy='ReduceLROnPlateau')),
('val_stopping', EarlyStopping(monitor='valid_loss', patience=30)),
('checkpoint', CustomCheckpoint(monitor='valid_loss_best')),
('scorer', EpochScoring(custom_scorer, lower_is_better=False, name='true_acc'))
],
train_split=CVSplit(cv=5, random_state=1, stratified=True),
device=device if model_args['gpu'] else 'cpu',
)
pipeline = Pipeline([
*path_tfms,
('classifier', model)
])
return pipeline
def prepare_NN(self, num_classes):
class ThreeLayerNN(nn.Module):
def __init__(self, hidden, num_classes, activation):
super(ThreeLayerNN, self).__init__()
self.hidden = hidden
self.num_classes = num_classes
self.activation = activation
self.Layer1 = nn.Linear(768, self.hidden, bias=True)
self.Layer2 = nn.Linear(self.hidden, self.hidden, bias=True)
self.Output = nn.Linear(self.hidden, self.num_classes, bias=True)
def forward(self, x):
z1 = self.Layer1(x)
a1 = self.activation(z1)
z2 = self.Layer2(a1)
a2 = self.activation(z2)
y_hat = F.softmax(a2)
return(y_hat)
monitor = lambda net: all(net.history[-1, ("train_loss_best", "valid_loss_best")])
net = NeuralNetClassifier(module=ThreeLayerNN,
module__hidden=self.hidden,
module__num_classes=num_classes,
module__activation=self.activation,
max_epochs=1000,
lr=0.1,
iterator_train__shuffle=True,
callbacks=[Checkpoint(monitor=monitor),
EarlyStopping(patience=5,
threshold=0.0001),
LRScheduler()],
verbose=0,
device=self.device,
batch_size=32)
return(net)
def sim_fit_model(args, X, y):
auc = EpochScoring(scoring='roc_auc', lower_is_better=False)
apr = EpochScoring(scoring='average_precision', lower_is_better=False)
lrs = LRScheduler(policy='StepLR', step_size=10, gamma=0.5)
if args.model == 'standard':
net = NeuralNetClassifier(models.MpraDense,
batch_size=256,
optimizer=torch.optim.Adam,
optimizer__weight_decay=2e-6,
lr=1e-4,
max_epochs=20,
module__n_input=1079,
module__n_units=(400, 250),
module__dropout=0.3,
callbacks=[auc, apr],
iterator_train__shuffle=True,
train_split=None)
elif args.model == 'neighbors':
net = NeuralNetClassifier(models.MpraFullCNN,
batch_size=256,
optimizer=torch.optim.Adam,
optimizer__weight_decay=1e-2,
lr=5e-5,
max_epochs=20,
callbacks=[auc, apr],
iterator_train__shuffle=True,
train_split=None)
# generate predictions
torch.manual_seed(1000)
net.fit(X, y)
return net
def main():
parser = argparse.ArgumentParser(
description='PyTorch RNN with variable-length numeric sequences wrapper'
)
parser.add_argument('--outcome_col_name', type=str, required=True)
parser.add_argument('--train_csv_files', type=str, required=True)
parser.add_argument('--test_csv_files', type=str, required=True)
parser.add_argument('--data_dict_files', type=str, required=True)
parser.add_argument('--batch_size',
type=int,
default=1024,
help='Number of sequences per minibatch')
parser.add_argument('--epochs',
type=int,
default=50,
help='Number of epochs')
parser.add_argument('--hidden_units',
type=int,
default=32,
help='Number of hidden units')
parser.add_argument('--hidden_layers',
type=int,
default=1,
help='Number of hidden layers')
parser.add_argument('--lr',
type=float,
default=0.0005,
help='Learning rate for the optimizer')
parser.add_argument('--dropout',
type=float,
default=0,
help='dropout for optimizer')
parser.add_argument('--weight_decay',
type=float,
default=0.0001,
help='weight decay for optimizer')
parser.add_argument('--seed', type=int, default=1111, help='random seed')
parser.add_argument('--validation_size',
type=float,
default=0.15,
help='validation split size')
parser.add_argument(
'--is_data_simulated',
type=bool,
default=False,
help='boolean to check if data is simulated or from mimic')
parser.add_argument(
'--simulated_data_dir',
type=str,
default='simulated_data/2-state/',
help=
'dir in which to simulated data is saved.Must be provide if is_data_simulated = True'
)
parser.add_argument(
'--output_dir',
type=str,
default=None,
help=
'directory where trained model and loss curves over epochs are saved')
parser.add_argument(
'--output_filename_prefix',
type=str,
default=None,
help='prefix for the training history jsons and trained classifier')
args = parser.parse_args()
torch.manual_seed(args.seed)
device = 'cpu'
x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split(
',')
x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',')
x_dict, y_dict = args.data_dict_files.split(',')
x_data_dict = load_data_dict_json(x_dict)
# get the id and feature columns
id_cols = parse_id_cols(x_data_dict)
feature_cols = parse_feature_cols(x_data_dict)
# extract data
train_vitals = TidySequentialDataCSVLoader(
x_csv_path=x_train_csv_filename,
y_csv_path=y_train_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_sequence')
test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename,
y_csv_path=y_test_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_sequence')
X_train, y_train = train_vitals.get_batch_data(batch_id=0)
X_test, y_test = test_vitals.get_batch_data(batch_id=0)
_, T, F = X_train.shape
print('number of time points : %s\n number of features : %s\n' % (T, F))
# set class weights as 1/(number of samples in class) for each class to handle class imbalance
class_weights = torch.tensor(
[1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).double()
# scale features
# X_train = standard_scaler_3d(X_train)
# X_test = standard_scaler_3d(X_test)
# callback to compute gradient norm
compute_grad_norm = ComputeGradientNorm(norm_type=2)
# LSTM
if args.output_filename_prefix == None:
output_filename_prefix = (
'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' %
(args.hidden_units, args.hidden_layers, args.lr, args.dropout,
args.weight_decay))
else:
output_filename_prefix = args.output_filename_prefix
print('RNN parameters : ' + output_filename_prefix)
# # from IPython import embed; embed()
rnn = RNNBinaryClassifier(
max_epochs=50,
batch_size=args.batch_size,
device=device,
lr=args.lr,
callbacks=[
EpochScoring('roc_auc',
lower_is_better=False,
on_train=True,
name='aucroc_score_train'),
EpochScoring('roc_auc',
lower_is_better=False,
on_train=False,
name='aucroc_score_valid'),
EarlyStopping(monitor='aucroc_score_valid',
patience=20,
threshold=0.002,
threshold_mode='rel',
lower_is_better=False),
LRScheduler(policy=ReduceLROnPlateau,
mode='max',
monitor='aucroc_score_valid',
patience=10),
compute_grad_norm,
GradientNormClipping(gradient_clip_value=0.3,
gradient_clip_norm_type=2),
Checkpoint(monitor='aucroc_score_valid',
f_history=os.path.join(
args.output_dir, output_filename_prefix + '.json')),
TrainEndCheckpoint(dirname=args.output_dir,
fn_prefix=output_filename_prefix),
],
criterion=torch.nn.CrossEntropyLoss,
criterion__weight=class_weights,
train_split=skorch.dataset.CVSplit(args.validation_size),
module__rnn_type='LSTM',
module__n_layers=args.hidden_layers,
module__n_hiddens=args.hidden_units,
module__n_inputs=X_train.shape[-1],
module__dropout_proba=args.dropout,
optimizer=torch.optim.Adam,
optimizer__weight_decay=args.weight_decay)
clf = rnn.fit(X_train, y_train)
y_pred_proba = clf.predict_proba(X_train)
y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_train_final = roc_auc_score(y_train, y_pred_proba_pos)
print('AUROC with LSTM (Train) : %.2f' % auroc_train_final)
y_pred_proba = clf.predict_proba(X_test)
y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_test_final = roc_auc_score(y_test, y_pred_proba_pos)
print('AUROC with LSTM (Test) : %.2f' % auroc_test_final)
# In[28]:
net = NeuralNetClassifier(
MyModule,
max_epochs=100,
lr=0.001,
batch_size=1024,
optimizer=Adam,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
iterator_train__pin_memory=True,
train_split=predefined_split(val),
callbacks=[LRScheduler(policy=CosineAnnealingLR, T_max=64),
EpochScoring(macrof1, use_caching=True, lower_is_better=False),
EpochScoring(microf1, use_caching=True, lower_is_better=False),
Checkpoint(monitor='macrof1_best', dirname='model')],
device='cuda',
verbose=1
)
print('start training')
_ = net.fit(tra, y=None)
# net.initialize()
net.load_params(f_params='model/params.pt', f_optimizer='model/optimizer.pt', f_history='model/history.json')
# In[ ]:
num_units=512,
drop=0.5,
num_units1=256,
drop1=0.2):
super().init()
model = torchvision.models.mobilenet_v2(pretrained=False)
n_inputs = model.classifier[1].in_features
model.classifier = nn.Sequential(nn.Dropout(p=drop1),
nn.Linear(n_inputs, output_features))
self.model = model
def forward(self, x):
return self.model(x)
lr_scheduler_mobilenet = LRScheduler(policy='StepLR', step_size=8, gamma=0.2)
# callback for saving the best on validation accuracy model
checkpoint_mobilenet = Checkpoint(
f_params='/content/drive/MyDrive/chess_weights/best_model_mobilenet.pkl',
monitor='valid_acc_best')
# callback for freezing all layer of the model except the last layer
#freezer_vgg = Freezer(lambda x: not x.startswith('model.classifier'))
# callback for early stopping
early_stopping_mobilenet = EarlyStopping(patience=10)
mobilenet = NeuralNetClassifier(
# pretrained ResNet50 + custom classifier
module=MobileNet,
# fine tuning model's inner parameters
module__output_features=13,
module__num_units=512,
module__drop=0.5,
net = NeuralNetBinaryClassifier(
module=AudioNet,
criterion=nn.BCEWithLogitsLoss,
max_epochs=5000,
lr=0.01,
optimizer=optim.SGD,
optimizer__momentum=0.9,
batch_size=160,
device=device,
callbacks=[
('tensorboard', TensorBoard(writer)),
('cp', cp),
('train_end_cp', train_end_cp),
# ("load_state", load_state),
('early_stoping', EarlyStopping(patience=5)),
('lr_scheduler', LRScheduler(
policy=ReduceLROnPlateau, monitor='valid_loss')),
],
)
print("Begin training")
try:
y_train = np.concatenate((
np.zeros((100, )), np.ones((100, )))).astype('float32')
net.fit(train_dataset, y_train)
except KeyboardInterrupt:
net.save_params(f_params=run+'.pkl')
net.save_params(f_params=run + '.pkl')
print("Finish training")
from skorch.callbacks import Checkpoint, EarlyStopping, EpochScoring
early_stopping = EarlyStopping(patience=30)
cp = Checkpoint(dirname='', f_criterion=None, f_optimizer=None, f_history=None)
clf = EEGClassifier(
model,
# criterion=torch.nn.NLLLoss,
criterion=torch.nn.CrossEntropyLoss,
optimizer=torch.optim.AdamW,
train_split=predefined_split(valid_set), # using valid_set for validation
optimizer__lr=lr,
optimizer__weight_decay=weight_decay,
batch_size=batch_size,
callbacks=[
"accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)),('cp', cp),('patience', early_stopping)
],
device=device,
)
# Model training for a specified number of epochs. `y` is None as it is already supplied
# in the dataset.
clf.fit(train_set, y=None, epochs=n_epochs)
clf.load_params(checkpoint=cp) # Load the model with the lowest valid_loss
import os
os.remove('./params.pt') # Delete parameters file
######################################################################
# Save the feature extractor model
# ------------
def exp(subject_id):
import torch
test_subj = np.r_[subject_id]
print('test subj:' + str(test_subj))
# train_subj = np.setdiff1d(np.r_[1:10], test_subj)
train_subj = np.setdiff1d(np.r_[1, 3, 7, 8], test_subj)
tr = []
val = []
for ids in train_subj:
train_size = int(0.99 * len(splitted[ids]))
test_size = len(splitted[ids]) - train_size
tr_i, val_i = torch.utils.data.random_split(splitted[ids],
[train_size, test_size])
tr.append(tr_i)
val.append(val_i)
train_set = torch.utils.data.ConcatDataset(tr)
valid_set = torch.utils.data.ConcatDataset(val)
valid_set = BaseConcatDataset([splitted[ids] for ids in test_subj])
######################################################################
# Create model
# ------------
#
######################################################################
# Now we create the deep learning model! Braindecode comes with some
# predefined convolutional neural network architectures for raw
# time-domain EEG. Here, we use the shallow ConvNet model from `Deep
# learning with convolutional neural networks for EEG decoding and
# visualization <https://arxiv.org/abs/1703.05051>`__. These models are
# pure `PyTorch <https://pytorch.org>`__ deep learning models, therefore
# to use your own model, it just has to be a normal PyTorch
# `nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__.
#
import torch
from braindecode.util import set_random_seeds
from braindecode.models import ShallowFBCSPNet, Deep4Net
cuda = torch.cuda.is_available(
) # check if GPU is available, if True chooses to use it
device = 'cuda:0' if cuda else 'cpu'
if cuda:
torch.backends.cudnn.benchmark = True
seed = 20200220 # random seed to make results reproducible
# Set random seed to be able to reproduce results
set_random_seeds(seed=seed, cuda=cuda)
n_classes = 3
# Extract number of chans and time steps from dataset
n_chans = train_set[0][0].shape[0]
input_window_samples = train_set[0][0].shape[1]
#
# model = ShallowFBCSPNet(
# n_chans,
# n_classes,
# input_window_samples=input_window_samples,
# final_conv_length='auto',
# )
from mynetworks import Deep4Net_origin, ConvClfNet, FcClfNet
model = Deep4Net(
n_chans,
n_classes,
input_window_samples=input_window_samples,
final_conv_length="auto",
)
#
# embedding_net = Deep4Net_origin(4, 22, input_window_samples)
# model = FcClfNet(embedding_net)
# #
print(model)
# Send model to GPU
if cuda:
model.cuda()
######################################################################
# Training
# --------
#
######################################################################
# Now we train the network! EEGClassifier is a Braindecode object
# responsible for managing the training of neural networks. It inherits
# from skorch.NeuralNetClassifier, so the training logic is the same as in
# `Skorch <https://skorch.readthedocs.io/en/stable/>`__.
#
######################################################################
# **Note**: In this tutorial, we use some default parameters that we
# have found to work well for motor decoding, however we strongly
# encourage you to perform your own hyperparameter optimization using
# cross validation on your training data.
#
from skorch.callbacks import LRScheduler
from skorch.helper import predefined_split
from braindecode import EEGClassifier
# # These values we found good for shallow network:
lr = 0.0625 * 0.01
weight_decay = 0
# For deep4 they should be:
# lr = 1 * 0.01
# weight_decay = 0.5 * 0.001
batch_size = 8
n_epochs = 100
clf = EEGClassifier(
model,
criterion=torch.nn.NLLLoss,
optimizer=torch.optim.AdamW,
train_split=predefined_split(
valid_set), # using valid_set for validation
optimizer__lr=lr,
optimizer__weight_decay=weight_decay,
batch_size=batch_size,
callbacks=[
"accuracy",
("lr_scheduler",
LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)),
],
device=device,
)
# Model training for a specified number of epochs. `y` is None as it is already supplied
# in the dataset.
clf.fit(train_set, y=None, epochs=n_epochs)
######################################################################
# Plot Results
# ------------
#
######################################################################
# Now we use the history stored by Skorch throughout training to plot
# accuracy and loss curves.
#
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
import pandas as pd
# Extract loss and accuracy values for plotting from history object
results_columns = [
'train_loss', 'valid_loss', 'train_accuracy', 'valid_accuracy'
]
df = pd.DataFrame(clf.history[:, results_columns],
columns=results_columns,
index=clf.history[:, 'epoch'])
# get percent of misclass for better visual comparison to loss
df = df.assign(train_misclass=100 - 100 * df.train_accuracy,
valid_misclass=100 - 100 * df.valid_accuracy)
plt.style.use('seaborn')
fig, ax1 = plt.subplots(figsize=(8, 3))
df.loc[:, ['train_loss', 'valid_loss']].plot(ax=ax1,
style=['-', ':'],
marker='o',
color='tab:blue',
legend=False,
fontsize=14)
ax1.tick_params(axis='y', labelcolor='tab:blue', labelsize=14)
ax1.set_ylabel("Loss", color='tab:blue', fontsize=14)
ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis
df.loc[:, ['train_misclass', 'valid_misclass']].plot(ax=ax2,
style=['-', ':'],
marker='o',
color='tab:red',
legend=False)
ax2.tick_params(axis='y', labelcolor='tab:red', labelsize=14)
ax2.set_ylabel("Misclassification Rate [%]", color='tab:red', fontsize=14)
ax2.set_ylim(ax2.get_ylim()[0], 85) # make some room for legend
ax1.set_xlabel("Epoch", fontsize=14)
# where some data has already been plotted to ax
handles = []
handles.append(
Line2D([0], [0],
color='black',
linewidth=1,
linestyle='-',
label='Train'))
handles.append(
Line2D([0], [0],
color='black',
linewidth=1,
linestyle=':',
label='Valid'))
plt.legend(handles, [h.get_label() for h in handles], fontsize=14)
plt.tight_layout()
# plt.show()
return df
import os
def setup_dirs():
for dir in [
"models", "optimizers", "outputs", "train_histories",
"feature_vectors"
]:
if not os.path.exists(dir):
os.makedirs(dir)
if __name__ == "__main__":
setup_dirs()
device = torch.device("cuda:0" if torch.cuda.is_available else "cpu")
lrscheduler = LRScheduler(policy='StepLR', step_size=10, gamma=0.1)
early_stopping = EarlyStopping(monitor="valid_loss",
patience=5,
threshold=0.01)
csv_path = "./dataset.csv"
train_df = pd.read_csv(csv_path)
train_df['num_label'], _ = pd.factorize(
train_df['label']) # turn labels (string) into numbers
train_df = train_df[train_df['type'] == "train"]
img_path = train_df['path'].values
labels = train_df['num_label'].values
X_train, X_val, y_train, y_val = train_test_split(img_path,
labels,
test_size=0.1,
stratify=labels,
random_state=42)
def fit(self, X, y, *args, w=None, **kwargs):
# Determine optional parameters
if self.claim_count_names is None:
self.claim_count_names = [
"claim_count_{}".format(x) for x in range(0, int(y.shape[1] / 2))
]
if self.claim_paid_names is None:
self.claim_paid_names = [
"claim_paid_{}".format(x) for x in range(0, int(y.shape[1] / 2))
]
if self.feature_dimension is None:
self.feature_dimension = X.shape[1]
if self.output_dimension is None:
self.output_dimension = len(self.claim_paid_names)
if self.categorical_dimensions is None:
self.categorical_dimensions = []
# TODO: This is a bit slow and unstable, is there a better way?
for i in range(X.shape[1]):
X_int = X[:, i].astype(int)
if np.all((X_int - X[:, i]) == 0):
self.categorical_dimensions += [(i, np.max(X_int))]
print(
"Auto detect categorical dimensions to be: {}".format(
self.categorical_dimensions
)
)
# Standardize outputs
# self.X_mean = np.mean(X, axis=0)
# self.X_std = np.std(X, axis=0)
# Except categoricals
# for i, j in self.categorical_dimensions:
# self.X_mean[i] = 0
# self.X_std[i] = 1
# X = (X - self.X_mean) / self.X_std
# Shuffle X, y
X, y = shuffle(X, y, random_state=0)
if w is None:
w = np.where(np.isnan(y), 0.0, 1.0)
else:
w = w * np.where(np.isnan(y), 0.0, 1.0)
X = np.hstack([X, w]).astype(np.float32)
y_mean = np.nanmean(y, axis=0)
y = np.hstack([y, y, y])
y = np.where(np.isnan(y), 0, np.maximum(EPSILON, y)).astype(np.float32)
earlystop = EarlyStopping(patience=self.patience, threshold=0.0)
gradclip = GradientNormClipping(gradient_clip_value=self.clipnorm)
if X.shape[0] < self.batch_size:
print("NOTE: Data size is small, outcomes may be odd.")
batch_size = X.shape[0]
else:
batch_size = self.batch_size
# One cycle policy (with Adam)
# Step 1: LR Range Finder
# Test which values fit
# Use earlystop to get an idea of epochs to 1 cycle policy as well.
for lr in self.lr_range:
super(PUNPPCIClaimRegressor, self).__init__(
PUNPPCIClaimModule(
feature_dim=self.feature_dimension,
output_dim=self.output_dimension,
cat_dim=self.categorical_dimensions,
y_mean=y_mean,
layer_size=self.layer_size,
device=self.device,
),
*args,
**kwargs,
max_epochs=self.max_epochs,
lr=lr,
device=self.device,
optimizer=self.optimizer,
# optimizer__momentum=self.momentum,
optimizer__param_groups=[
("embeddings_linear*", {"weight_decay": self.l1_l2_linear}),
(
"embeddings_residual*",
{"weight_decay": self.l2_weights_residual},
),
("dense_pricing*", {"weight_decay": self.l2_weights_residual}),
("count_linear_0.weight", {"weight_decay": self.l1_l2_linear}),
("paid_linear_0.weight", {"weight_decay": self.l1_l2_linear}),
(
"count_residual_spread.bias",
{"weight_decay": self.l2_bias_residual},
),
(
"paid_residual_spread.bias",
{"weight_decay": self.l2_bias_residual},
),
("count_residual_0.bias", {"weight_decay": self.l2_bias_residual}),
("paid_residual_0.bias", {"weight_decay": self.l2_bias_residual}),
],
batch_size=batch_size,
criterion=nn.MSELoss,
callbacks=[gradclip, earlystop],
verbose=0
)
self.initialize_module()
super(PUNPPCIClaimRegressor, self).fit(X, y)
if not np.isnan(self.history[-1]["valid_loss"]):
self.lr_min = self.lr_range[-1]
self.lr_max = lr
break
# Still broke?
if np.isnan(self.history[-1]["valid_loss"]):
warn(
"This model may fail to converge on the data. Please review data and parameters."
)
self.lr_min = self.lr_range[-1]
self.lr_max = 0.001
print("Setting maximum learn rate to {}.".format(self.lr_max))
# Step 2: Cyclic LR with expected epoch count...
valid_losses = [x["valid_loss"] for x in self.history]
expected_epoch_count = valid_losses.index(min(valid_losses)) + 1
expected_epoch_count = int(np.ceil(expected_epoch_count / 2) * 2)
expected_epoch_count = 4 if expected_epoch_count < 4 else expected_epoch_count
print("Setting epochs for training model to {}".format(expected_epoch_count))
cyclic_lr = LRScheduler(
policy=CyclicLR,
base_lr=self.lr_min,
max_lr=self.lr_max,
step_size_up=expected_epoch_count / 2,
step_size_down=expected_epoch_count / 2,
)
# ... but still keep training for as many epochs as required.
super(PUNPPCIClaimRegressor, self).__init__(
PUNPPCIClaimModule(
feature_dim=self.feature_dimension,
output_dim=self.output_dimension,
cat_dim=self.categorical_dimensions,
y_mean=y_mean,
layer_size=self.layer_size,
device=self.device,
),
max_epochs=expected_epoch_count,
lr=self.lr_min,
device=self.device,
optimizer=self.optimizer,
# optimizer__momentum=self.momentum,
optimizer__param_groups=[
("embeddings_linear*", {"weight_decay": self.l1_l2_linear}),
("embeddings_residual*", {"weight_decay": self.l2_weights_residual}),
("dense_pricing*", {"weight_decay": self.l2_weights_residual}),
("count_linear_0.weight", {"weight_decay": self.l1_l2_linear}),
("paid_linear_0.weight", {"weight_decay": self.l1_l2_linear}),
("count_residual_spread.bias", {"weight_decay": self.l2_bias_residual}),
("paid_residual_spread.bias", {"weight_decay": self.l2_bias_residual}),
("count_residual_0.bias", {"weight_decay": self.l2_bias_residual}),
("paid_residual_0.bias", {"weight_decay": self.l2_bias_residual}),
],
batch_size=batch_size,
criterion=nn.MSELoss,
callbacks=[
CheckNaN(),
# CheckMean(X, self.output_dimension, 1), # expected_epoch_count
cyclic_lr,
gradclip,
# earlystop,
],
)
self.initialize_module()
super(PUNPPCIClaimRegressor, self).fit(X, y)
# Finished fitting!
self.is_fitted_ = True
def prepare_non_learnt_model(model_args, path_tfms, is_meta=False, verbose=2):
"""Prepares a model that does not use learnt projections.
We have a separate function for learnt and non-learnt transforms since the learnt transform requires signatures to
be computed each time we evaluate over a batch. If the model is non-learnt then all signatures can be computed in
advance which significantly reduces the training time.
Args:
model_args (dict): Experiment model arguments. See the main e
path_tfms (Pipeline): A pipeline of path transformations.
is_meta (bool): Set true if the model is a dyadic-meta-model.
verbose (int): Level of output verbosity.
Returns:
sklearn transformer: Trained model.
"""
# Check if sklearn
is_sklearn = check_sklearn(model_args['clf'])
# Window arguments
arg_names = ['ds_length', 'window_name', 'window_kwargs', 'depth', 'sig_tfm', 'rescaling']
window_args = {
k: v for k, v in model_args.items() if k in arg_names
}
# Setup the disintegrator
darg = model_args['disintegrations']
disintegrator = Disintegrator(size=darg) if isinstance(darg, int) else NullTransformer()
# Random projector
proj_args = model_args['num_projections'], model_args['projection_channels']
proj_bool = all([x is not None for x in proj_args])
if proj_bool:
projector = Sklearnify(RandomProjections(model_args['in_channels'], proj_args[0], proj_args[1]))
else:
projector = NullTransformer()
# Signature computation
window = SklearnComputeWindowSignature(**window_args) if is_sklearn else TrickSignature(ComputeWindowSignature(**window_args))
# Set classifier
classifier = CLASSIFIERS[model_args['clf']]
if not is_sklearn:
# We initialise with some defaults if not specified.
init_clf_params = DEFAULT_CLF_PARAMS[model_args['clf']]
clf_args = inspect.getfullargspec(classifier).args
if 'length' in clf_args:
init_clf_params['length'] = window.module.num_windows(model_args['ds_length'])
# Update if Dyadic
module = classifier(in_channels=model_args['in_channels_clf'], out_channels=model_args['out_channels'],
**init_clf_params)
if is_meta:
module = DyadicModelIndividual(
out_channels=model_args['out_channels'], dyadic_depth=window_args['depth'], hidden_channels=100,
model=module
)
classifier = NeuralNetClassifier(
module=module,
criterion=nn.BCEWithLogitsLoss if model_args['out_channels'] == 1 else nn.CrossEntropyLoss,
batch_size=64,
verbose=verbose,
optimizer=optim.Adam,
iterator_train__drop_last=True,
callbacks=[
('scheduler', LRScheduler(policy='ReduceLROnPlateau')),
('val_stopping', EarlyStopping(monitor='valid_loss', patience=30)),
('checkpoint', CustomCheckpoint(monitor='valid_loss_best')),
('scorer', EpochScoring(custom_scorer, lower_is_better=False, name='true_acc'))
],
train_split=CVSplit(cv=5, random_state=1, stratified=True),
device=device if model_args['gpu'] else 'cpu',
)
model = Pipeline([
*path_tfms,
('disintegrator', disintegrator),
('random_projections', projector),
('signatures', window),
('classifier', classifier)
])
return model
out = self.model(x)
return out
class ClearCache(Callback):
def on_batch_end(self, net, X=None, y=None, training=None, **kwargs):
torch.cuda.empty_cache()
def on_train_begin(self, net, X=None, y=None, **kwargs):
print(f'\nModel {model_num}')
clear_cache = ClearCache()
lrscheduler = LRScheduler(policy='StepLR', step_size=25, gamma=0.25)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
data_dir = 'data'
normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
train_transforms = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomGrayscale(p=0.35),
transforms.RandomRotation((-11, 11)),
transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor(), normalize
])
# drop_bad_windows=True,
# )
# splits = dataset.split("session")
# train_set = splits["train"]
# valid_set = splits["eval"]
regressor = EEGRegressor(
model,
cropped=True,
criterion=CroppedLoss,
criterion__loss_function=torch.nn.functional.mse_loss,
optimizer=torch.optim.AdamW,
train_split=predefined_split(valid_set),
optimizer__lr=optimizer_lr,
optimizer__weight_decay=optimizer_weight_decay,
iterator_train__shuffle=True,
batch_size=batch_size,
callbacks=[
"neg_root_mean_squared_error",
# seems n_epochs -1 leads to desired behavior of lr=0 after end of data?
("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)),
],
device=device,
)
regressor.fit(train_set, y=None, epochs=n_epochs)
if __name__ == '__main__':
create_compatible_dataset('./data/BCICIV_4_mat/sub1_comp.mat')
def fit_model(args, X, y):
print(f'Fitting model for {args.model}:')
auc = EpochScoring(scoring='roc_auc', lower_is_better=False)
apr = EpochScoring(scoring='average_precision', lower_is_better=False)
lrs = LRScheduler(policy='StepLR', step_size=10, gamma=0.5)
if args.model == 'glm':
glm = LogitNet(alpha=0.5, n_lambda=50, n_jobs=-1)
glm.fit(X, y)
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1111)
net = LogitNet(alpha=0.5, n_lambda=1, lambda_path=[glm.lambda_best_])
if args.model == 'standard':
net = NeuralNetClassifier(
models.MpraDense,
batch_size=256,
optimizer=torch.optim.Adam,
optimizer__weight_decay=2e-6,
lr=1e-4,
max_epochs=20,
module__n_input=1079,
module__n_units=(400, 250),
module__dropout=0.3,
callbacks=[auc, apr],
iterator_train__shuffle=True,
train_split=None
)
elif args.model == 'neighbors':
net = NeuralNetClassifier(
models.MpraFullCNN,
batch_size=256,
optimizer=torch.optim.Adam,
optimizer__weight_decay=1e-2,
lr=5e-5,
max_epochs=20,
callbacks=[auc, apr],
iterator_train__shuffle=True,
train_split=None
)
# generate CV predictions
np.random.seed(1000)
torch.manual_seed(1000)
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1000)
cv_scores = cross_val_predict(net, X, y, cv=kf,
method='predict_proba', n_jobs=-1)
AUC = roc_auc_score(y, cv_scores[:, 1])
APR = average_precision_score(y, cv_scores[:, 1])
print('\tAUC ', np.round(AUC, 4))
print('\tAPR ', np.round(APR, 4))
save_scores(args, cv_scores[:, 1], y)
# refit and store model on all data
net.fit(X, y)
save_model(net, args.project, args.model)
def build_estimator(hyperparams, train_data, test=False):
device = "cuda" if torch.cuda.is_available() else "cpu"
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
n_classes = len(np.unique(y))
n_samples = y.shape[0]
bal_weights = (torch.from_numpy(
n_samples / (n_classes * np.bincount(y))).float().to(device))
callbacks = [
(
"f1_score_valid",
EpochScoring(
"f1" if n_classes == 2 else "f1_macro",
name="valid_f1",
lower_is_better=False,
),
),
(
"early_stopping",
EarlyStopping(monitor="valid_loss",
patience=5,
lower_is_better=True),
),
(
"learning_rate_scheduler",
LRScheduler(
policy=lr_scheduler.ReduceLROnPlateau,
monitor="valid_loss",
# Following kargs are passed to the
# lr scheduler constructor
mode="min",
min_lr=1e-5,
),
),
]
return NeuralNetClassifier(
NNModule,
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5,
stratified=True,
random_state=RANDOM_STATE),
lr=hyperparams["lr"],
batch_size=hyperparams["batch_size"],
module__in_features=in_features,
module__n_classes=n_classes,
module__n_layers=hyperparams["n_layers"],
module__n_neuron_per_layer=hyperparams["n_neuron_per_layer"],
module__activation=getattr(F, hyperparams["activation"]),
module__p_dropout=hyperparams["p_dropout"],
criterion__weight=bal_weights
if hyperparams["class_weight"] == "balanced" else None,
optimizer__momentum=hyperparams["momentum"],
optimizer__weight_decay=hyperparams["weight_decay"],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4,
)
calc_fp_valid = EpochScoring(calc_fp,
lower_is_better=False,
on_train=False,
name='fp_valid')
cp = Checkpoint(monitor='precision_valid',
f_history=os.path.join(
args.output_dir,
args.output_filename_prefix + '.json'))
train_end_cp = TrainEndCheckpoint(dirname=args.output_dir,
fn_prefix=args.output_filename_prefix)
lr_scheduler = LRScheduler(policy=ReduceLROnPlateau,
mode='max',
factor=0.1,
patience=3,
min_lr=1e-04,
verbose=True)
# n_cv_folds = int(np.floor(1/args.validation_size))
## start training
fixed_precision = 0.2
thr_list = [0.5]
if args.scoring == 'cross_entropy_loss':
mlp = SkorchMLP(
n_features=len(feature_cols),
l2_penalty_weights=args.weight_decay,
lr=args.lr,
batch_size=args.batch_size,
max_epochs=max_epochs,
train_split=predefined_split(valid_ds),