def __init__(self,
net,
early_stopping: EarlyStopping = None,
**l1_regularized_nnr_params):
self.dataset = None
self.early_stopping = early_stopping
self.cpu_count = psutil.cpu_count(logical=True)
cp['checkpoint'] = Checkpoint(dirname=tempfile.gettempdir())
if 'train_split' in l1_regularized_nnr_params:
train_split = l1_regularized_nnr_params.get('train_split')
if train_split is None:
cp['checkpoint'].monitor = 'train_loss_best'
if early_stopping is not None:
if early_stopping.monitor == 'valid_loss':
raise ValueError(
# pylint: disable=line-too-long
'Select a valid train_split or disable early_stopping! A valid train_split needs to be selected when valid_loss monitor is selected as early stopping criteria.'
)
if 'device' in l1_regularized_nnr_params:
device = l1_regularized_nnr_params.get('device')
else:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
l1_regularized_nnr_params['device'] = device
self.l1_regularized_nnr_params = l1_regularized_nnr_params
self.ttr = TransformedTargetRegressor(regressor=self._get_pipeline(
net, cp['checkpoint']),
transformer=Scaler())
def train(data_folder: str, out_model: str):
out_model = Path(out_model)
out_model.mkdir()
data_paths = list(Path(data_folder).rglob("*.npy"))
train_paths, valid_paths = train_test_split(data_paths, train_size=0.7)
train_dataset = LibriSpeechDataset(
train_paths,
Path(data_folder).parent / "SPEAKERS.TXT",
Compose([ExtractStft(),
RandomCrop(constants.STFT_CROP_WIDTH)]))
valid_dataset = LibriSpeechDataset(
valid_paths,
Path(data_folder).parent / "SPEAKERS.TXT",
Compose([ExtractStft(),
RandomCrop(constants.STFT_CROP_WIDTH)]))
net = NeuralNet(Classifier,
module__n_classes=constants.NUMBER_OF_CLASSES,
criterion=nn.CrossEntropyLoss,
batch_size=8,
max_epochs=100,
optimizer=optim.Adam,
lr=0.001,
iterator_train__shuffle=True,
iterator_train__num_workers=2,
iterator_valid__shuffle=False,
iterator_valid__num_workers=2,
train_split=predefined_split(valid_dataset),
device="cuda",
callbacks=[
Checkpoint(
f_params=(out_model / "params.pt").as_posix(),
f_optimizer=(out_model / "optim.pt").as_posix(),
f_history=(out_model / "history.pt").as_posix()),
ProgressBar(postfix_keys=["train_loss", "train_acc"]),
EarlyStopping(),
EpochScoring(acc,
name="val_acc",
lower_is_better=False,
on_train=False),
EpochScoring(acc,
name="train_acc",
lower_is_better=False,
on_train=True),
Tensorboard((out_model / "train").as_posix(),
metrics={"acc": acc_as_metric},
is_training=True),
Tensorboard((out_model / "valid").as_posix(),
metrics={"acc": acc_as_metric},
is_training=False),
])
net.fit(train_dataset)
def main(args):
# Step 1: Fetch Datasets for training/validation/test datasets
train_set, val_set, test_set = get_datasets(args)
# Step 2: Initialize callbacks for NeuralNetClassifier
lrscheduler = LRScheduler(
policy='StepLR', step_size=7, gamma=0.1)
checkpoint = Checkpoint(
f_params='best_model.pt', monitor='valid_acc_best')
freezer = Freezer(lambda x: not x.startswith('model.classifier.1'))
net = NeuralNetClassifier(
TwoClassSqueezeNet,
criterion=nn.CrossEntropyLoss,
batch_size=args.batch_size,
max_epochs=args.num_epochs,
module__num_classes=args.num_classes,
optimizer=optim.SGD,
iterator_train__shuffle=True,
iterator_train__num_workers=args.num_workers,
iterator_valid__shuffle=True,
iterator_valid__num_workers=args.num_workers,
# train_split fixes bug in skorch library, see:
# https://github.com/skorch-dev/skorch/issues/599
train_split=None,
device='cuda' # comment to train on cpu
)
params = {
'optimizer__lr': [1e-5, 1e-4, 1e-3],
'optimizer__momentum': [0.5, 0.9, 0.99, 0.999],
'optimizer__nesterov': [True, False]
}
gs = GridSearchCV(net, params, refit=False, cv=3, scoring='accuracy',
verbose=10)
X_sl = SliceDataset(train_set, idx=0) # idx=0 is the default
y_sl = SliceDataset(train_set, idx=1)
# net.fit(train_set, y=None)
gs.fit(X_sl, y_sl)
print(gs.best_score_, gs.best_params_)
def calorie_model(val_ds):
lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1)
checkpoint = Checkpoint(f_params='models/calorie_net.pt',
monitor='valid_acc_best')
return NeuralNet(CalorieNet,
criterion=nn.MSELoss(),
lr=0.001,
batch_size=64,
max_epochs=25,
optimizer=optim.SGD,
optimizer__momentum=0.9,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
iterator_valid__shuffle=True,
iterator_valid__num_workers=4,
train_split=predefined_split(val_ds),
callbacks=[lrscheduler, checkpoint],
device='cuda')
def _default_callbacks(self):
default_cb_list = [('epoch_timer', EpochTimer()),
('train_loss',
BatchScoring(train_loss_score,
name='train_loss',
on_train=True,
target_extractor=noop)),
('valid_loss',
BatchScoring(valid_loss_score,
name='valid_loss',
target_extractor=noop)),
('valid_acc',
EpochScoring(
'accuracy',
name='valid_acc',
lower_is_better=False,
)),
('checkpoint', Checkpoint(dirname=self.model_path)),
('end_checkpoint',
TrainEndCheckpoint(dirname=self.model_path)),
('report', ReportLog()),
('progressbar', ProgressBar())]
if 'stop_patience' in self.hyperparamters.keys() and \
self.hyperparamters['stop_patience']:
earlystop_cb = ('earlystop',
EarlyStopping(
patience=self.hyperparamters['stop_patience'],
threshold=1e-4))
default_cb_list.append(earlystop_cb)
if 'lr_step' in self.hyperparamters.keys() and \
self.hyperparamters['lr_step']:
lr_callback = ('lr_schedule',
DecayLR(self.hyperparamters['lr'],
self.hyperparamters['lr_step'],
gamma=0.5))
default_cb_list.append(lr_callback)
return default_cb_list
def prepare_NN(self, num_classes):
class ThreeLayerNN(nn.Module):
def __init__(self, hidden, num_classes, activation):
super(ThreeLayerNN, self).__init__()
self.hidden = hidden
self.num_classes = num_classes
self.activation = activation
self.Layer1 = nn.Linear(768, self.hidden, bias=True)
self.Layer2 = nn.Linear(self.hidden, self.hidden, bias=True)
self.Output = nn.Linear(self.hidden, self.num_classes, bias=True)
def forward(self, x):
z1 = self.Layer1(x)
a1 = self.activation(z1)
z2 = self.Layer2(a1)
a2 = self.activation(z2)
y_hat = F.softmax(a2)
return(y_hat)
monitor = lambda net: all(net.history[-1, ("train_loss_best", "valid_loss_best")])
net = NeuralNetClassifier(module=ThreeLayerNN,
module__hidden=self.hidden,
module__num_classes=num_classes,
module__activation=self.activation,
max_epochs=1000,
lr=0.1,
iterator_train__shuffle=True,
callbacks=[Checkpoint(monitor=monitor),
EarlyStopping(patience=5,
threshold=0.0001),
LRScheduler()],
verbose=0,
device=self.device,
batch_size=32)
return(net)
drop1=0.2):
super().init()
model = torchvision.models.mobilenet_v2(pretrained=False)
n_inputs = model.classifier[1].in_features
model.classifier = nn.Sequential(nn.Dropout(p=drop1),
nn.Linear(n_inputs, output_features))
self.model = model
def forward(self, x):
return self.model(x)
lr_scheduler_mobilenet = LRScheduler(policy='StepLR', step_size=8, gamma=0.2)
# callback for saving the best on validation accuracy model
checkpoint_mobilenet = Checkpoint(
f_params='/content/drive/MyDrive/chess_weights/best_model_mobilenet.pkl',
monitor='valid_acc_best')
# callback for freezing all layer of the model except the last layer
#freezer_vgg = Freezer(lambda x: not x.startswith('model.classifier'))
# callback for early stopping
early_stopping_mobilenet = EarlyStopping(patience=10)
mobilenet = NeuralNetClassifier(
# pretrained ResNet50 + custom classifier
module=MobileNet,
# fine tuning model's inner parameters
module__output_features=13,
module__num_units=512,
module__drop=0.5,
module__num_units1=512,
module__drop1=0.5,
# criterion
#print('Passed Thru VGG', x1)
y = self.final_layer(x1)
#print(y, 'y')
#y_pred=self.log_softmax(y)
#print(y_pred, 'y_pred')
return y
model = Vgg16Module()
#print(vgg16)
#### Model Callbacks
lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1)
checkpoint = Checkpoint(dirname='exp',
f_params='best_model.pt',
monitor='train_loss_best')
#### Neural Net Classifier
net = NeuralNetClassifier(
module=model,
criterion=nn.CrossEntropyLoss,
lr=args.learning_rate,
batch_size=args.batch_size,
max_epochs=args.num_epochs,
optimizer=optim.SGD,
optimizer__momentum=args.momentum,
train_split=None,
#callbacks=[lrscheduler],
device=args.device # comment to train on cpu
)
def train_calc(inputs):
images, filename, file_dir, Gs, lj, forcesonly, scaling = inputs
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params(
"./results/checkpoints/{}_params.pt".format(filename))
cp = Checkpoint(
monitor="forces_score_best",
fn_prefix="./results/checkpoints/{}_".format(filename),
)
if not os.path.exists(file_dir):
os.makedirs(file_dir, exist_ok=True)
forcetraining = True
training_data = AtomsDataset(
images,
SNN_Gaussian,
Gs,
forcetraining=forcetraining,
label=filename,
cores=1,
lj_data=None,
scaling=scaling,
)
unique_atoms = training_data.elements
fp_length = training_data.fp_length
device = "cpu"
torch.set_num_threads(1)
net = NeuralNetRegressor(
module=FullNN(unique_atoms, [fp_length, 3, 20],
device,
forcetraining=forcetraining),
criterion=CustomMSELoss,
criterion__force_coefficient=0.04,
optimizer=torch.optim.LBFGS,
lr=1e-1,
batch_size=len(training_data),
max_epochs=200,
iterator_train__collate_fn=collate_amp,
iterator_train__shuffle=False,
iterator_valid__collate_fn=collate_amp,
iterator_valid__shuffle=False,
device=device,
train_split=CVSplit(cv=5, random_state=1),
callbacks=[
EpochScoring(
forces_score,
on_train=False,
use_caching=True,
target_extractor=target_extractor,
),
EpochScoring(
energy_score,
on_train=False,
use_caching=True,
target_extractor=target_extractor,
),
],
)
calc = AMP(training_data, net, label=filename)
calc.train()
return [training_data, net, filename]
from braindecode import EEGClassifier
# These values we found good for shallow network:
lr = 0.0625 * 0.01
weight_decay = 0
# For deep4 they should be:
# lr = 1 * 0.01
# weight_decay = 0.5 * 0.001
batch_size = 64
n_epochs = 100
from skorch.callbacks import Checkpoint, EarlyStopping, EpochScoring
early_stopping = EarlyStopping(patience=30)
cp = Checkpoint(dirname='', f_criterion=None, f_optimizer=None, f_history=None)
clf = EEGClassifier(
model,
# criterion=torch.nn.NLLLoss,
criterion=torch.nn.CrossEntropyLoss,
optimizer=torch.optim.AdamW,
train_split=predefined_split(valid_set), # using valid_set for validation
optimizer__lr=lr,
optimizer__weight_decay=weight_decay,
batch_size=batch_size,
callbacks=[
"accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)),('cp', cp),('patience', early_stopping)
],
device=device,
def train_model_adam(dic_param, log_exp_run, wdir, device, tensor_embedding,
train_data, test_data, gscv_best_model):
# Defining a param distribution for hyperparameter-tuning for model and fit params
param_grid = {
'lr': dic_param['alpha_distribution'],
'mode': ["Adam"] # Modes: Adam,SGD
}
fit_param = {
'patientia': dic_param['sgd_early_stopping_patientia'],
'min_diference': dic_param['sgd_min_difference'],
'checkpoint_path': wdir + "checkpoints/"
}
checkpoint = Checkpoint(dirname=fit_param['checkpoint_path'],
f_params=dic_param['f_params_name'],
f_optimizer=dic_param['f_optimizer_name'],
f_history=dic_param['f_history_name'],
f_criterion=dic_param['f_criterion_name'],
monitor=None)
load_state = LoadInitState(checkpoint)
# Defining skorch-based neural network
model = Trainer(
module=ModelCNN,
module__word_embedding_size=dic_param['word_embedding_size'],
module__labels=dic_param['labels'],
module__weights_tensor=tensor_embedding,
module__batch_size=dic_param['sgd_batch_size'],
max_epochs=dic_param['epochs_gs_cv'],
iterator_train__shuffle=True,
criterion=torch.nn.CrossEntropyLoss,
train_split=None,
device=device,
callbacks=[checkpoint],
optimizer=torch.optim.Adam,
mode="Adam"
# optimizer__weight_decay=dic_param['l2_reg'] #L2 regularization
)
# model.initialize()
# print(summary(model.module_,torch.zeros((1,1000),dtype=torch.long), show_input=True))
# Defining GridSearch using k-fold cross validation
log_exp_run.experiments(
"GridSearch using k-fold cross validation with for Adam")
start_time = time.time()
gs = GridSearchCV(model,
param_grid,
cv=dic_param['grid_search_cross_val_cv'],
verbose=2)
if gscv_best_model is None:
gs.fit(train_data, fit_param=fit_param)
log_exp_run.experiments(
"Time elapsed for GridSearch using k-fold cross validation with k=5 for Adam: "
+ str(time.time() - start_time))
log_exp_run.experiments("Best param estimated for Adam: ")
log_exp_run.experiments(gs.best_params_)
log_exp_run.experiments("Best score for Adam: ")
log_exp_run.experiments(gs.best_score_)
log_exp_run.experiments("GridSearch scores")
log_exp_run.experiments(gs.cv_results_)
gscv_best_model = gs.best_estimator_
best_model = gscv_best_model
best_model.set_params(max_epochs=dic_param['epochs'])
start_time = time.time()
best_model.fit(train_data, fit_param=fit_param)
log_exp_run.experiments("Time elapsed for Adam : " +
str(time.time() - start_time))
best_model.score(test_data)
best_model.score(train_data)
log_exp_run.experiments("Adam as optimizer: Process ends successfully!")
log_exp_run.experiments("--------------------------\n\n\n")
return gscv_best_model
# for i in range(NUM_GPUs):
# CUDA_VISIBLE_DEVICES=i dask-worker 127.0.0.1:8786 --nthreads 1
# CUDA_VISIBLE_DEVICES=i dask-worker 127.0.0.1:8786 --nthreads 1
#######################################################################
# CALLBACK FUNCTIONS
#######################################################################
################## skorch_callbacks ###################################
# checkpoint saver
# monitor = lambda net: all(net.history[-1, (
# 'train_loss_best', 'valid_loss_best')])
cp_best_model = Checkpoint(
monitor='valid_loss_best',
# monitor = monitor,
dirname='results/model_checkpoints/best_model',
)
cp_best_train = Checkpoint(
monitor='train_loss_best',
# monitor = monitor,
dirname='results/model_checkpoints/best_train',
)
# learning rate scheduler
cyclicLR = skorch.callbacks.LRScheduler(policy='CyclicLR', )
# lr = skorch.callbacks.CyclicLR(optimizer = Adam)
# lr_scheduler = LRScheduler(policy="StepLR", step_size=7, gamma=0.1)
#display progressbar == True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Using " + torch.cuda.get_device_name(device))
# Prepare data
print("Prepare data")
train_feat = get_audio_feature(
train_dir, negative_num=100, positive_num=100)
test_feat = get_audio_feature(
test_dir, test_num=100)
train_dataset = AudioDataset(
train_feat=train_feat, test_feat=test_feat, negative_num=100, positive_num=100)
cp_from_final = Checkpoint(
dirname='checkpoints/' + run, fn_prefix='from_train_end_')
cp = Checkpoint(
dirname='checkpoints/' + run)
train_end_cp = TrainEndCheckpoint(dirname='checkpoints/'+run)
load_state = LoadInitState(train_end_cp)
net = NeuralNetBinaryClassifier(
module=AudioNet,
criterion=nn.BCEWithLogitsLoss,
max_epochs=5000,
lr=0.01,
optimizer=optim.SGD,
optimizer__momentum=0.9,
batch_size=160,
device=device,
callbacks=[
net = Net(
# Architecture
module=MLP,
module__input_dim=input_dim,
module__output_dim=output_dim,
module__hidden_dim=opt['hidden_layer_dim'],
optimizer__weight_decay=opt['l2_weight'],
module__dropout=opt['dropout'],
device='cuda',
# Training
max_epochs=opt['max_epochs'],
batch_size=opt['batch_size'],
callbacks=[
Checkpoint(dirname=save_dir,
f_params='params.pt',
f_optimizer=None,
f_history=None,
monitor='valid_loss_best')
],
# train_split is validation data
train_split=predefined_split(Dataset(X_val, y_val)),
# Optimizer
optimizer=optim.Adam,
lr=opt['learning_rate'],
# Data
iterator_train__shuffle=True,
verbose=(runs == 1))
net.fit(X_train, y_train)
# Reload best valid loss checkpoint
binary=False,
save_df=True)
print("Preprocessing: Preparing for stratified sampling")
y_train = np.array([y for _, y in tqdm(iter(ds))])
print("Preprocessing: Done")
net = NeuralNetClassifier(
module=DPROMModule,
module__num_classes=2,
module__seqs_length=ds.seqs_length,
criterion=torch.nn.CrossEntropyLoss,
max_epochs=50,
lr=0.001,
callbacks=[
EarlyStopping(patience=5),
ProgressBar(),
Checkpoint(dirname=model_folder, f_params='model.pt')
],
batch_size=32,
optimizer=torch.optim.Adam,
train_split=CVSplit(cv=0.1, stratified=True),
device='cuda' if torch.cuda.is_available() else 'cpu')
print("Cross Validation: Started")
#scoring metrics can be modified. Predefined metrics: https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter
scorer = MultiScorer({
'accuracy': (accuracy_score, {}),
'precision': (precision_score, {}),
'recall': (recall_score, {}),
'mcc': (matthews_corrcoef, {})
})
cross_validate(net, ds, y_train, scoring=scorer, cv=2, verbose=1)
def evaluator(
val_split,
metric,
identifier,
forcetraining,
):
callbacks = []
isval = val_split != 0
if isval:
cp_on = "val"
else:
cp_on = "train"
if metric == "mae":
energy_score = mae_energy_score
forces_score = mae_forces_score
elif metric == "mse":
energy_score = mse_energy_score
forces_score = mse_forces_score
else:
raise NotImplementedError(f"{metric} metric not available!")
callbacks.append(
EpochScoring(
energy_score,
on_train=True,
use_caching=True,
name="train_energy_{}".format(metric),
target_extractor=target_extractor,
))
if isval:
callbacks.append(
EpochScoring(
energy_score,
on_train=False,
use_caching=True,
name="val_energy_{}".format(metric),
target_extractor=target_extractor,
))
callbacks.append(
Checkpoint(
monitor="{}_energy_{}_best".format(cp_on, metric),
fn_prefix="checkpoints/{}/".format(identifier),
))
if forcetraining:
callbacks.append(
EpochScoring(
forces_score,
on_train=True,
use_caching=True,
name="train_forces_{}".format(metric),
target_extractor=target_extractor,
))
if isval:
callbacks.append(
EpochScoring(
forces_score,
on_train=False,
use_caching=True,
name="val_forces_{}".format(metric),
target_extractor=target_extractor,
))
callbacks.append(
Checkpoint(
monitor="{}_forces_{}_best".format(cp_on, metric),
fn_prefix="checkpoints/{}/".format(identifier),
))
return callbacks
transforms.ToTensor(), normalize
])
train_ds = datasets.ImageFolder(os.path.join(data_dir, 'training'),
train_transforms)
valid_ds = datasets.ImageFolder(os.path.join(data_dir, 'validation'),
val_transforms)
model_num = 0
for rate in [0.0001]:
for arch in ['wide_resnet50_2']:
model_num += 1
checkpoint = Checkpoint(f_params=f'best_model_{model_num}.pt',
monitor='valid_acc_best')
freezer = Freezer(lambda x: not x.startswith('model.fc'))
CNN = NeuralNetClassifier(
NewResNet,
max_epochs=115,
lr=rate,
criterion=nn.CrossEntropyLoss,
device=device,
optimizer=torch.optim.Adam,
train_split=predefined_split(valid_ds),
batch_size=64,
callbacks=[checkpoint, clear_cache, lrscheduler],
iterator_train__shuffle=True,
iterator_valid__shuffle=True,
iterator_train__num_workers=4,
lower_is_better=False,
on_train=False,
name='tp_valid')
calc_fp_train = EpochScoring(calc_fp,
lower_is_better=False,
on_train=True,
name='fp_train')
calc_fp_valid = EpochScoring(calc_fp,
lower_is_better=False,
on_train=False,
name='fp_valid')
cp = Checkpoint(monitor='precision_valid',
f_history=os.path.join(
args.output_dir,
args.output_filename_prefix + '.json'))
train_end_cp = TrainEndCheckpoint(dirname=args.output_dir,
fn_prefix=args.output_filename_prefix)
lr_scheduler = LRScheduler(policy=ReduceLROnPlateau,
mode='max',
factor=0.1,
patience=3,
min_lr=1e-04,
verbose=True)
# n_cv_folds = int(np.floor(1/args.validation_size))
## start training
fixed_precision = 0.2
def main():
parser = argparse.ArgumentParser(
description='PyTorch RNN with variable-length numeric sequences wrapper'
)
parser.add_argument('--outcome_col_name', type=str, required=True)
parser.add_argument('--train_csv_files', type=str, required=True)
parser.add_argument('--valid_csv_files', type=str, required=True)
parser.add_argument('--test_csv_files', type=str, required=True)
parser.add_argument('--data_dict_files', type=str, required=True)
parser.add_argument('--batch_size',
type=int,
default=1024,
help='Number of sequences per minibatch')
parser.add_argument('--epochs',
type=int,
default=50,
help='Number of epochs')
parser.add_argument('--hidden_units',
type=int,
default=32,
help='Number of hidden units')
parser.add_argument('--hidden_layers',
type=int,
default=1,
help='Number of hidden layers')
parser.add_argument('--lr',
type=float,
default=0.0005,
help='Learning rate for the optimizer')
parser.add_argument('--dropout',
type=float,
default=0,
help='dropout for optimizer')
parser.add_argument('--weight_decay',
type=float,
default=0.0001,
help='weight decay for optimizer')
parser.add_argument('--seed', type=int, default=1111, help='random seed')
parser.add_argument('--validation_size',
type=float,
default=0.15,
help='validation split size')
parser.add_argument(
'--is_data_simulated',
type=bool,
default=False,
help='boolean to check if data is simulated or from mimic')
parser.add_argument(
'--output_dir',
type=str,
default=None,
help=
'directory where trained model and loss curves over epochs are saved')
parser.add_argument(
'--output_filename_prefix',
type=str,
default=None,
help='prefix for the training history jsons and trained classifier')
args = parser.parse_args()
torch.manual_seed(args.seed)
device = 'cpu'
x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split(
',')
x_valid_csv_filename, y_valid_csv_filename = args.valid_csv_files.split(
',')
x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',')
x_dict, y_dict = args.data_dict_files.split(',')
x_data_dict = load_data_dict_json(x_dict)
# get the id and feature columns
id_cols = parse_id_cols(x_data_dict)
feature_cols = parse_feature_cols(x_data_dict)
# extract data
train_vitals = TidySequentialDataCSVLoader(
x_csv_path=x_train_csv_filename,
y_csv_path=y_train_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_tstep')
valid_vitals = TidySequentialDataCSVLoader(
x_csv_path=x_valid_csv_filename,
y_csv_path=y_valid_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_tstep')
test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename,
y_csv_path=y_test_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_tstep')
X_train, y_train = train_vitals.get_batch_data(batch_id=0)
X_valid, y_valid = valid_vitals.get_batch_data(batch_id=0)
X_test, y_test = test_vitals.get_batch_data(batch_id=0)
N, T, F = X_train.shape
# from IPython import embed; embed()
# X_train = (X_train - np.min(X_train))/(np.max(X_train)-np.min(X_train))
# X_valid = (X_valid - np.min(X_train))/(np.max(X_train)-np.min(X_train))
# X_test = (X_test - np.min(X_train))/(np.max(X_train)-np.min(X_train))
valid_ds = Dataset(X_valid, y_valid)
print('number of time points : %s\nnumber of features : %s\n' % (T, F))
# set class weights as 1/(number of samples in class) for each class to handle class imbalance
class_weights = torch.tensor(
[1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).float()
print('Number of training sequences : %s' % N)
print('Number of test sequences : %s' % X_test.shape[0])
print('Ratio positive in train : %.2f' %
((y_train == 1).sum() / len(y_train)))
print('Ratio positive in test : %.2f' %
((y_test == 1).sum() / len(y_test)))
# callback to compute gradient norm
compute_grad_norm = ComputeGradientNorm(norm_type=2)
# LSTM
if args.output_filename_prefix == None:
output_filename_prefix = (
'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' %
(args.hidden_units, args.hidden_layers, args.lr, args.dropout,
args.weight_decay))
else:
output_filename_prefix = args.output_filename_prefix
print('RNN parameters : ' + output_filename_prefix)
loss_early_stopping_cp = EarlyStopping(monitor='valid_loss',
patience=15,
threshold=0.002,
threshold_mode='rel',
lower_is_better=True)
rnn = RNNPerTStepBinaryClassifier(
max_epochs=250,
batch_size=args.batch_size,
device=device,
lr=args.lr,
callbacks=[
EpochScoring(calc_auprc,
lower_is_better=False,
on_train=True,
name='auprc_train'),
EpochScoring(calc_auprc,
lower_is_better=False,
on_train=False,
name='auprc_valid'),
EpochScoring(calc_auroc,
lower_is_better=False,
on_train=True,
name='auroc_train'),
EpochScoring(calc_auroc,
lower_is_better=False,
on_train=False,
name='auroc_valid'),
# EpochScoring(calc_precision, lower_is_better=False, on_train=True, name='precision_train'),
# EpochScoring(calc_precision, lower_is_better=False, on_train=False, name='precision_valid'),
# EpochScoring(calc_recall, lower_is_better=False, on_train=True, name='recall_train'),
# EpochScoring(calc_recall, lower_is_better=False, on_train=False, name='recall_valid'),
# EpochScoring('roc_auc', lower_is_better=False, on_train=True, name='aucroc_score_train'),
# EpochScoring('roc_auc', lower_is_better=False, on_train=False, name='aucroc_score_valid'),
# EarlyStopping(monitor='auprc_valid', patience=5, threshold=0.002, threshold_mode='rel',
# lower_is_better=False),
# LRScheduler(policy=ReduceLROnPlateau, mode='max', monitor='aucroc_score_valid', patience=10),
# compute_grad_norm,
# GradientNormClipping(gradient_clip_value=0.5, gradient_clip_norm_type=2),
loss_early_stopping_cp,
Checkpoint(monitor='auprc_valid',
f_history=os.path.join(
args.output_dir, output_filename_prefix + '.json')),
TrainEndCheckpoint(dirname=args.output_dir,
fn_prefix=output_filename_prefix),
],
# criterion=torch.nn.CrossEntropyLoss,
# criterion__weight=class_weights,
train_split=predefined_split(valid_ds),
module__rnn_type='GRU',
module__n_layers=args.hidden_layers,
module__n_hiddens=args.hidden_units,
module__n_inputs=X_train.shape[-1],
module__dropout_proba=args.dropout,
optimizer=torch.optim.Adam,
optimizer__weight_decay=args.weight_decay)
# N=len(X_train)
# X_train = X_train[:N]
# y_train = y_train[:N]
clf = rnn.fit(X_train, y_train)
# get threshold with max recall at fixed precision
fixed_precision = 0.1
# get predict probas for y=1 on validation set
keep_inds_va = torch.logical_not(
torch.all(torch.isnan(torch.FloatTensor(X_valid)), dim=-1))
y_va_pred_proba = clf.predict_proba(
X_valid)[keep_inds_va][:, 1].detach().numpy()
unique_probas = np.unique(y_va_pred_proba)
thr_grid_G = np.linspace(np.percentile(unique_probas, 1),
max(unique_probas), 100)
precision_scores_G, recall_scores_G = [
np.zeros(thr_grid_G.size),
np.zeros(thr_grid_G.size)
]
for gg, thr in enumerate(thr_grid_G):
# logistic_clf.module_.linear_transform_layer.bias.data = torch.tensor(thr_grid[gg]).double()
curr_thr_y_preds = clf.predict_proba(
torch.FloatTensor(X_valid))[keep_inds_va][:, 1] >= thr_grid_G[gg]
precision_scores_G[gg] = precision_score(y_valid[keep_inds_va],
curr_thr_y_preds)
recall_scores_G[gg] = recall_score(y_valid[keep_inds_va],
curr_thr_y_preds)
keep_inds = precision_scores_G >= fixed_precision
if keep_inds.sum() > 0:
print('Choosing threshold with precision >= %.3f' % fixed_precision)
else:
fixed_precision_old = fixed_precision
fixed_precision = np.percentile(precision_scores_G, 99)
keep_inds = precision_scores_G >= fixed_precision
print(
'Could not find threshold with precision >= %.3f \n Choosing threshold to maximize recall at precision %.3f'
% (fixed_precision_old, fixed_precision))
thr_grid_G = thr_grid_G[keep_inds]
precision_scores_G = precision_scores_G[keep_inds]
recall_scores_G = recall_scores_G[keep_inds]
thr_perf_df = pd.DataFrame(
np.vstack([
thr_grid_G[np.newaxis, :], precision_scores_G[np.newaxis, :],
recall_scores_G[np.newaxis, :]
]).T,
columns=['thr', 'precision_score', 'recall_score'])
print(thr_perf_df)
best_ind = np.argmax(recall_scores_G)
best_thr = thr_grid_G[best_ind]
print('chosen threshold : %.3f' % best_thr)
splits = ['train', 'valid', 'test']
# data_splits = ((x_tr, y_tr), (x_va, y_va), (X_test, y_test))
auroc_per_split, auprc_per_split, precisions_per_split, recalls_per_split = [
np.zeros(len(splits)),
np.zeros(len(splits)),
np.zeros(len(splits)),
np.zeros(len(splits))
]
for ii, (X, y) in enumerate([(X_train, y_train), (X_valid, y_valid),
(X_test, y_test)]):
keep_inds = torch.logical_not(
torch.all(torch.isnan(torch.FloatTensor(X)), dim=-1))
y_pred_proba_pos = clf.predict_proba(X)[keep_inds][:,
1].detach().numpy()
# y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_per_split[ii] = roc_auc_score(y[keep_inds], y_pred_proba_pos)
# y_pred_proba_pos = np.asarray(y_pred_proba_pos)
auprc_per_split[ii] = average_precision_score(y[keep_inds],
y_pred_proba_pos)
y_pred = y_pred_proba_pos >= best_thr
precisions_per_split[ii] = precision_score(y[keep_inds], y_pred)
recalls_per_split[ii] = recall_score(y[keep_inds], y_pred)
auroc_train, auroc_valid, auroc_test = auroc_per_split
auprc_train, auprc_valid, auprc_test = auprc_per_split
precision_train, precision_valid, precision_test = precisions_per_split
recall_train, recall_valid, recall_test = recalls_per_split
# save performance
perf_dict = {
'auroc_train': auroc_train,
'auroc_valid': auroc_valid,
'auroc_test': auroc_test,
'auprc_train': auprc_train,
'auprc_valid': auprc_valid,
'auprc_test': auprc_test,
'precision_train': precision_train,
'precision_valid': precision_valid,
'precision_test': precision_test,
'recall_train': recall_train,
'recall_valid': recall_valid,
'recall_test': recall_test,
'threshold': best_thr
}
perf_df = pd.DataFrame([perf_dict])
perf_csv = os.path.join(args.output_dir, output_filename_prefix + '.csv')
print('Final performance on train, valid and test :\n')
print(perf_df)
print('Final performance saved to %s' % perf_csv)
perf_df.to_csv(perf_csv, index=False)
from skorch.callbacks import LRScheduler
lrscheduler = LRScheduler(policy='StepLR', step_size=7, gamma=0.1)
from skorch.callbacks import Checkpoint
from skorch.callbacks import Freezer
from skorch.callbacks import EpochScoring
#freezer = Freezer(lambda x: not x.startswith('model.fc'))
auc = EpochScoring(scoring='roc_auc', lower_is_better=False)
f1 = EpochScoring(scoring='f1', lower_is_better=False)
for i in range(1):
checkpoint = Checkpoint(f_params='./' + str(i + 1) + 'best_model.pt',
monitor='valid_acc_best')
print("data folder: " + str('./combined_HE'))
data = load_data.load('./combined_he')
train_ds, valid_ds = data[0], data[1] #data[2]
net = NeuralNetClassifier(
Inception3,
criterion=nn.CrossEntropyLoss,
lr=0.002,
batch_size=10,
max_epochs=40,
#module__output_features=2,
#module__dropout = 0.5,
optimizer=optim.SGD,
optimizer__momentum=0.9,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
def train(data,
dilation,
kernel_size,
lr,
patient_index,
model_string,
correlation_monitor,
output_dir,
max_train_epochs=300,
split=None,
cropped=True,
padding=False):
"""
Creates and fits a model with the specified parameters onto the specified data
:param data: dataset on which the model is to be trained
:param dilation: dilation parameters of the model max-pool layers
:param kernel_size: kernel sizes of the model's max-pool layers
:param lr: learning rate
:param patient_index: index of the patient on whose data the model is trained
:param model_string: string specifying the setting of the data
:param correlation_monitor: correlation monitor object calculating the correlations while fitting
:param output_dir: where the trained model should be saved
:param max_train_epochs: number of epochs for which to train the model
:param split: the fold from cross-validation for which we are currently trainig the model
:param cropped: if the decoding is cropped, alwasy True in thesis experiments
:param padding: if padding should be added, always False in thesis experiments
:return:
"""
model, changed_model, model_name = get_model(data.in_channels,
input_time_length,
dilations=dilation,
kernel_sizes=kernel_size,
padding=padding)
if cuda:
device = 'cuda'
model.model = changed_model.cuda()
else:
model.model = changed_model
device = 'cpu'
if not padding:
n_preds_per_input = get_output_shape(model.model, model.input_channels,
model.input_time_length)[1]
else:
n_preds_per_input = 1
Path(home + f'/models/saved_models/{output_dir}/').mkdir(parents=True,
exist_ok=True)
# cutting the input into batches compatible with model
# if data.num_of_folds != -1, then also pre-whitening or filtering takes place
# as part of the cut_input method
data.cut_input(input_time_length=input_time_length,
n_preds_per_input=n_preds_per_input,
shuffle=False)
print(
f'starting cv epoch {split} out of {data.num_of_folds} for model: {model_string}_{model_name}'
)
correlation_monitor.step_number = 0
if split is not None:
correlation_monitor.split = split
monitor = 'validation_correlation_best'
monitors = [
('correlation monitor', correlation_monitor),
('checkpoint',
Checkpoint(
monitor=monitor,
f_history=home +
f'/logs/model_{model_name}/histories/{model_string}_k_{model_name}_p_{patient_index}.json',
)),
]
# cropped=False
print('cropped:', cropped)
# object EEGRegressor from the braindecode library suited for fitting models for regression tasks
regressor = EEGRegressor(cropped=cropped,
module=model.model,
criterion=model.loss_function,
optimizer=model.optimizer,
max_epochs=max_train_epochs,
verbose=1,
train_split=data.cv_split,
callbacks=monitors,
lr=lr,
device=device,
batch_size=32).initialize()
torch.save(
model.model, home +
f'/models/saved_models/{output_dir}/initial_{model_string}_{model_name}_p_{patient_index}'
)
regressor.max_correlation = -1000
if padding:
regressor.fit(data.train_set[0], data.train_set[1])
regressor.fit(np.stack(data.train_set.X), np.stack(data.train_set.y))
# best_model = load_model(
# f'/models/saved_models/{output_dir}/best_model_split_0')
torch.save(model.model,
home + f'/models/saved_models/{output_dir}/last_model_{split}')
if cuda:
best_corr = get_corr_coef(correlation_monitor.validation_set,
model.model.cuda(device=device))
else:
best_corr = get_corr_coef(correlation_monitor.validation_set,
model.model)
print(patient_index, best_corr)
return best_corr
import torch
from torch.nn import init
from ase import Atoms
from ase.calculators.emt import EMT
from ase.io import read
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params("valid_best_params.pt")
LR_schedule = LRScheduler("CosineAnnealingLR", T_max=5)
# saves best validation loss
cp = Checkpoint(monitor="valid_loss_best", fn_prefix="valid_best_")
# loads best validation loss at the end of training
load_best_valid_loss = train_end_load_best_valid_loss()
distances = np.linspace(2, 5, 10)
label = "delta_ml_example"
images = []
for l in distances:
image = Atoms(
"CuCO",
[
(-l * np.sin(0.65), l * np.cos(0.65), 0),
(0, 0, 0),
(l * np.sin(0.65), l * np.cos(0.65), 0),
],
)
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params('./histories/%i_valid_best_params.pt' % k)
nets = []
# Fold the CV data
k_folder = KFold(n_splits=5)
for k, (indices_train, _) in enumerate(k_folder.split(sdts_train)):
stds_train_ = [sdts_train[index] for index in indices_train]
targets_train_ = np.array(
[targets_train[index] for index in indices_train])
# Define various callbacks and checkpointers for this network
LR_schedule = LRScheduler('MultiStepLR', milestones=[75], gamma=0.1)
cp = Checkpoint(monitor='valid_loss_best',
fn_prefix='./histories/%i_valid_best_' % k)
load_best_valid_loss = train_end_load_best_valid_loss()
# Train this fold's network
net = NeuralNetRegressor(CrystalGraphConvNet,
module__orig_atom_fea_len=orig_atom_fea_len,
module__nbr_fea_len=nbr_fea_len,
batch_size=214,
module__classification=False,
lr=0.0056,
max_epochs=100,
module__atom_fea_len=46,
module__h_fea_len=83,
module__n_conv=8,
module__n_h=4,
optimizer=Adam,
print("Progress: {:.2%}".format(cnt / len(settings)))
print(setting)
# Prepare neural network for training
hidden_layer_sizes = tuple(x_train.shape[1] // (i + 1)**2
for i in range(setting['num_hidden_layers']))
# # LR scheduler used with SGD.
# schedule = LRScheduler(
# policy=lr_scheduler.CosineAnnealingLR,
# T_max=epochs * (5 / 4), # lr = 0.04 for max learning with audio_text
# eta_min=setting['eta_min'],
# )
model_id = start_time.strftime('%Y%m%d%H%M%S')
base_checkpoint_dir = "/home/cgn/skorch_checkpoints/"
cp = Checkpoint(
dirname=base_checkpoint_dir + model_id,
f_params="nn.pt", # Name of checkpoint parameters saved.
)
# Set-up neural network for training.
model = NeuralNetwork( # Skorch NeuralNetClassifer
module=NNClassifier,
callbacks=[
cp,
# ('lr_scheduler', schedule), # Use with SGD optimizer
],
lr=setting['initial_lr'],
module__input_size=x_train.shape[1],
module__hidden_layer_sizes=hidden_layer_sizes,
module__dropout=setting['dropout'],
module__output_size=output_size,
criterion=criterion,
optimizer=torch.optim.Adam, # torch.optim.SGD,
MODELS, BATCH_SIZE, SAVE_HIST, SAVE_MODELS, SAVE_OPT, MAX_EPs):
if model == "alexnet": # Alexnet requires lower lr
lr = 0.001
else:
lr = 0.01
print("Training with model ", model)
model, input_size = get_pretrained_models(model)
dataset_train = Alzheimer_Dataset(X_train,
y_train,
transform=data_transforms['train'])
dataset_val = Alzheimer_Dataset(X_val,
y_val,
transform=data_transforms['train'])
checkpoint = Checkpoint(f_params=save_model,
monitor='valid_acc_best',
f_optimizer=save_opt,
f_history=hist)
seed_everything = FixRandomSeed()
net = NeuralNetClassifier(model,
criterion=nn.CrossEntropyLoss,
optimizer=optim.SGD,
lr=lr,
batch_size=batch_size,
max_epochs=ep,
optimizer__momentum=0.90,
iterator_train__shuffle=True,
iterator_train__num_workers=8,
iterator_valid__shuffle=True,
iterator_valid__num_workers=8,
train_split=predefined_split(dataset_val),
callbacks=[
net = NeuralNetClassifier(
MyModule,
max_epochs=100,
lr=0.001,
batch_size=1024,
optimizer=Adam,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
iterator_train__pin_memory=True,
train_split=predefined_split(val),
callbacks=[LRScheduler(policy=CosineAnnealingLR, T_max=64),
EpochScoring(macrof1, use_caching=True, lower_is_better=False),
EpochScoring(microf1, use_caching=True, lower_is_better=False),
Checkpoint(monitor='macrof1_best', dirname='model')],
device='cuda',
verbose=1
)
print('start training')
_ = net.fit(tra, y=None)
# net.initialize()
net.load_params(f_params='model/params.pt', f_optimizer='model/optimizer.pt', f_history='model/history.json')
# In[ ]:
submission = pd.read_csv('../data/submission.csv')
ann_prob = np.zeros(len(submission))
ann_labels = []
nn.MaxUnpool2d(self.encoder.pooling_3.size()),
nn.Conv2d(4, 4, 3, stride=2),
nn.ReLU(True),
nn.MaxUnpool2d(self.encoder.pooling_2.size()),
nn.Conv2d(16, 4, 3, padding=1),
nn.ReLU(True),
nn.MaxUnpool2d(self.encoder.pooling_1.size()),
nn.Conv2d(3, 16, 4, padding=1)
)
def forward(self, x):
return self.encoder(x)
if __name__ == "__main__":
cp = Checkpoint(dirname='segnet_mse_no_sigmoid_sgd_150ep_b8_lr_0.01_30enc/checkpoints')
train_end_cp = TrainEndCheckpoint(dirname='segnet_mse_no_sigmoid_sgd_150ep_b8_lr_0.01_30enc/checkpoints')
net = NeuralNetRegressor(
SegNet,
module__encoding_size=30,
device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu"),
max_epochs=150,
batch_size=8,
criterion=MSELoss,
lr=0.01,
iterator_train__shuffle=True,
optimizer=torch.optim.SGD,
optimizer__momentum=.9,
callbacks=[cp, train_end_cp]
)
net.initialize()
def test_skorch_delta():
from amptorch.skorch_model import AMP
cp = Checkpoint(monitor="valid_loss_best", fn_prefix="valid_best_")
distances = np.linspace(2, 5, 10)
label = "skorch_example"
images = []
energies = []
forces = []
for l in distances:
image = Atoms(
"CuCO",
[
(-l * np.sin(0.65), l * np.cos(0.65), 0),
(0, 0, 0),
(l * np.sin(0.65), l * np.cos(0.65), 0),
],
)
image.set_cell([10, 10, 10])
image.wrap(pbc=True)
image.set_calculator(EMT())
images.append(image)
energies.append(image.get_potential_energy())
forces.append(image.get_forces())
image = Atoms("CuC", [(-1, 1, 0), (1, 1, 0)])
image.set_cell([10, 10, 10])
image.wrap(pbc=True)
image.set_calculator(EMT())
images.append(image)
energies.append(image.get_potential_energy())
forces.append(image.get_forces())
energies = np.array(energies)
forces = np.concatenate(np.array(forces))
Gs = {}
Gs["G2_etas"] = np.logspace(np.log10(0.05), np.log10(5.0), num=2)
Gs["G2_rs_s"] = [0] * 2
Gs["G4_etas"] = [0.005]
Gs["G4_zetas"] = [1.0]
Gs["G4_gammas"] = [+1.0, -1]
Gs["cutoff"] = 6.5
params = {
"C": {"re": 0.972, "D": 6.379, "sig": 0.477},
"O": {"re": 1.09, "D": 8.575, "sig": 0.603},
"Cu": {"re": 2.168, "D": 3.8386, "sig": 1.696},
}
morse_model = morse_potential(images, params, Gs["cutoff"], label)
morse_energies, morse_forces, num_atoms = morse_model.morse_pred(
images, params)
morse_data = [morse_energies, morse_forces, num_atoms, params, morse_model]
forcetraining = True
training_data = AtomsDataset(
images,
SNN_Gaussian,
Gs,
forcetraining=forcetraining,
label=label,
cores=1,
delta_data=morse_data,
)
batch_size = len(training_data)
unique_atoms = training_data.elements
fp_length = training_data.fp_length
device = "cpu"
net = NeuralNetRegressor(
module=FullNN(
unique_atoms, [fp_length, 2, 2], device, forcetraining=forcetraining
),
criterion=CustomMSELoss,
criterion__force_coefficient=0.3,
optimizer=torch.optim.LBFGS,
optimizer__line_search_fn="strong_wolfe",
lr=1e-2,
batch_size=batch_size,
max_epochs=100,
iterator_train__collate_fn=collate_amp,
iterator_train__shuffle=False,
iterator_valid__collate_fn=collate_amp,
device=device,
train_split=0,
verbose=0,
callbacks=[
EpochScoring(
forces_score,
on_train=True,
use_caching=True,
target_extractor=target_extractor,
),
EpochScoring(
energy_score,
on_train=True,
use_caching=True,
target_extractor=target_extractor,
),
],
)
calc = AMP(training_data, net, "test")
calc.train(overwrite=True)
num_of_atoms = 3
calculated_energies = np.array(
[calc.get_potential_energy(image) for idx, image in enumerate(images)]
)
energy_rmse = np.sqrt(
(((calculated_energies - energies) / num_of_atoms) ** 2).sum() / len(images)
)
last_energy_score = net.history[-1]["energy_score"]
assert round(energy_rmse, 4) == round(
last_energy_score, 4
), "Energy errors incorrect!"
last_forces_score = net.history[-1]["forces_score"]
calculated_forces = np.concatenate(
np.array([calc.get_forces(image) for image in images])
)
force_rmse = np.sqrt(
(((calculated_forces - forces)) ** 2).sum() / (3 * num_of_atoms * len(images))
)
assert round(force_rmse, 4) == round(
last_forces_score, 4
), "Force errors incorrect!"
def main():
parser = argparse.ArgumentParser(
description='PyTorch RNN with variable-length numeric sequences wrapper'
)
parser.add_argument('--outcome_col_name', type=str, required=True)
parser.add_argument('--train_csv_files', type=str, required=True)
parser.add_argument('--test_csv_files', type=str, required=True)
parser.add_argument('--data_dict_files', type=str, required=True)
parser.add_argument('--batch_size',
type=int,
default=1024,
help='Number of sequences per minibatch')
parser.add_argument('--epochs',
type=int,
default=50,
help='Number of epochs')
parser.add_argument('--hidden_units',
type=int,
default=32,
help='Number of hidden units')
parser.add_argument('--hidden_layers',
type=int,
default=1,
help='Number of hidden layers')
parser.add_argument('--lr',
type=float,
default=0.0005,
help='Learning rate for the optimizer')
parser.add_argument('--dropout',
type=float,
default=0,
help='dropout for optimizer')
parser.add_argument('--weight_decay',
type=float,
default=0.0001,
help='weight decay for optimizer')
parser.add_argument('--seed', type=int, default=1111, help='random seed')
parser.add_argument('--validation_size',
type=float,
default=0.15,
help='validation split size')
parser.add_argument(
'--is_data_simulated',
type=bool,
default=False,
help='boolean to check if data is simulated or from mimic')
parser.add_argument(
'--simulated_data_dir',
type=str,
default='simulated_data/2-state/',
help=
'dir in which to simulated data is saved.Must be provide if is_data_simulated = True'
)
parser.add_argument(
'--output_dir',
type=str,
default=None,
help=
'directory where trained model and loss curves over epochs are saved')
parser.add_argument(
'--output_filename_prefix',
type=str,
default=None,
help='prefix for the training history jsons and trained classifier')
args = parser.parse_args()
torch.manual_seed(args.seed)
device = 'cpu'
x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split(
',')
x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',')
x_dict, y_dict = args.data_dict_files.split(',')
x_data_dict = load_data_dict_json(x_dict)
# get the id and feature columns
id_cols = parse_id_cols(x_data_dict)
feature_cols = parse_feature_cols(x_data_dict)
# extract data
train_vitals = TidySequentialDataCSVLoader(
x_csv_path=x_train_csv_filename,
y_csv_path=y_train_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_sequence')
test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename,
y_csv_path=y_test_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_sequence')
X_train, y_train = train_vitals.get_batch_data(batch_id=0)
X_test, y_test = test_vitals.get_batch_data(batch_id=0)
_, T, F = X_train.shape
print('number of time points : %s\n number of features : %s\n' % (T, F))
# set class weights as 1/(number of samples in class) for each class to handle class imbalance
class_weights = torch.tensor(
[1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).double()
# scale features
# X_train = standard_scaler_3d(X_train)
# X_test = standard_scaler_3d(X_test)
# callback to compute gradient norm
compute_grad_norm = ComputeGradientNorm(norm_type=2)
# LSTM
if args.output_filename_prefix == None:
output_filename_prefix = (
'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' %
(args.hidden_units, args.hidden_layers, args.lr, args.dropout,
args.weight_decay))
else:
output_filename_prefix = args.output_filename_prefix
print('RNN parameters : ' + output_filename_prefix)
# # from IPython import embed; embed()
rnn = RNNBinaryClassifier(
max_epochs=50,
batch_size=args.batch_size,
device=device,
lr=args.lr,
callbacks=[
EpochScoring('roc_auc',
lower_is_better=False,
on_train=True,
name='aucroc_score_train'),
EpochScoring('roc_auc',
lower_is_better=False,
on_train=False,
name='aucroc_score_valid'),
EarlyStopping(monitor='aucroc_score_valid',
patience=20,
threshold=0.002,
threshold_mode='rel',
lower_is_better=False),
LRScheduler(policy=ReduceLROnPlateau,
mode='max',
monitor='aucroc_score_valid',
patience=10),
compute_grad_norm,
GradientNormClipping(gradient_clip_value=0.3,
gradient_clip_norm_type=2),
Checkpoint(monitor='aucroc_score_valid',
f_history=os.path.join(
args.output_dir, output_filename_prefix + '.json')),
TrainEndCheckpoint(dirname=args.output_dir,
fn_prefix=output_filename_prefix),
],
criterion=torch.nn.CrossEntropyLoss,
criterion__weight=class_weights,
train_split=skorch.dataset.CVSplit(args.validation_size),
module__rnn_type='LSTM',
module__n_layers=args.hidden_layers,
module__n_hiddens=args.hidden_units,
module__n_inputs=X_train.shape[-1],
module__dropout_proba=args.dropout,
optimizer=torch.optim.Adam,
optimizer__weight_decay=args.weight_decay)
clf = rnn.fit(X_train, y_train)
y_pred_proba = clf.predict_proba(X_train)
y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_train_final = roc_auc_score(y_train, y_pred_proba_pos)
print('AUROC with LSTM (Train) : %.2f' % auroc_train_final)
y_pred_proba = clf.predict_proba(X_test)
y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_test_final = roc_auc_score(y_test, y_pred_proba_pos)
print('AUROC with LSTM (Test) : %.2f' % auroc_test_final)
from torch.nn import init
from skorch.utils import to_numpy
import numpy as np
from ase import Atoms
from ase.calculators.emt import EMT
from ase.io import read
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params('valid_best_params.pt')
LR_schedule = LRScheduler('CosineAnnealingLR', T_max=5)
# saves best validation loss
cp = Checkpoint(monitor='valid_loss_best', fn_prefix='valid_best_')
# loads best validation loss at the end of training
load_best_valid_loss = train_end_load_best_valid_loss()
distances = np.linspace(2, 5, 100)
label = "skorch_example"
images = []
for l in distances:
image = Atoms(
"CuCO",
[
(-l * np.sin(0.65), l * np.cos(0.65), 0),
(0, 0, 0),
(l * np.sin(0.65), l * np.cos(0.65), 0),
],
)
