def grid_search(train, grid_params, module, loss, outpath):
net = NeuralNet(module,
loss,
optimizer=torch.optim.Adam,
max_epochs=MAX_EPOCHS,
device=DEVICE,
iterator_train__shuffle=False,
callbacks=[
EarlyStopping("valid_loss",
lower_is_better=True,
patience=PATIENCE)
],
train_split=CVSplit(5, random_state=SEED)) #, verbose=0)
kf = KFold(n_splits=5, shuffle=False, random_state=SEED)
gs = GridSearchCV(net,
grid_params,
cv=kf,
refit=True,
scoring=loss(),
verbose=10)
grid_res = gs.fit(SliceDataset(train), train.y)
df = pd.DataFrame(grid_res.cv_results_)
# Adding some meta info
df["param_feature_set"] = FLEVEL
df["param_overlapping"] = WINDOW_OVERLAPPING
df["param_context_len"] = CONTEXT_LEN
df["param_discretized"] = DISCRETIZED
df["param_module"] = module.__name__
fname = f"gridsearch_{FLEVEL}_{CONTEXT_LEN}_{WINDOW_OVERLAPPING}_{DISCRETIZED}_{module.__name__}.pkl"
df.to_pickle(outpath / fname)
def simple_aleo_nn_init(x_train, y_train, short_term, crps_loss=False):
if crps_loss:
loss = CRPSLoss
else:
loss = HeteroscedasticLoss
if short_term:
hs = [24, 64, 32]
lr = 0.0005
epochs = 3500
else:
hs = [132, 77, 50]
lr = 5e-05
epochs = 1253
es = EarlyStopping(patience=75)
simple_nn = AleatoricNNSkorch(
module=SimpleNN,
module__input_size=x_train.shape[-1],
module__output_size=y_train.shape[-1] * 2,
module__hidden_size=hs,
lr=lr,
batch_size=1024,
max_epochs=epochs,
train_split=None,
optimizer=torch.optim.Adam,
criterion=loss,
device=device,
verbose=1,
# callbacks=[es]
)
return simple_nn
def test_regular(device):
"""
Tests the LSTMTimeSeriesPredictor fitting
"""
cuda_check(device)
start = time.time()
tsp = TimeSeriesPredictor(
BenchmarkLSTM(hidden_dim=16),
lr=1e-3,
lambda1=1e-8,
optimizer__weight_decay=1e-8,
iterator_train__shuffle=True,
early_stopping=EarlyStopping(patience=50),
max_epochs=250,
train_split=CVSplit(10),
optimizer=Adam,
device=device,
)
past_pattern_length = 24
future_pattern_length = 12
pattern_length = past_pattern_length + future_pattern_length
fsd = FlightSeriesDataset(pattern_length,
past_pattern_length,
pattern_length,
stride=1)
tsp.fit(fsd)
end = time.time()
elapsed = timedelta(seconds=end - start)
print(f"Fitting in {device} time delta: {elapsed}")
mean_r2_score = tsp.score(tsp.dataset)
print(f"Achieved R2 score: {mean_r2_score}")
assert mean_r2_score > -20
def build_estimator(hyperparams, train_data, test=False):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
n_classes = len(np.unique(y))
n_samples = y.shape[0]
bal_weights = torch.from_numpy(
n_samples / (n_classes * np.bincount(y))).float().to(device)
callbacks = [
('f1_score_valid',
EpochScoring('f1' if n_classes == 2 else 'f1_macro',
name='valid_f1',
lower_is_better=False)),
('early_stopping',
EarlyStopping(monitor='valid_loss',
patience=5,
lower_is_better=True)),
(
'learning_rate_scheduler',
LRScheduler(
policy=lr_scheduler.ReduceLROnPlateau,
monitor='valid_loss',
# Following kargs are passed to the
# lr scheduler constructor
mode='min',
min_lr=1e-5)),
]
return NeuralNetClassifier(
NNModule,
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5,
stratified=True,
random_state=RANDOM_STATE),
lr=hyperparams['lr'],
batch_size=hyperparams['batch_size'],
module__in_features=in_features,
module__n_classes=n_classes,
module__n_layers=hyperparams['n_layers'],
module__n_neuron_per_layer=hyperparams['n_neuron_per_layer'],
module__activation=getattr(F, hyperparams['activation']),
module__p_dropout=hyperparams['p_dropout'],
criterion__weight=bal_weights
if hyperparams['class_weight'] == 'balanced' else None,
optimizer__momentum=hyperparams['momentum'],
optimizer__weight_decay=hyperparams['weight_decay'],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4)
def _default_callbacks(self):
cbs = list(super()._default_callbacks)
if self.early_stopping:
threshold, patience = self.early_stopping
cbs += [('early_stopping',
EarlyStopping(monitor='train_loss',
threshold=threshold,
patience=patience))]
return cbs
def train(data_folder: str, out_model: str):
out_model = Path(out_model)
out_model.mkdir()
data_paths = list(Path(data_folder).rglob("*.npy"))
train_paths, valid_paths = train_test_split(data_paths, train_size=0.7)
train_dataset = LibriSpeechDataset(
train_paths,
Path(data_folder).parent / "SPEAKERS.TXT",
Compose([ExtractStft(),
RandomCrop(constants.STFT_CROP_WIDTH)]))
valid_dataset = LibriSpeechDataset(
valid_paths,
Path(data_folder).parent / "SPEAKERS.TXT",
Compose([ExtractStft(),
RandomCrop(constants.STFT_CROP_WIDTH)]))
net = NeuralNet(Classifier,
module__n_classes=constants.NUMBER_OF_CLASSES,
criterion=nn.CrossEntropyLoss,
batch_size=8,
max_epochs=100,
optimizer=optim.Adam,
lr=0.001,
iterator_train__shuffle=True,
iterator_train__num_workers=2,
iterator_valid__shuffle=False,
iterator_valid__num_workers=2,
train_split=predefined_split(valid_dataset),
device="cuda",
callbacks=[
Checkpoint(
f_params=(out_model / "params.pt").as_posix(),
f_optimizer=(out_model / "optim.pt").as_posix(),
f_history=(out_model / "history.pt").as_posix()),
ProgressBar(postfix_keys=["train_loss", "train_acc"]),
EarlyStopping(),
EpochScoring(acc,
name="val_acc",
lower_is_better=False,
on_train=False),
EpochScoring(acc,
name="train_acc",
lower_is_better=False,
on_train=True),
Tensorboard((out_model / "train").as_posix(),
metrics={"acc": acc_as_metric},
is_training=True),
Tensorboard((out_model / "valid").as_posix(),
metrics={"acc": acc_as_metric},
is_training=False),
])
net.fit(train_dataset)
def get_deep_learning_model(model_args, valid_dataset):
cuda = torch.cuda.is_available()
device = model_args["device"] if cuda else 'cpu'
if cuda:
torch.backends.cudnn.benchmark = True
seed = model_args["seed"]
# = 20200220 random seed to make results reproducible
# Set random seed to be able to reproduce results
if seed:
set_random_seeds(seed=seed, cuda=cuda)
if model_args["model_type"] == "ShallowFBCSPNet":
model = ShallowFBCSPNet(
model_args["n_chans"],
model_args["n_classes"] + 1,
input_window_samples=model_args["input_window_samples"],
final_conv_length='auto',
)
elif model_args["model_type"] == "SleepStager":
model = model = SleepStager(
n_channels=model_args["n_chans"],
sfreq=model_args["sfreq"],
n_classes=model_args["n_classes"] + 1,
input_size_s=model_args["input_window_samples"] /
model_args["sfreq"],
)
else:
raise ValueError("Boom !")
if cuda:
model.cuda()
clf = EEGClassifier(
model,
criterion=model_args["criterion"],
optimizer=torch.optim.AdamW,
# using test_sample for validation
train_split=predefined_split(valid_dataset),
optimizer__lr=model_args["lr"],
optimizer__weight_decay=model_args["weight_decay"],
batch_size=model_args["batch_size"],
callbacks=[
"accuracy",
("lr_scheduler",
LRScheduler('CosineAnnealingLR',
T_max=model_args["n_epochs"] - 1)),
("early_stopping",
EarlyStopping(monitor='valid_loss',
patience=model_args["patience"]))
],
device=device,
iterator_train__num_workers=20,
iterator_train__pin_memory=True) # torch.in torch.out
return clf
def build_estimator(cls,
hyperparams,
train_data,
verbose=True,
test=False): # change default verbose to false later
early_stopping_val_percent = 10
n_training_examples = len(
train_data[0]) * (1 - (early_stopping_val_percent / 100))
n_iter_per_epoch = n_training_examples / hyperparams['batch_size']
n_iter_btw_restarts = int(hyperparams['epochs_btw_restarts'] *
n_iter_per_epoch)
callbacks = [
('fix_seed', cls.FixRandomSeed(RANDOM_STATE)),
('lr_monitor', cls.LRMonitor()),
('accuracy_score_valid',
EpochScoring('accuracy', lower_is_better=False, on_train=True)),
('early_stopping',
EarlyStopping(monitor='valid_acc',
lower_is_better=False,
patience=100)),
('learning_rate_scheduler',
LRScheduler(policy=cls.SkorchCosineAnnealingWarmRestarts,
T_0=n_iter_btw_restarts,
T_mult=hyperparams['epochs_btw_restarts_mult']))
]
def validation_split(X, y):
""" Custom split is used to apply augmentation to the training set only """
splitter = CVSplit(cv=int(100 / early_stopping_val_percent),
random_state=RANDOM_STATE)
dataset_train, dataset_valid = splitter(X)
dataset_train = cls.AugmentedDataset(dataset_train)
return dataset_train, dataset_valid
return NeuralNetClassifier(
cls.CifarCustomNet,
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.SGD,
max_epochs=hyperparams['max_epochs'] if not test else 1,
iterator_train__shuffle=True,
iterator_train__num_workers=4,
iterator_valid__num_workers=4,
dataset=cls.NormalizedDataset,
callbacks=callbacks,
device=cls.device,
train_split=validation_split,
lr=hyperparams['learning_rate'],
batch_size=hyperparams['batch_size'],
optimizer__momentum=hyperparams['momentum'],
optimizer__weight_decay=hyperparams['weight_decay'],
optimizer__nesterov=hyperparams['nesterov'],
module__conv_dropout=hyperparams['conv_dropout'],
module__fc_dropout=hyperparams['fc_dropout'],
verbose=3 if verbose else 0)
def test_train_loss_monitor(user_name, user_password):
"""
Tests the LSTMTimeSeriesPredictor fitting
"""
tsp = TimeSeriesPredictor(
BenchmarkLSTM(hidden_dim=10),
early_stopping=EarlyStopping(monitor='train_loss', patience=15),
max_epochs=150,
# train_split=None, # default = skorch.dataset.CVSplit(5)
optimizer=torch.optim.Adam)
tsp.fit(_get_dataset(user_name, user_password))
mean_r2_score = tsp.score(tsp.dataset)
assert mean_r2_score > -300
def test_train_loss_monitor_no_train_split():
"""
Tests the LSTMTimeSeriesPredictor fitting
"""
tsp = TimeSeriesPredictor(BenchmarkLSTM(hidden_dim=10),
early_stopping=EarlyStopping(
monitor='train_loss', patience=15),
max_epochs=150,
train_split=None,
optimizer=torch.optim.Adam)
tsp.fit(FlightsDataset())
mean_r2_score = tsp.score(tsp.dataset)
assert mean_r2_score > -300
def test_no_train_split():
"""
Tests the LSTMTimeSeriesPredictor fitting
"""
with pytest.raises(ValueError) as error:
TimeSeriesPredictor(BenchmarkLSTM(hidden_dim=16),
early_stopping=EarlyStopping(),
max_epochs=500,
train_split=None,
optimizer=torch.optim.Adam)
# pylint: disable=line-too-long
assert error.match(
'Select a valid train_split or disable early_stopping! A valid train_split needs to be selected when valid_loss monitor is selected as early stopping criteria.'
)
def fnp_init(x_train, y_train, short_term):
# Early Stopping when crps does not improve
# (fnp can get worse if trained too long)
def scoring(model, X, y):
y_pred = model.predict(X.X['XM'], samples=5)
score = crps(y_pred[..., 0], np.sqrt(y_pred[..., 1]), y)
return score
scorer = EpochScoring(scoring, on_train=True, name='crps')
es = EarlyStopping(monitor='crps', patience=100)
if short_term:
epochs = 300
hs_enc = [24, 64]
hs_dec = [32]
dim_u = 2
dim_z = 32
fb_z = 1.0
else:
epochs = 300
hs_enc = [132, 77]
hs_dec = [50]
dim_u = 9
dim_z = 32
fb_z = 1.0
fnp = RegressionFNPSkorch(
module=RegressionFNP,
module__dim_x=x_train.shape[-1],
module__dim_y=y_train.shape[-1],
module__hidden_size_enc=hs_enc,
module__hidden_size_dec=hs_dec,
optimizer=torch.optim.Adam,
device=device,
seed=42,
module__dim_u=dim_u,
module__dim_z=dim_z,
module__fb_z=fb_z,
lr=0.001,
reference_set_size_ratio=0.003,
max_epochs=epochs,
batch_size=1024,
train_size=x_train.size,
train_split=None,
verbose=1,
# callbacks=[scorer, es]
)
return fnp
def build_estimator(hyperparams, train_data, test=False):
device = "cuda" if torch.cuda.is_available() else "cpu"
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
callbacks = [
("r2_score_valid", EpochScoring("r2", lower_is_better=False)),
(
"early_stopping",
EarlyStopping(monitor="valid_loss", patience=5, lower_is_better=True),
),
(
"learning_rate_scheduler",
LRScheduler(
policy=lr_scheduler.ReduceLROnPlateau,
monitor="valid_loss",
# Following kargs are passed to the
# lr scheduler constructor
mode="min",
min_lr=1e-5,
),
),
]
return NeuralNetRegressor(
NNModule,
criterion=nn.MSELoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5, random_state=RANDOM_STATE),
lr=hyperparams["lr"],
batch_size=hyperparams["batch_size"],
module__in_features=in_features,
module__n_layers=hyperparams["n_layers"],
module__n_neuron_per_layer=hyperparams["n_neuron_per_layer"],
module__activation=getattr(F, hyperparams["activation"]),
module__p_dropout=hyperparams["p_dropout"],
optimizer__momentum=hyperparams["momentum"],
optimizer__weight_decay=hyperparams["weight_decay"],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4,
)
def train_(self,
train_set,
valid_set,
lr=5e-4,
batch_size=16,
max_nb_epochs=20,
early_stopping_patience=5,
early_stopping_monitor='valid_bal_acc'):
# Train using a GPU if possible
device = "cuda" if torch.cuda.is_available() else "cpu"
# Callbacks
train_bal_acc = EpochScoring(scoring='balanced_accuracy',
on_train=True,
name='train_bal_acc',
lower_is_better=False)
valid_bal_acc = EpochScoring(scoring='balanced_accuracy',
on_train=False,
name='valid_bal_acc',
lower_is_better=False)
early_stopping = EarlyStopping(monitor=early_stopping_monitor,
patience=early_stopping_patience,
lower_is_better='loss'
in early_stopping_monitor)
callbacks = [
('train_bal_acc', train_bal_acc),
('valid_bal_acc', valid_bal_acc),
('progress_bar', ProgressBar()),
('early_stopping', early_stopping),
]
# Skorch model creation
skorch_net = EEGTransformer(self.to(device),
criterion=torch.nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
optimizer__lr=lr,
train_split=predefined_split(valid_set),
batch_size=batch_size,
callbacks=callbacks,
device=device)
# Training: `y` is None since it is already supplied in the dataset.
skorch_net.fit(train_set, y=None, epochs=max_nb_epochs)
return skorch_net
def build_estimator(hyperparams, train_data, test=False):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Extract info from training data
X, y, *_ = train_data
in_features = X.shape[1]
callbacks = [
('r2_score_valid', EpochScoring('r2',
lower_is_better=False)),
('early_stopping', EarlyStopping(monitor='valid_loss',
patience=5,
lower_is_better=True)),
('learning_rate_scheduler', LRScheduler(policy=lr_scheduler.ReduceLROnPlateau,
monitor='valid_loss',
# Following kargs are passed to the
# lr scheduler constructor
mode='min',
min_lr=1e-5
)),
]
return NeuralNetRegressor(
NNModule,
criterion=nn.MSELoss,
optimizer=torch.optim.SGD,
max_epochs=300,
iterator_train__shuffle=True, # Shuffle training data on each epoch
callbacks=callbacks,
device=device,
train_split=CVSplit(cv=5, random_state=RANDOM_STATE),
lr=hyperparams['lr'],
batch_size=hyperparams['batch_size'],
module__in_features=in_features,
module__n_layers=hyperparams['n_layers'],
module__n_neuron_per_layer=hyperparams['n_neuron_per_layer'],
module__activation=getattr(F, hyperparams['activation']),
module__p_dropout=hyperparams['p_dropout'],
optimizer__momentum=hyperparams['momentum'],
optimizer__weight_decay=hyperparams['weight_decay'],
optimizer__nesterov=True,
verbose=3,
iterator_train__num_workers=4,
iterator_valid__num_workers=4
)
def test_transformer_tsp_multisamples(device):
'''multivariate test'''
cuda_check(device)
start = time.time()
tsp = TimeSeriesPredictor(
Transformer(d_model=12),
lr=1e-5,
lambda1=1e-8,
optimizer__weight_decay=1e-8,
iterator_train__shuffle=True,
early_stopping=EarlyStopping(patience=100),
max_epochs=500,
train_split=CVSplit(10),
optimizer=Adam,
device=device,
)
past_pattern_length = 24
future_pattern_length = 12
pattern_length = past_pattern_length + future_pattern_length
# pylint: disable-next=line-too-long
fsd = FlightSeriesDataset(pattern_length,
future_pattern_length,
pattern_length,
stride=1,
generate_test_dataset=True)
tsp.fit(fsd)
end = time.time()
elapsed = timedelta(seconds=end - start)
print(f"Fitting in {device} time delta: {elapsed}")
mean_r2_score = tsp.score(tsp.dataset)
assert mean_r2_score > -0.5
netout = tsp.predict(fsd.test.x)
idx = np.random.randint(0, len(fsd.test.x))
y_true = fsd.test.y[idx, :, :]
y_hat = netout[idx, :, :]
r2s = r2_score(y_true, y_hat)
assert r2s > -1
print(f"Final R2 score: {r2s}")
def _default_callbacks(self):
default_cb_list = [('epoch_timer', EpochTimer()),
('train_loss',
BatchScoring(train_loss_score,
name='train_loss',
on_train=True,
target_extractor=noop)),
('valid_loss',
BatchScoring(valid_loss_score,
name='valid_loss',
target_extractor=noop)),
('valid_acc',
EpochScoring(
'accuracy',
name='valid_acc',
lower_is_better=False,
)),
('checkpoint', Checkpoint(dirname=self.model_path)),
('end_checkpoint',
TrainEndCheckpoint(dirname=self.model_path)),
('report', ReportLog()),
('progressbar', ProgressBar())]
if 'stop_patience' in self.hyperparamters.keys() and \
self.hyperparamters['stop_patience']:
earlystop_cb = ('earlystop',
EarlyStopping(
patience=self.hyperparamters['stop_patience'],
threshold=1e-4))
default_cb_list.append(earlystop_cb)
if 'lr_step' in self.hyperparamters.keys() and \
self.hyperparamters['lr_step']:
lr_callback = ('lr_schedule',
DecayLR(self.hyperparamters['lr'],
self.hyperparamters['lr_step'],
gamma=0.5))
default_cb_list.append(lr_callback)
return default_cb_list
def configureAnchor(outpath, checkpoint: Path = None):
batch_size = 4096
lr = 1e-4
model_args = {
"module__pool": "last",
"module__input_size": 19,
"module__rnn_size": 128,
"module__rnn_layers": 3,
"module__latent_size": 128
}
dist_plot = Callbacks.DistPlot(outpath)
loss_plot = Callbacks.EpochPlot(outpath, ["train_loss", "valid_loss"])
net = ad.WindowedAnomalyDetector(tripletloss.GruLinear,
tripletloss.ContextualCoherency,
optimizer=torch.optim.Adam,
iterator_train__shuffle=False,
lr=lr,
batch_size=batch_size,
max_epochs=MAX_EPOCHS,
**model_args,
device=DEVICE,
verbose=1,
train_split=CVSplit(5, random_state=SEED),
callbacks=[
dist_plot, loss_plot,
EarlyStopping("valid_loss",
lower_is_better=True,
patience=PATIENCE)
])
if checkpoint is not None:
net.initialize_context(int(TL_CONTEXT_LEN / 2))
net.initialize()
state_dict = torch.load(str(checkpoint),
map_location=torch.device("cpu"))
net.module_.load_state_dict(state_dict)
return net
def prepare_learnt_model(model_args, path_tfms, is_meta, verbose=2):
"""Model builder if learnt transforms are involved.
The key difference (as explained in prepare_non_learnt_model) between this function and prepare_non_learnt_model
is that the
Args:
model_args (dict): Experiment model args as defined in the main experiment function.
path_tfms (Pipeline): An sklearn pipeline of path transformations to be applied before model training.
is_meta (bool): Set True for a dyadic meta model.
verbose (int): Output verbosity level.
Returns:
"""
# Initialise the signature string class.
model_args['is_meta'] = is_meta
module = SignatureStringModel(**model_args)
model = NeuralNetClassifier(
module=module,
criterion=nn.BCEWithLogitsLoss if model_args['out_channels'] == 1 else nn.CrossEntropyLoss,
batch_size=64,
verbose=verbose,
iterator_train__drop_last=True,
callbacks=[
('scheduler', LRScheduler(policy='ReduceLROnPlateau')),
('val_stopping', EarlyStopping(monitor='valid_loss', patience=30)),
('checkpoint', CustomCheckpoint(monitor='valid_loss_best')),
('scorer', EpochScoring(custom_scorer, lower_is_better=False, name='true_acc'))
],
train_split=CVSplit(cv=5, random_state=1, stratified=True),
device=device if model_args['gpu'] else 'cpu',
)
pipeline = Pipeline([
*path_tfms,
('classifier', model)
])
return pipeline
def test_main(stride, test_main_context):
context = test_main_context(stride)
past_pattern_length = context['past_pattern_length']
future_pattern_length = context['future_pattern_length']
pattern_length = past_pattern_length + future_pattern_length
tsp = TimeSeriesPredictor(
BenchmarkLSTM(
initial_forget_gate_bias=1,
hidden_dim=7,
num_layers=1,
),
lr=context['lr'],
lambda1=1e-8,
optimizer__weight_decay=1e-8,
iterator_train__shuffle=True,
early_stopping=EarlyStopping(patience=100),
max_epochs=500,
train_split=CVSplit(context['n_cv_splits']),
optimizer=Adam,
)
fsd = FlightSeriesDataset(pattern_length,
future_pattern_length,
context['except_last_n'],
stride=stride,
generate_test_dataset=True)
tsp.fit(fsd)
mean_r2_score = tsp.score(tsp.dataset)
assert mean_r2_score > context['mean_r2_score']
netout = tsp.predict(fsd.test.x)
idx = np.random.randint(0, len(fsd.test.x))
y_true = fsd.test.y[idx, :, :]
y_hat = netout[idx, :, :]
r2s = r2_score(y_true, y_hat)
print("Final R2 score: {}".format(r2s))
assert r2s > context['final_r2_score']
def prepare_NN(self, num_classes):
class ThreeLayerNN(nn.Module):
def __init__(self, hidden, num_classes, activation):
super(ThreeLayerNN, self).__init__()
self.hidden = hidden
self.num_classes = num_classes
self.activation = activation
self.Layer1 = nn.Linear(768, self.hidden, bias=True)
self.Layer2 = nn.Linear(self.hidden, self.hidden, bias=True)
self.Output = nn.Linear(self.hidden, self.num_classes, bias=True)
def forward(self, x):
z1 = self.Layer1(x)
a1 = self.activation(z1)
z2 = self.Layer2(a1)
a2 = self.activation(z2)
y_hat = F.softmax(a2)
return(y_hat)
monitor = lambda net: all(net.history[-1, ("train_loss_best", "valid_loss_best")])
net = NeuralNetClassifier(module=ThreeLayerNN,
module__hidden=self.hidden,
module__num_classes=num_classes,
module__activation=self.activation,
max_epochs=1000,
lr=0.1,
iterator_train__shuffle=True,
callbacks=[Checkpoint(monitor=monitor),
EarlyStopping(patience=5,
threshold=0.0001),
LRScheduler()],
verbose=0,
device=self.device,
batch_size=32)
return(net)
def configureSeq2Seq(outpath, checkpoint: Path = None):
batch_size = 64
lr = 1e-4
model_args = {
"module__pool": "mean",
"module__input_size": 19,
"module__teacher_forcing_ratio": 7.,
"module__rnn_layers": 1,
"module__latent_size": 128
}
loss_plot = Callbacks.EpochPlot(outpath, ["train_loss", "valid_loss"])
net = ad.WindowedAnomalyDetector(autoencoder.Seq2Seq,
autoencoder.ReconstructionError,
optimizer=torch.optim.Adam,
iterator_train__shuffle=False,
lr=lr,
batch_size=batch_size,
max_epochs=MAX_EPOCHS,
**model_args,
device=DEVICE,
verbose=1,
train_split=CVSplit(5, random_state=SEED),
callbacks=[
loss_plot,
EarlyStopping("valid_loss",
lower_is_better=True,
patience=PATIENCE)
])
if checkpoint is not None:
net.initialize_context(SEQ2SEQ_CONTEXT_LEN)
net.initialize()
state_dict = torch.load(str(checkpoint),
map_location=torch.device("cpu"))
net.module_.load_state_dict(state_dict)
return net
def test_quantum_lstm_tsp_forecast(device):
"""
Tests the Quantum LSTM forecast
"""
cuda_check(device)
tsp = TimeSeriesPredictor(
QuantumLSTM(hidden_dim = 2),
max_epochs=250,
lr = 1e-4,
early_stopping=EarlyStopping(patience=100, monitor='train_loss'),
train_split=None,
optimizer=Adam,
device=device
)
whole_fd = FlightsDataset()
# leave last N months for error assertion
last_n = 24
start = time.time()
tsp.fit(FlightsDataset(pattern_length = 120, except_last_n = last_n))
end = time.time()
elapsed = timedelta(seconds = end - start)
print(f"Fitting in {device} time delta: {elapsed}")
mean_r2_score = tsp.score(tsp.dataset)
assert mean_r2_score > -5
netout, _ = tsp.forecast(last_n)
# Select any training example just for comparison
idx = np.random.randint(0, len(tsp.dataset))
_, whole_y = whole_fd[idx]
y_true = whole_y[-last_n:, :] # get only known future outputs
y_pred = netout[idx, -last_n:, :] # get only last N predicted outputs
r2s = r2_score(y_true, y_pred)
assert r2s > -60
def my_main():
callbacks = []
writer = SummaryWriter()
callbacks.append(MyTensorBoard(writer, X=fmnist_train.train_data[:, None, :, :].float()[:2]))
callbacks.append(ShuffleOrder())
callbacks.append(EarlyStopping())
exp = sys.argv[1]
today = datetime.now()
today_str = today.strftime("%d-%m-%H-%M")
os.mkdir(f'results/{today_str}')
dict_list = []
if exp == 'linear':
linear_experiment(callbacks, today_str)
elif exp == 'options':
options_experiment(callbacks, today_str)
elif exp == 'skip':
skip_experiment(callbacks, today_str)
elif exp == 'skip_linear':
skip_linear_experiment(callbacks, today_str)
elif exp == 'paths':
paths_experiment(callbacks, today_str)
filename = 'SoupNAS_fashion_mnist.csv'
pd.DataFrame(dict_list).to_csv(filename)
ex.add_artifact(filename)
def main():
parser = argparse.ArgumentParser(
description='PyTorch RNN with variable-length numeric sequences wrapper'
)
parser.add_argument('--outcome_col_name', type=str, required=True)
parser.add_argument('--train_csv_files', type=str, required=True)
parser.add_argument('--test_csv_files', type=str, required=True)
parser.add_argument('--data_dict_files', type=str, required=True)
parser.add_argument('--batch_size',
type=int,
default=1024,
help='Number of sequences per minibatch')
parser.add_argument('--epochs',
type=int,
default=50,
help='Number of epochs')
parser.add_argument('--hidden_units',
type=int,
default=32,
help='Number of hidden units')
parser.add_argument('--hidden_layers',
type=int,
default=1,
help='Number of hidden layers')
parser.add_argument('--lr',
type=float,
default=0.0005,
help='Learning rate for the optimizer')
parser.add_argument('--dropout',
type=float,
default=0,
help='dropout for optimizer')
parser.add_argument('--weight_decay',
type=float,
default=0.0001,
help='weight decay for optimizer')
parser.add_argument('--seed', type=int, default=1111, help='random seed')
parser.add_argument('--validation_size',
type=float,
default=0.15,
help='validation split size')
parser.add_argument(
'--is_data_simulated',
type=bool,
default=False,
help='boolean to check if data is simulated or from mimic')
parser.add_argument(
'--simulated_data_dir',
type=str,
default='simulated_data/2-state/',
help=
'dir in which to simulated data is saved.Must be provide if is_data_simulated = True'
)
parser.add_argument(
'--output_dir',
type=str,
default=None,
help=
'directory where trained model and loss curves over epochs are saved')
parser.add_argument(
'--output_filename_prefix',
type=str,
default=None,
help='prefix for the training history jsons and trained classifier')
args = parser.parse_args()
torch.manual_seed(args.seed)
device = 'cpu'
x_train_csv_filename, y_train_csv_filename = args.train_csv_files.split(
',')
x_test_csv_filename, y_test_csv_filename = args.test_csv_files.split(',')
x_dict, y_dict = args.data_dict_files.split(',')
x_data_dict = load_data_dict_json(x_dict)
# get the id and feature columns
id_cols = parse_id_cols(x_data_dict)
feature_cols = parse_feature_cols(x_data_dict)
# extract data
train_vitals = TidySequentialDataCSVLoader(
x_csv_path=x_train_csv_filename,
y_csv_path=y_train_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_sequence')
test_vitals = TidySequentialDataCSVLoader(x_csv_path=x_test_csv_filename,
y_csv_path=y_test_csv_filename,
x_col_names=feature_cols,
idx_col_names=id_cols,
y_col_name=args.outcome_col_name,
y_label_type='per_sequence')
X_train, y_train = train_vitals.get_batch_data(batch_id=0)
X_test, y_test = test_vitals.get_batch_data(batch_id=0)
_, T, F = X_train.shape
print('number of time points : %s\n number of features : %s\n' % (T, F))
# set class weights as 1/(number of samples in class) for each class to handle class imbalance
class_weights = torch.tensor(
[1 / (y_train == 0).sum(), 1 / (y_train == 1).sum()]).double()
# scale features
# X_train = standard_scaler_3d(X_train)
# X_test = standard_scaler_3d(X_test)
# callback to compute gradient norm
compute_grad_norm = ComputeGradientNorm(norm_type=2)
# LSTM
if args.output_filename_prefix == None:
output_filename_prefix = (
'hiddens=%s-layers=%s-lr=%s-dropout=%s-weight_decay=%s' %
(args.hidden_units, args.hidden_layers, args.lr, args.dropout,
args.weight_decay))
else:
output_filename_prefix = args.output_filename_prefix
print('RNN parameters : ' + output_filename_prefix)
# # from IPython import embed; embed()
rnn = RNNBinaryClassifier(
max_epochs=50,
batch_size=args.batch_size,
device=device,
lr=args.lr,
callbacks=[
EpochScoring('roc_auc',
lower_is_better=False,
on_train=True,
name='aucroc_score_train'),
EpochScoring('roc_auc',
lower_is_better=False,
on_train=False,
name='aucroc_score_valid'),
EarlyStopping(monitor='aucroc_score_valid',
patience=20,
threshold=0.002,
threshold_mode='rel',
lower_is_better=False),
LRScheduler(policy=ReduceLROnPlateau,
mode='max',
monitor='aucroc_score_valid',
patience=10),
compute_grad_norm,
GradientNormClipping(gradient_clip_value=0.3,
gradient_clip_norm_type=2),
Checkpoint(monitor='aucroc_score_valid',
f_history=os.path.join(
args.output_dir, output_filename_prefix + '.json')),
TrainEndCheckpoint(dirname=args.output_dir,
fn_prefix=output_filename_prefix),
],
criterion=torch.nn.CrossEntropyLoss,
criterion__weight=class_weights,
train_split=skorch.dataset.CVSplit(args.validation_size),
module__rnn_type='LSTM',
module__n_layers=args.hidden_layers,
module__n_hiddens=args.hidden_units,
module__n_inputs=X_train.shape[-1],
module__dropout_proba=args.dropout,
optimizer=torch.optim.Adam,
optimizer__weight_decay=args.weight_decay)
clf = rnn.fit(X_train, y_train)
y_pred_proba = clf.predict_proba(X_train)
y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_train_final = roc_auc_score(y_train, y_pred_proba_pos)
print('AUROC with LSTM (Train) : %.2f' % auroc_train_final)
y_pred_proba = clf.predict_proba(X_test)
y_pred_proba_neg, y_pred_proba_pos = zip(*y_pred_proba)
auroc_test_final = roc_auc_score(y_test, y_pred_proba_pos)
print('AUROC with LSTM (Test) : %.2f' % auroc_test_final)
from skorch import NeuralNet
from skorch.callbacks import EarlyStopping
from torch.nn import MSELoss
from torch.optim import SGD
import numpy as np
import sys
sys.path.append('..')
from helpers import load_data_in_chunks, save_model
from model import Net
from CustomLoss import CustomLoss
(Xs, Ys) = load_data_in_chunks('train', chunk_size=5)
Xs = Xs.astype(np.float32)
Ys = Ys.astype(np.float32)
regr = NeuralNet(Net,
max_epochs=10000000000,
batch_size=100,
iterator_train__shuffle=True,
criterion=MSELoss,
optimizer=SGD,
optimizer__lr=1e-5,
optimizer__momentum=0.95,
verbose=5,
callbacks=[('early_stop', EarlyStopping())])
regr.fit(Xs, Ys / 5000)
save_model(regr, 'lstm-mse')
def main(cfg: DictConfig) -> None:
x, y = get_data(cfg)
assert_binary_data(x, y)
x_tr, x_te, y_tr, y_te = train_test_split(
x,
y,
test_size=cfg.evaluation.test_ratio,
random_state=_seed(cfg.seed))
mlflow.set_tracking_uri(hydra.utils.get_original_cwd() + '/mlruns')
experiment_id = get_mlflow_experiment_id(cfg.name)
with mlflow.start_run(experiment_id=experiment_id):
loss = get_loss_class(cfg)
stopping_criterion = EarlyStopping(
monitor='train_loss',
lower_is_better=True,
patience=min(10, cfg.training.max_epochs),
threshold=cfg.training.tol,
threshold_mode='rel',
sink=logger.info,
)
clf = Classifier(
module=LinearClassifier,
module__n_features=x.shape[1],
max_epochs=cfg.training.max_epochs,
criterion=loss,
predict_nonlinearity=get_loss_link(cfg),
optimizer=torch.optim.Adam,
iterator_train__batch_size=cfg.training.batch_size,
iterator_train__shuffle=True,
train_split=False,
callbacks=[('stopping_criterion', stopping_criterion)],
verbose=cfg.verbose,
)
if check_if_validation(cfg):
params = get_validation_params(cfg)
clf = GridSearchCV(
clf,
params,
refit=True,
cv=cfg.evaluation.n_cv,
scoring=negative_brier_score,
n_jobs=-1,
)
else:
clf.set_params(
lr=cfg.training.lr,
optimizer__weight_decay=cfg.training.regularization,
)
if check_if_gev_loss(loss):
clf.set_params(criterion__xi=cfg.loss.xi)
mlflow.log_param('dataset', cfg.dataset)
mlflow.log_param('loss', cfg.loss.name)
mlflow.log_param('lr', cfg.training.lr)
mlflow.log_param('max_epochs', cfg.training.max_epochs)
mlflow.log_param('tol', cfg.training.tol)
mlflow.log_param('regularization', cfg.training.regularization)
mlflow.log_param('seed', _seed(cfg.seed))
if check_if_gev_loss(loss):
mlflow.log_param('xi', cfg.loss.xi)
# Step 1: fit the base CPE model
x_tr, x_vl, y_tr, y_vl = train_test_split(x_tr, y_tr, test_size=0.3)
clf.fit(x_tr, y_tr)
# Step 2: search for the best threshold
threshold_candidates = np.arange(0.05, 1., 0.05)
validation_scores = [
f1_score(y_vl,
(clf.predict_proba(x_vl)[:, 1] > th).astype(np.uint8))
for th in threshold_candidates
]
threshold = threshold_candidates[np.argmax(validation_scores)]
y_prob = clf.predict_proba(x_te)[:, 1]
y_pred = (y_prob > threshold).astype(np.uint8)
log_metric('f1_score', f1_score(y_te, y_pred))
log_metric('roc_auc_score', roc_auc_score(y_te, y_pred))
log_metric('best_threshold', threshold)
def main(cfg: DictConfig) -> None:
x, y = get_data(cfg)
assert_binary_data(x, y)
x_tr, x_te, y_tr, y_te = train_test_split(
x,
y,
test_size=cfg.evaluation.test_ratio,
random_state=_seed(cfg.seed))
mlflow.set_tracking_uri(hydra.utils.get_original_cwd() + '/mlruns')
experiment_id = get_mlflow_experiment_id(cfg.name)
with mlflow.start_run(experiment_id=experiment_id):
positive_ratio = len(y[y == 1]) / len(y)
loss = get_loss_class(cfg)
stopping_criterion = EarlyStopping(
monitor='train_loss',
lower_is_better=True,
patience=min(10, cfg.training.max_epochs),
threshold=cfg.training.tol,
threshold_mode='rel',
sink=logger.info,
)
clf = Classifier(
module=LinearClassifier,
module__n_features=x.shape[1],
max_epochs=cfg.training.max_epochs,
criterion=loss,
predict_nonlinearity=get_loss_link(cfg),
optimizer=torch.optim.Adam,
iterator_train__batch_size=cfg.training.batch_size,
iterator_train__shuffle=True,
train_split=False,
callbacks=[('stopping_criterion', stopping_criterion)],
verbose=cfg.verbose,
)
if check_if_weight(cfg):
pos_weight = torch.FloatTensor([1 / positive_ratio - 1])
clf.set_params(criterion__pos_weight=pos_weight)
if check_if_validation(cfg):
params = get_validation_params(cfg)
clf = GridSearchCV(
clf,
params,
refit=True,
cv=cfg.evaluation.n_cv,
scoring='accuracy'
if loss is HingeLoss else negative_brier_score,
n_jobs=-1,
)
else:
clf.set_params(
lr=cfg.training.lr,
optimizer__weight_decay=cfg.training.regularization,
)
if check_if_gev_loss(loss):
clf.set_params(criterion__xi=cfg.loss.xi)
mlflow.log_param('dataset', cfg.dataset)
mlflow.log_param('dataset.positive_ratio', positive_ratio)
mlflow.log_param('loss', cfg.loss.name)
mlflow.log_param('lr', cfg.training.lr)
mlflow.log_param('max_epochs', cfg.training.max_epochs)
mlflow.log_param('tol', cfg.training.tol)
mlflow.log_param('regularization', cfg.training.regularization)
mlflow.log_param('seed', _seed(cfg.seed))
if check_if_gev_loss(loss):
mlflow.log_param('xi', cfg.loss.xi)
if loss is HingeLoss:
# Step 1: fit linear model with hinge loss
x_tr, x_vl, y_tr, y_vl = train_test_split(x, y, test_size=0.5)
clf.fit(x_tr, y_tr)
# Step 2: calibrate linear model with Platt's scaling
clf = CalibratedClassifierCV(clf, method='sigmoid', cv='prefit')
clf.fit(x_vl, y_vl)
elif cfg.loss.name == "isotonic":
# Step 1: fit linear model with logistic regression (AUC maximization)
x_tr, x_vl, y_tr, y_vl = train_test_split(x, y, test_size=0.5)
clf.fit(x_tr, y_tr)
# Step 2: calibrate linear model with isotonic regression
clf = CalibratedClassifierCV(clf, method='isotonic', cv='prefit')
clf.fit(x_vl, y_vl)
elif cfg.loss.name == "bagging":
clf = BalancedBaggingRegressor(
clf,
n_estimators=10,
bootstrap=True,
sampling_strategy='majority',
n_jobs=1,
)
clf.fit(x_tr, y_tr)
else:
clf.fit(x_tr, y_tr)
y_pred = clf.predict(x_te)
y_prob = clf.predict_proba(x_te)[:, 1]
log_metric('brier_score', brier_score_loss(y_te, y_prob))
net = NeuralNetBinaryClassifier(
module=AudioNet,
criterion=nn.BCEWithLogitsLoss,
max_epochs=5000,
lr=0.01,
optimizer=optim.SGD,
optimizer__momentum=0.9,
batch_size=160,
device=device,
callbacks=[
('tensorboard', TensorBoard(writer)),
('cp', cp),
('train_end_cp', train_end_cp),
# ("load_state", load_state),
('early_stoping', EarlyStopping(patience=5)),
('lr_scheduler', LRScheduler(
policy=ReduceLROnPlateau, monitor='valid_loss')),
],
)
print("Begin training")
try:
y_train = np.concatenate((
np.zeros((100, )), np.ones((100, )))).astype('float32')
net.fit(train_dataset, y_train)
except KeyboardInterrupt:
net.save_params(f_params=run+'.pkl')
net.save_params(f_params=run + '.pkl')
def setup_dirs():
for dir in [
"models", "optimizers", "outputs", "train_histories",
"feature_vectors"
]:
if not os.path.exists(dir):
os.makedirs(dir)
if __name__ == "__main__":
setup_dirs()
device = torch.device("cuda:0" if torch.cuda.is_available else "cpu")
lrscheduler = LRScheduler(policy='StepLR', step_size=10, gamma=0.1)
early_stopping = EarlyStopping(monitor="valid_loss",
patience=5,
threshold=0.01)
csv_path = "./dataset.csv"
train_df = pd.read_csv(csv_path)
train_df['num_label'], _ = pd.factorize(
train_df['label']) # turn labels (string) into numbers
train_df = train_df[train_df['type'] == "train"]
img_path = train_df['path'].values
labels = train_df['num_label'].values
X_train, X_val, y_train, y_val = train_test_split(img_path,
labels,
test_size=0.1,
stratify=labels,
random_state=42)
print("train set: %d, test set : %d" % (len(X_train), len(X_val)))
for model, batch_size, hist, save_model, save_opt, ep in zip(