def test_simulate_lrs_batch_step(self, policy):
lr_sch = LRScheduler(policy,
base_lr=1,
max_lr=5,
step_size_up=4,
step_every='batch')
lrs = lr_sch.simulate(11, 1)
expected = np.array([1, 2, 3, 4, 5, 4, 3, 2, 1, 2, 3])
assert np.allclose(expected, lrs)
def test_lr_scheduler_record_epoch_step(self, classifier_module,
classifier_data, policy, kwargs):
epochs = 3
scheduler = LRScheduler(policy, **kwargs)
lrs = scheduler.simulate(epochs, initial_lr=123.)
net = NeuralNetClassifier(classifier_module,
max_epochs=epochs,
lr=123.,
callbacks=[('scheduler', scheduler)])
net.fit(*classifier_data)
assert np.all(net.history[:, 'event_lr'] == lrs)
def test_lr_callback_batch_steps_correctly(
self,
classifier_module,
classifier_data,
policy,
kwargs,
):
batch_size = 100
max_epochs = 2
X, y = classifier_data
num_examples = len(X)
lr_policy = LRScheduler(policy, **kwargs)
net = NeuralNetClassifier(classifier_module(),
max_epochs=max_epochs,
batch_size=batch_size,
callbacks=[lr_policy])
net.fit(X, y)
total_iterations_per_epoch = num_examples / batch_size
# 80% of sample used for training by default
total_training_iterations_per_epoch = 0.8 * total_iterations_per_epoch
expected = int(total_training_iterations_per_epoch * max_epochs)
# pylint: disable=protected-access
assert lr_policy.batch_idx_ == expected
def get_net_with_mock(self,
classifier_data,
classifier_module,
monitor='train_loss'):
"""Returns a net with a mocked lr policy that allows to check what
it's step method was called with.
"""
X, y = classifier_data
net = NeuralNetClassifier(
classifier_module,
callbacks=[
('scheduler', LRScheduler(ReduceLROnPlateau, monitor=monitor)),
],
max_epochs=1,
).fit(X, y)
# mock the policy
policy = dict(net.callbacks_)['scheduler'].lr_scheduler_
mock_step = Mock(side_effect=policy.step)
policy.step = mock_step
# make sure that mocked policy is set
scheduler = dict(net.callbacks_)['scheduler']
# pylint: disable=protected-access
scheduler._get_scheduler = lambda *args, **kwargs: policy
net.partial_fit(X, y)
return net, mock_step
def train_model(wavelet_scale, scattering_operators):
print('Start training model ' f'{wavelet_scale} {scattering_operators}')
model_save_path = osp.join(
MODELS_SAVE_PATH, f'model_{wavelet_scale}_'
f'{scop_to_str(scattering_operators)}.pkl')
if not osp.exists(model_save_path):
data_loader = DataLoader(
DATASET, f'data_{wavelet_scale}_'
f'{scop_to_str(scattering_operators)}')
data = data_loader.load_data()
x_train, y_train, _ = data[f'{DATASET}_training']
x_train = x_train.astype(np.float32)
y_train = y_train.astype(np.float32)
# normalize data to 0 mean and unit std
scaler = StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
n_in = x_train.shape[1]
lr_policy = LRScheduler(StepLR, step_size=15, gamma=0.5)
net = NeuralNetRegressor(
GSGNN,
module__n_in=n_in,
criterion=torch.nn.MSELoss,
max_epochs=400,
optimizer=torch.optim.Adam,
optimizer__lr=.005,
callbacks=[lr_policy],
device='cpu',
batch_size=256,
verbose=0,
)
params = {
'module__n_h': [100, 200, 300, 400],
'module__dropout': [0.0, 0.2, 0.4],
'module__n_layers': [1, 2, 3, 4],
}
gs = GridSearchCV(net,
params,
refit=True,
cv=5,
scoring='r2',
n_jobs=-1)
gs.fit(x_train, y_train)
# save the trained model
print(f"Save the model in {model_save_path}")
torch.save(gs.best_estimator_, model_save_path)
report(gs.cv_results_, 10)
def test_lr_scheduler_set_params(self, classifier_module, classifier_data):
scheduler = LRScheduler(CyclicLR, base_lr=123)
net = NeuralNetClassifier(
classifier_module,
max_epochs=0,
callbacks=[('scheduler', scheduler)],
)
net.set_params(callbacks__scheduler__base_lr=456)
net.fit(*classifier_data) # we need to trigger on_train_begin
assert net.callbacks[0][1].lr_scheduler_.base_lrs[0] == 456
def test_lr_scheduler_record_batch_step(self, classifier_module,
classifier_data):
X, y = classifier_data
batch_size = 128
scheduler = LRScheduler(TorchCyclicLR,
base_lr=1,
max_lr=5,
step_size_up=4)
net = NeuralNetClassifier(classifier_module,
max_epochs=1,
lr=123.,
batch_size=batch_size,
callbacks=[('scheduler', scheduler)])
net.fit(X, y)
new_lrs = scheduler.simulate(
net.history[-1, 'train_batch_count'],
initial_lr=123.,
)
assert np.all(net.history[-1, 'batches', :, 'event_lr'] == new_lrs)
def test_cyclic_lr_with_epoch_step_warning(self, classifier_module,
classifier_data):
msg = ("The LRScheduler now makes a step every epoch by default. "
"To have the cyclic lr scheduler update "
"every batch set step_every='batch'")
with pytest.warns(FutureWarning, match=msg) as record:
scheduler = LRScheduler(TorchCyclicLR, base_lr=123, max_lr=999)
net = NeuralNetClassifier(
classifier_module,
max_epochs=0,
callbacks=[('scheduler', scheduler)],
)
net.initialize()
assert len(record) == 1
def _lr_callback_init_policies(self, classifier_module, policy, instance,
**kwargs):
X, y = make_classification(1000, 20, n_informative=10, random_state=0)
X = X.astype(np.float32)
lr_policy = LRScheduler(policy, **kwargs)
net = NeuralNetClassifier(classifier_module,
max_epochs=2,
callbacks=[lr_policy])
net.fit(X, y)
assert any(
list(
map(
lambda x: isinstance(getattr(x[1], '_lr_scheduler', None),
instance), net.callbacks_)))
def test_reduce_lr_monitor_max(
self, classifier_data, classifier_module, mode, score):
X, y = classifier_data
net = NeuralNetClassifier(
classifier_module,
callbacks=[
('scheduler', LRScheduler(
ReduceLROnPlateau, monitor='train_loss', mode=mode)),
],
max_epochs=1,
)
net.fit(X, y)
policy = dict(net.callbacks_)['scheduler'].lr_scheduler_
assert policy.best == score
def test_reduce_lr_raise_error_when_key_does_not_exist(
self, classifier_data, classifier_module):
X, y = classifier_data
net = NeuralNetClassifier(
classifier_module,
callbacks=[
('scheduler', LRScheduler(
ReduceLROnPlateau, monitor='bad_key')),
],
max_epochs=1,
)
msg = ("'bad_key' was not found in history. A Scoring "
"callback with name='bad_key' should be placed before the "
"LRScheduler callback")
with pytest.raises(ValueError, match=msg):
net.fit(X, y)
def test_reduce_lr_monitor_passes_monitored_loss(
self, classifier_data, classifier_module, mode):
X, y = classifier_data
net = NeuralNetClassifier(
classifier_module,
callbacks=[
('scheduler', LRScheduler(
ReduceLROnPlateau, monitor='valid_loss', mode=mode)),
],
max_epochs=1,
)
net.fit(X, y)
expected = net.history_[-1, "valid_loss"]
policy = dict(net.callbacks_)['scheduler'].lr_scheduler_
assert policy.best == pytest.approx(expected)
def test_lr_callback_batch_steps_correctly_fallback(
self,
classifier_module,
classifier_data,
policy,
kwargs,
):
batch_size = 100
max_epochs = 2
X, y = classifier_data
num_examples = len(X)
lr_policy = LRScheduler(policy, **kwargs)
net = NeuralNetClassifier(classifier_module(),
max_epochs=max_epochs,
batch_size=batch_size,
callbacks=[lr_policy])
net.fit(X, y)
# Removes batch count information in the last two epochs
for i in range(max_epochs):
del net.history[i]["train_batch_count"]
del net.history[i]["valid_batch_count"]
net.partial_fit(X, y)
total_iterations_per_epoch = num_examples / batch_size
# batch_counts were removed thus the total iterations of the last
# epoch is used
total_iterations_fit_run = total_iterations_per_epoch * max_epochs
# 80% of sample used for training by default
total_iterations_partial_fit_run = (0.8 * total_iterations_per_epoch *
max_epochs)
# called fit AND partial_fit
total_iterations = (total_iterations_fit_run +
total_iterations_partial_fit_run)
# Failback to using both valid and training batches counts on
# second run
expected = int(total_iterations)
# pylint: disable=protected-access
assert lr_policy.batch_idx_ == expected
def test_lr_callback_batch_steps_correctly(
self,
classifier_module,
classifier_data,
policy,
kwargs,
):
num_examples = 1000
batch_size = 100
max_epochs = 2
X, y = classifier_data
lr_policy = LRScheduler(policy, **kwargs)
net = NeuralNetClassifier(classifier_module(), max_epochs=max_epochs,
batch_size=batch_size, callbacks=[lr_policy])
net.fit(X, y)
expected = (num_examples // batch_size) * max_epochs - 1
# pylint: disable=protected-access
assert lr_policy.lr_scheduler_.last_batch_idx == expected
def test_lr_callback_steps_correctly(
self,
classifier_module,
classifier_data,
policy,
kwargs,
):
max_epochs = 2
X, y = classifier_data
lr_policy = LRScheduler(policy, **kwargs)
net = NeuralNetClassifier(
classifier_module(),
max_epochs=max_epochs,
batch_size=16,
callbacks=[lr_policy],
)
net.fit(X, y)
# pylint: disable=protected-access
assert lr_policy.lr_scheduler_.last_epoch == max_epochs - 1
def test_lr_callback_init_policies(
self,
classifier_module,
classifier_data,
policy,
instance,
kwargs,
):
X, y = classifier_data
lr_policy = LRScheduler(policy, **kwargs)
net = NeuralNetClassifier(classifier_module,
max_epochs=2,
callbacks=[lr_policy])
net.fit(X, y)
assert any(
list(
map(
lambda x: isinstance(getattr(x[1], 'lr_scheduler_', None),
instance), net.callbacks_)))
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params('./histories/%i_valid_best_params.pt' % k)
nets = []
# Fold the CV data
k_folder = KFold(n_splits=5)
for k, (indices_train, _) in enumerate(k_folder.split(sdts_train)):
stds_train_ = [sdts_train[index] for index in indices_train]
targets_train_ = np.array(
[targets_train[index] for index in indices_train])
# Define various callbacks and checkpointers for this network
LR_schedule = LRScheduler('MultiStepLR', milestones=[75], gamma=0.1)
cp = Checkpoint(monitor='valid_loss_best',
fn_prefix='./histories/%i_valid_best_' % k)
load_best_valid_loss = train_end_load_best_valid_loss()
# Train this fold's network
net = NeuralNetRegressor(CrystalGraphConvNet,
module__orig_atom_fea_len=orig_atom_fea_len,
module__nbr_fea_len=nbr_fea_len,
batch_size=214,
module__classification=False,
lr=0.0056,
max_epochs=100,
module__atom_fea_len=46,
module__h_fea_len=83,
module__n_conv=8,
def test_simulate_lrs_epoch_step(self):
lr_policy = LRScheduler(StepLR, step_size=2)
lrs = lr_policy.simulate(6, 1)
expected = np.array([1.0, 1.0, 0.1, 0.1, 0.01, 0.01])
assert np.allclose(expected, lrs)
def test_simulate_lrs_batch_step(self):
lr_policy = LRScheduler(CyclicLR, base_lr=1, max_lr=5, step_size_up=4)
lrs = lr_policy.simulate(11, 1)
expected = np.array([1, 2, 3, 4, 5, 4, 3, 2, 1, 2, 3])
assert np.allclose(expected, lrs)
def test_raise_invalid_policy_string(self):
with pytest.raises(AttributeError):
LRScheduler("invalid_policy")
def test_raise_invalid_policy_class(self):
class DummyClass():
pass
with pytest.raises(AssertionError):
LRScheduler(DummyClass)
def test_lr_scheduler_cloneable(self):
# reproduces bug #271
scheduler = LRScheduler(CyclicLR, base_lr=123)
clone(scheduler) # does not raise
from amptorch.analysis import parity_plot
import torch
from torch.nn import init
from ase import Atoms
from ase.calculators.emt import EMT
from ase.io import read
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params("valid_best_params.pt")
LR_schedule = LRScheduler("CosineAnnealingLR", T_max=5)
# saves best validation loss
cp = Checkpoint(monitor="valid_loss_best", fn_prefix="valid_best_")
# loads best validation loss at the end of training
load_best_valid_loss = train_end_load_best_valid_loss()
distances = np.linspace(2, 5, 10)
label = "delta_ml_example"
images = []
for l in distances:
image = Atoms(
"CuCO",
[
(-l * np.sin(0.65), l * np.cos(0.65), 0),
(0, 0, 0),
(l * np.sin(0.65), l * np.cos(0.65), 0),
from amptorch.analysis import parity_plot
from torch.utils.data import DataLoader
from torch.nn import init
from skorch.utils import to_numpy
import numpy as np
from ase import Atoms
from ase.calculators.emt import EMT
from ase.io import read
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback):
def on_train_end(self, net, X, y):
net.load_params('valid_best_params.pt')
LR_schedule = LRScheduler('CosineAnnealingLR', T_max=5)
# saves best validation loss
cp = Checkpoint(monitor='valid_loss_best', fn_prefix='valid_best_')
# loads best validation loss at the end of training
load_best_valid_loss = train_end_load_best_valid_loss()
distances = np.linspace(2, 5, 100)
label = "skorch_example"
images = []
for l in distances:
image = Atoms(
"CuCO",
[
(-l * np.sin(0.65), l * np.cos(0.65), 0),
(0, 0, 0),
(l * np.sin(0.65), l * np.cos(0.65), 0),
