def test_simulate_lrs_batch_step(self, policy): lr_sch = LRScheduler(policy, base_lr=1, max_lr=5, step_size_up=4, step_every='batch') lrs = lr_sch.simulate(11, 1) expected = np.array([1, 2, 3, 4, 5, 4, 3, 2, 1, 2, 3]) assert np.allclose(expected, lrs)
def test_lr_scheduler_record_epoch_step(self, classifier_module, classifier_data, policy, kwargs): epochs = 3 scheduler = LRScheduler(policy, **kwargs) lrs = scheduler.simulate(epochs, initial_lr=123.) net = NeuralNetClassifier(classifier_module, max_epochs=epochs, lr=123., callbacks=[('scheduler', scheduler)]) net.fit(*classifier_data) assert np.all(net.history[:, 'event_lr'] == lrs)
def test_lr_callback_batch_steps_correctly( self, classifier_module, classifier_data, policy, kwargs, ): batch_size = 100 max_epochs = 2 X, y = classifier_data num_examples = len(X) lr_policy = LRScheduler(policy, **kwargs) net = NeuralNetClassifier(classifier_module(), max_epochs=max_epochs, batch_size=batch_size, callbacks=[lr_policy]) net.fit(X, y) total_iterations_per_epoch = num_examples / batch_size # 80% of sample used for training by default total_training_iterations_per_epoch = 0.8 * total_iterations_per_epoch expected = int(total_training_iterations_per_epoch * max_epochs) # pylint: disable=protected-access assert lr_policy.batch_idx_ == expected
def get_net_with_mock(self, classifier_data, classifier_module, monitor='train_loss'): """Returns a net with a mocked lr policy that allows to check what it's step method was called with. """ X, y = classifier_data net = NeuralNetClassifier( classifier_module, callbacks=[ ('scheduler', LRScheduler(ReduceLROnPlateau, monitor=monitor)), ], max_epochs=1, ).fit(X, y) # mock the policy policy = dict(net.callbacks_)['scheduler'].lr_scheduler_ mock_step = Mock(side_effect=policy.step) policy.step = mock_step # make sure that mocked policy is set scheduler = dict(net.callbacks_)['scheduler'] # pylint: disable=protected-access scheduler._get_scheduler = lambda *args, **kwargs: policy net.partial_fit(X, y) return net, mock_step
def train_model(wavelet_scale, scattering_operators): print('Start training model ' f'{wavelet_scale} {scattering_operators}') model_save_path = osp.join( MODELS_SAVE_PATH, f'model_{wavelet_scale}_' f'{scop_to_str(scattering_operators)}.pkl') if not osp.exists(model_save_path): data_loader = DataLoader( DATASET, f'data_{wavelet_scale}_' f'{scop_to_str(scattering_operators)}') data = data_loader.load_data() x_train, y_train, _ = data[f'{DATASET}_training'] x_train = x_train.astype(np.float32) y_train = y_train.astype(np.float32) # normalize data to 0 mean and unit std scaler = StandardScaler() scaler.fit(x_train) x_train = scaler.transform(x_train) n_in = x_train.shape[1] lr_policy = LRScheduler(StepLR, step_size=15, gamma=0.5) net = NeuralNetRegressor( GSGNN, module__n_in=n_in, criterion=torch.nn.MSELoss, max_epochs=400, optimizer=torch.optim.Adam, optimizer__lr=.005, callbacks=[lr_policy], device='cpu', batch_size=256, verbose=0, ) params = { 'module__n_h': [100, 200, 300, 400], 'module__dropout': [0.0, 0.2, 0.4], 'module__n_layers': [1, 2, 3, 4], } gs = GridSearchCV(net, params, refit=True, cv=5, scoring='r2', n_jobs=-1) gs.fit(x_train, y_train) # save the trained model print(f"Save the model in {model_save_path}") torch.save(gs.best_estimator_, model_save_path) report(gs.cv_results_, 10)
def test_lr_scheduler_set_params(self, classifier_module, classifier_data): scheduler = LRScheduler(CyclicLR, base_lr=123) net = NeuralNetClassifier( classifier_module, max_epochs=0, callbacks=[('scheduler', scheduler)], ) net.set_params(callbacks__scheduler__base_lr=456) net.fit(*classifier_data) # we need to trigger on_train_begin assert net.callbacks[0][1].lr_scheduler_.base_lrs[0] == 456
def test_lr_scheduler_record_batch_step(self, classifier_module, classifier_data): X, y = classifier_data batch_size = 128 scheduler = LRScheduler(TorchCyclicLR, base_lr=1, max_lr=5, step_size_up=4) net = NeuralNetClassifier(classifier_module, max_epochs=1, lr=123., batch_size=batch_size, callbacks=[('scheduler', scheduler)]) net.fit(X, y) new_lrs = scheduler.simulate( net.history[-1, 'train_batch_count'], initial_lr=123., ) assert np.all(net.history[-1, 'batches', :, 'event_lr'] == new_lrs)
def test_cyclic_lr_with_epoch_step_warning(self, classifier_module, classifier_data): msg = ("The LRScheduler now makes a step every epoch by default. " "To have the cyclic lr scheduler update " "every batch set step_every='batch'") with pytest.warns(FutureWarning, match=msg) as record: scheduler = LRScheduler(TorchCyclicLR, base_lr=123, max_lr=999) net = NeuralNetClassifier( classifier_module, max_epochs=0, callbacks=[('scheduler', scheduler)], ) net.initialize() assert len(record) == 1
def _lr_callback_init_policies(self, classifier_module, policy, instance, **kwargs): X, y = make_classification(1000, 20, n_informative=10, random_state=0) X = X.astype(np.float32) lr_policy = LRScheduler(policy, **kwargs) net = NeuralNetClassifier(classifier_module, max_epochs=2, callbacks=[lr_policy]) net.fit(X, y) assert any( list( map( lambda x: isinstance(getattr(x[1], '_lr_scheduler', None), instance), net.callbacks_)))
def test_reduce_lr_monitor_max( self, classifier_data, classifier_module, mode, score): X, y = classifier_data net = NeuralNetClassifier( classifier_module, callbacks=[ ('scheduler', LRScheduler( ReduceLROnPlateau, monitor='train_loss', mode=mode)), ], max_epochs=1, ) net.fit(X, y) policy = dict(net.callbacks_)['scheduler'].lr_scheduler_ assert policy.best == score
def test_reduce_lr_raise_error_when_key_does_not_exist( self, classifier_data, classifier_module): X, y = classifier_data net = NeuralNetClassifier( classifier_module, callbacks=[ ('scheduler', LRScheduler( ReduceLROnPlateau, monitor='bad_key')), ], max_epochs=1, ) msg = ("'bad_key' was not found in history. A Scoring " "callback with name='bad_key' should be placed before the " "LRScheduler callback") with pytest.raises(ValueError, match=msg): net.fit(X, y)
def test_reduce_lr_monitor_passes_monitored_loss( self, classifier_data, classifier_module, mode): X, y = classifier_data net = NeuralNetClassifier( classifier_module, callbacks=[ ('scheduler', LRScheduler( ReduceLROnPlateau, monitor='valid_loss', mode=mode)), ], max_epochs=1, ) net.fit(X, y) expected = net.history_[-1, "valid_loss"] policy = dict(net.callbacks_)['scheduler'].lr_scheduler_ assert policy.best == pytest.approx(expected)
def test_lr_callback_batch_steps_correctly_fallback( self, classifier_module, classifier_data, policy, kwargs, ): batch_size = 100 max_epochs = 2 X, y = classifier_data num_examples = len(X) lr_policy = LRScheduler(policy, **kwargs) net = NeuralNetClassifier(classifier_module(), max_epochs=max_epochs, batch_size=batch_size, callbacks=[lr_policy]) net.fit(X, y) # Removes batch count information in the last two epochs for i in range(max_epochs): del net.history[i]["train_batch_count"] del net.history[i]["valid_batch_count"] net.partial_fit(X, y) total_iterations_per_epoch = num_examples / batch_size # batch_counts were removed thus the total iterations of the last # epoch is used total_iterations_fit_run = total_iterations_per_epoch * max_epochs # 80% of sample used for training by default total_iterations_partial_fit_run = (0.8 * total_iterations_per_epoch * max_epochs) # called fit AND partial_fit total_iterations = (total_iterations_fit_run + total_iterations_partial_fit_run) # Failback to using both valid and training batches counts on # second run expected = int(total_iterations) # pylint: disable=protected-access assert lr_policy.batch_idx_ == expected
def test_lr_callback_batch_steps_correctly( self, classifier_module, classifier_data, policy, kwargs, ): num_examples = 1000 batch_size = 100 max_epochs = 2 X, y = classifier_data lr_policy = LRScheduler(policy, **kwargs) net = NeuralNetClassifier(classifier_module(), max_epochs=max_epochs, batch_size=batch_size, callbacks=[lr_policy]) net.fit(X, y) expected = (num_examples // batch_size) * max_epochs - 1 # pylint: disable=protected-access assert lr_policy.lr_scheduler_.last_batch_idx == expected
def test_lr_callback_steps_correctly( self, classifier_module, classifier_data, policy, kwargs, ): max_epochs = 2 X, y = classifier_data lr_policy = LRScheduler(policy, **kwargs) net = NeuralNetClassifier( classifier_module(), max_epochs=max_epochs, batch_size=16, callbacks=[lr_policy], ) net.fit(X, y) # pylint: disable=protected-access assert lr_policy.lr_scheduler_.last_epoch == max_epochs - 1
def test_lr_callback_init_policies( self, classifier_module, classifier_data, policy, instance, kwargs, ): X, y = classifier_data lr_policy = LRScheduler(policy, **kwargs) net = NeuralNetClassifier(classifier_module, max_epochs=2, callbacks=[lr_policy]) net.fit(X, y) assert any( list( map( lambda x: isinstance(getattr(x[1], 'lr_scheduler_', None), instance), net.callbacks_)))
class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params('./histories/%i_valid_best_params.pt' % k) nets = [] # Fold the CV data k_folder = KFold(n_splits=5) for k, (indices_train, _) in enumerate(k_folder.split(sdts_train)): stds_train_ = [sdts_train[index] for index in indices_train] targets_train_ = np.array( [targets_train[index] for index in indices_train]) # Define various callbacks and checkpointers for this network LR_schedule = LRScheduler('MultiStepLR', milestones=[75], gamma=0.1) cp = Checkpoint(monitor='valid_loss_best', fn_prefix='./histories/%i_valid_best_' % k) load_best_valid_loss = train_end_load_best_valid_loss() # Train this fold's network net = NeuralNetRegressor(CrystalGraphConvNet, module__orig_atom_fea_len=orig_atom_fea_len, module__nbr_fea_len=nbr_fea_len, batch_size=214, module__classification=False, lr=0.0056, max_epochs=100, module__atom_fea_len=46, module__h_fea_len=83, module__n_conv=8,
def test_simulate_lrs_epoch_step(self): lr_policy = LRScheduler(StepLR, step_size=2) lrs = lr_policy.simulate(6, 1) expected = np.array([1.0, 1.0, 0.1, 0.1, 0.01, 0.01]) assert np.allclose(expected, lrs)
def test_simulate_lrs_batch_step(self): lr_policy = LRScheduler(CyclicLR, base_lr=1, max_lr=5, step_size_up=4) lrs = lr_policy.simulate(11, 1) expected = np.array([1, 2, 3, 4, 5, 4, 3, 2, 1, 2, 3]) assert np.allclose(expected, lrs)
def test_raise_invalid_policy_string(self): with pytest.raises(AttributeError): LRScheduler("invalid_policy")
def test_raise_invalid_policy_class(self): class DummyClass(): pass with pytest.raises(AssertionError): LRScheduler(DummyClass)
def test_lr_scheduler_cloneable(self): # reproduces bug #271 scheduler = LRScheduler(CyclicLR, base_lr=123) clone(scheduler) # does not raise
from amptorch.analysis import parity_plot import torch from torch.nn import init from ase import Atoms from ase.calculators.emt import EMT from ase.io import read class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params("valid_best_params.pt") LR_schedule = LRScheduler("CosineAnnealingLR", T_max=5) # saves best validation loss cp = Checkpoint(monitor="valid_loss_best", fn_prefix="valid_best_") # loads best validation loss at the end of training load_best_valid_loss = train_end_load_best_valid_loss() distances = np.linspace(2, 5, 10) label = "delta_ml_example" images = [] for l in distances: image = Atoms( "CuCO", [ (-l * np.sin(0.65), l * np.cos(0.65), 0), (0, 0, 0), (l * np.sin(0.65), l * np.cos(0.65), 0),
from amptorch.analysis import parity_plot from torch.utils.data import DataLoader from torch.nn import init from skorch.utils import to_numpy import numpy as np from ase import Atoms from ase.calculators.emt import EMT from ase.io import read class train_end_load_best_valid_loss(skorch.callbacks.base.Callback): def on_train_end(self, net, X, y): net.load_params('valid_best_params.pt') LR_schedule = LRScheduler('CosineAnnealingLR', T_max=5) # saves best validation loss cp = Checkpoint(monitor='valid_loss_best', fn_prefix='valid_best_') # loads best validation loss at the end of training load_best_valid_loss = train_end_load_best_valid_loss() distances = np.linspace(2, 5, 100) label = "skorch_example" images = [] for l in distances: image = Atoms( "CuCO", [ (-l * np.sin(0.65), l * np.cos(0.65), 0), (0, 0, 0), (l * np.sin(0.65), l * np.cos(0.65), 0),