def test_logging_append(self): train_gen = some_data_generator(self.batch_size) valid_gen = some_data_generator(self.batch_size) logger = self.CSVGradientLogger(self.csv_filename) memgrad = MemoryGradientLogger() self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[memgrad, logger]) memgrad2 = MemoryGradientLogger() logger = self.CSVGradientLogger(self.csv_filename, append=True) self.model.fit_generator( train_gen, valid_gen, epochs=20, steps_per_epoch=5, initial_epoch=self.num_epochs, callbacks=[memgrad2, logger], ) history = { layer: stats1 + memgrad2.history[layer] for layer, stats1 in memgrad.history.items() } self._test_logging(history)
def test_integration(self): train_gen = some_data_generator(OptimizerCheckpointTest.batch_size) valid_gen = some_data_generator(OptimizerCheckpointTest.batch_size) checkpointer = OptimizerCheckpoint(self.checkpoint_filename, period=1) self.model.fit_generator( train_gen, valid_gen, epochs=OptimizerCheckpointTest.epochs, steps_per_epoch=5, callbacks=[checkpointer] )
def test_logging(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) logger = self.CSVLogger(self.csv_filename) history = self.model.fit_generator( train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger] ) self._test_logging(history)
def test_logging(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) logger = TensorBoardLogger(self.writer) history = self.model.fit_generator( train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger] ) self._test_logging(history)
def test_logging_with_batch_granularity(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) logger = self.CSVLogger(self.csv_filename, batch_granularity=True) history = History() self.model.fit_generator( train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger, history] ) self._test_logging(history.history)
def setUp(self): torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) self.train_gen = some_data_generator(20) self.valid_gen = some_data_generator(20)
def test_logging_overwrite(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) logger = self.CSVLogger(self.csv_filename) self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger]) logger = self.CSVLogger(self.csv_filename, append=False) history = self.model.fit_generator( train_gen, valid_gen, epochs=20, steps_per_epoch=5, initial_epoch=self.num_epochs, callbacks=[logger] ) self._test_logging(history)
def test_logging(self): train_gen = some_data_generator(self.batch_size) valid_gen = some_data_generator(self.batch_size) memgrad = MemoryGradientLogger() logger = self.CSVGradientLogger(self.csv_filename) self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[memgrad, logger]) self._test_logging(memgrad.history)
def test_integration(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) earlystopper = EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=False) self.model.fit_generator(train_gen, valid_gen, epochs=10, steps_per_epoch=5, callbacks=[earlystopper])
def test_reduce_lr_on_plateau_integration(self): train_gen = some_data_generator(OptimizerCheckpointTest.batch_size) valid_gen = some_data_generator(OptimizerCheckpointTest.batch_size) reduce_lr = ReduceLROnPlateau(monitor='loss', patience=3) checkpointer = LRSchedulerCheckpoint(reduce_lr, self.checkpoint_filename, period=1) self.model.fit_generator(train_gen, valid_gen, epochs=OptimizerCheckpointTest.epochs, steps_per_epoch=5, callbacks=[checkpointer])
def test_any_scheduler_integration(self): train_gen = some_data_generator(OptimizerCheckpointTest.batch_size) valid_gen = some_data_generator(OptimizerCheckpointTest.batch_size) lr_scheduler = ExponentialLR(gamma=0.01) checkpointer = LRSchedulerCheckpoint(lr_scheduler, self.checkpoint_filename, period=1) self.model.fit_generator(train_gen, valid_gen, epochs=OptimizerCheckpointTest.epochs, steps_per_epoch=5, callbacks=[checkpointer])
def test_tracking_N_layers_model_with_bias(self): self.num_layer = 4 self.pytorch_network = nn.Sequential(nn.Linear(1, 1), nn.Linear(1, 1), nn.Linear(1, 1), nn.Linear(1, 1)) self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=self.lr) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) keep_bias = True train_gen = some_data_generator(20) valid_gen = some_data_generator(20) tracker = TensorBoardGradientTracker(self.writer, keep_bias=keep_bias) self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[tracker]) self._test_tracking(keep_bias)
def test_integration(self): train_gen = some_data_generator(PeriodicSaveTest.batch_size) valid_gen = some_data_generator(PeriodicSaveTest.batch_size) saver = PeriodicEpochSave(self.save_filename, monitor='val_loss', verbose=True, save_best_only=True) self.model.fit_generator(train_gen, valid_gen, epochs=10, steps_per_epoch=5, callbacks=[saver])
def test_integration(self): train_gen = some_data_generator(ModelCheckpointTest.batch_size) valid_gen = some_data_generator(ModelCheckpointTest.batch_size) checkpointer = ModelCheckpoint(self.checkpoint_filename, monitor='val_loss', verbose=True, save_best_only=True) self.model.fit_generator(train_gen, valid_gen, epochs=10, steps_per_epoch=5, callbacks=[checkpointer])
def test_multiple_learning_rates(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) logger = self.CSVLogger(self.csv_filename) lrs = [BaseCSVLoggerTest.lr, BaseCSVLoggerTest.lr / 2] optimizer = torch.optim.SGD( [dict(params=[self.pytorch_network.weight], lr=lrs[0]), dict(params=[self.pytorch_network.bias], lr=lrs[1])] ) model = Model(self.pytorch_network, optimizer, self.loss_function) history = model.fit_generator( train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger] ) self._test_logging(history, lrs=lrs)
def test_non_atomic_write(self): checkpoint_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint.ckpt') train_gen = some_data_generator(ModelCheckpointTest.batch_size) valid_gen = some_data_generator(ModelCheckpointTest.batch_size) checkpointer = ModelCheckpoint(checkpoint_filename, monitor='val_loss', verbose=True, period=1, atomic_write=False) self.model.fit_generator(train_gen, valid_gen, epochs=10, steps_per_epoch=5, callbacks=[checkpointer]) self.assertTrue(os.path.isfile(checkpoint_filename))
def _test_checkpointer_with_val_losses(self, checkpointer, val_losses, has_checkpoints): generator = some_data_generator(ModelCheckpointTest.batch_size) checkpointer.set_params({'epochs': len(val_losses), 'steps': 1}) checkpointer.set_model(self.model) checkpointer.on_train_begin({}) for epoch, (val_loss, has_checkpoint) in enumerate( zip(val_losses, has_checkpoints), 1): checkpointer.on_epoch_begin(epoch, {}) checkpointer.on_train_batch_begin(1, {}) loss = self._update_model(generator) checkpointer.on_train_batch_end(1, { 'batch': 1, 'size': ModelCheckpointTest.batch_size, 'loss': loss }) checkpointer.on_epoch_end(epoch, { 'epoch': epoch, 'loss': loss, 'val_loss': val_loss }) filename = self.checkpoint_filename.format(epoch=epoch) self.assertEqual(has_checkpoint, os.path.isfile(filename)) checkpointer.on_train_end({})
def _test_checkpointer(self, checkpointer, lr_scheduler): scheduler_states = {} generator = some_data_generator(OptimizerCheckpointTest.batch_size) checkpointer.set_params({ 'epochs': OptimizerCheckpointTest.epochs, 'steps': 1 }) checkpointer.set_model(self.model) checkpointer.on_train_begin({}) for epoch in range(1, OptimizerCheckpointTest.epochs + 1): checkpointer.on_epoch_begin(epoch, {}) checkpointer.on_train_batch_begin(1, {}) loss = self._update_model(generator) checkpointer.on_train_batch_end( 1, { 'batch': 1, 'size': OptimizerCheckpointTest.batch_size, 'loss': loss }) checkpointer.on_epoch_end(epoch, { 'epoch': epoch, 'loss': loss, 'val_loss': 1 }) filename = self.checkpoint_filename.format(epoch=epoch) self.assertTrue(os.path.isfile(filename)) scheduler_states[epoch] = torch_to_numpy( lr_scheduler.scheduler.state_dict(), copy=True) checkpointer.on_train_end({}) self._test_checkpoint(scheduler_states, lr_scheduler)
def _test_early_stopping(self, earlystopper, val_losses, early_stop_epoch): generator = some_data_generator(EarlyStoppingTest.batch_size) self.model.stop_training = False earlystopper.set_params({'epochs': len(val_losses), 'steps': 1}) earlystopper.set_model(self.model) earlystopper.on_train_begin({}) for epoch, val_loss in enumerate(val_losses, 1): earlystopper.on_epoch_begin(epoch, {}) earlystopper.on_train_batch_begin(1, {}) loss = self._update_model(generator) earlystopper.on_train_batch_end(1, { 'batch': 1, 'size': EarlyStoppingTest.batch_size, 'loss': loss }) earlystopper.on_epoch_end(epoch, { 'epoch': epoch, 'loss': loss, 'val_loss': val_loss }) self.assertEqual(self.model.stop_training, epoch == early_stop_epoch) if epoch == early_stop_epoch: break earlystopper.on_train_end({})
def _test_batch_delay(self, epoch_delay, batch_in_epoch_delay): batch_delay = epoch_delay * DelayCallbackTest.steps_per_epoch + batch_in_epoch_delay delay_callback = DelayCallback(self.mock_callback, batch_delay=batch_delay) train_generator = some_data_generator(DelayCallbackTest.batch_size) valid_generator = some_data_generator(DelayCallbackTest.batch_size) self.model.fit_generator( train_generator, valid_generator, epochs=DelayCallbackTest.epochs, steps_per_epoch=DelayCallbackTest.steps_per_epoch, validation_steps=DelayCallbackTest.steps_per_epoch, callbacks=[delay_callback]) params = { 'epochs': DelayCallbackTest.epochs, 'steps': DelayCallbackTest.steps_per_epoch, 'valid_steps': DelayCallbackTest.steps_per_epoch } call_list = [] call_list.append(call.on_train_begin({})) for epoch in range(epoch_delay + 1, DelayCallbackTest.epochs + 1): call_list.append(call.on_epoch_begin(epoch, {})) start_step = batch_in_epoch_delay + 1 if epoch == epoch_delay + 1 else 1 for step in range(start_step, params['steps'] + 1): call_list.append(call.on_train_batch_begin(step, {})) call_list.append(call.on_backward_end(step)) call_list.append( call.on_train_batch_end( step, { 'batch': step, 'size': DelayCallbackTest.batch_size, **self.train_dict })) call_list.append( call.on_epoch_end(epoch, { 'epoch': epoch, **self.log_dict })) call_list.append(call.on_train_end({})) method_calls = self.mock_callback.method_calls self.assertIn(call.set_model(self.model), method_calls[:2]) self.assertIn(call.set_params(params), method_calls[:2]) self.assertEqual(len(method_calls), len(call_list) + 2) self.assertEqual(method_calls[2:], call_list)
def test_temporary_filename_arg(self): tmp_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint.tmp.ckpt') save_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint.ckpt') train_gen = some_data_generator(PeriodicSaveTest.batch_size) valid_gen = some_data_generator(PeriodicSaveTest.batch_size) saver = PeriodicEpochSave(save_filename, monitor='val_loss', verbose=True, period=1, temporary_filename=tmp_filename) self.model.fit_generator(train_gen, valid_gen, epochs=10, steps_per_epoch=5, callbacks=[saver]) self.assertFalse(os.path.isfile(tmp_filename)) self.assertTrue(os.path.isfile(save_filename))
def test_logging_overwrite(self): train_gen = some_data_generator(self.batch_size) valid_gen = some_data_generator(self.batch_size) logger = self.CSVGradientLogger(self.csv_filename) self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger]) memgrad = MemoryGradientLogger() logger = self.CSVGradientLogger(self.csv_filename, append=False) self.model.fit_generator( train_gen, valid_gen, epochs=20, steps_per_epoch=5, initial_epoch=self.num_epochs, callbacks=[memgrad, logger], ) self._test_logging(memgrad.history)
def test_temporary_filename_arg_with_differing_save_filename(self): epochs = 10 tmp_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint.tmp.ckpt') save_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint_{epoch}.ckpt') train_gen = some_data_generator(PeriodicSaveTest.batch_size) valid_gen = some_data_generator(PeriodicSaveTest.batch_size) saver = PeriodicEpochSave(save_filename, monitor='val_loss', verbose=True, period=1, temporary_filename=tmp_filename) self.model.fit_generator(train_gen, valid_gen, epochs=epochs, steps_per_epoch=5, callbacks=[saver]) self.assertFalse(os.path.isfile(tmp_filename)) for i in range(1, epochs + 1): self.assertTrue(os.path.isfile(save_filename.format(epoch=i)))
def _test_saver_with_val_losses(self, saver, val_losses, has_checkpoints, keep_only_last_best=False): generator = some_data_generator(PeriodicSaveTest.batch_size) best_checkpoint_filenames = [] saver.set_params({'epochs': len(val_losses), 'steps': 1}) saver.set_model(self.model) saver.on_train_begin({}) for epoch, (val_loss, has_checkpoint) in enumerate( zip(val_losses, has_checkpoints), 1): saver.on_epoch_begin(epoch, {}) saver.on_train_batch_begin(1, {}) loss = self._update_model(generator) saver.on_train_batch_end(1, { 'batch': 1, 'size': PeriodicSaveTest.batch_size, 'loss': loss }) saver.on_epoch_end(epoch, { 'epoch': epoch, 'loss': loss, 'val_loss': val_loss }) filename = self.save_filename.format(epoch=epoch) self.assertEqual(has_checkpoint, os.path.isfile(filename)) if has_checkpoint: self.assertEqual(f'{epoch}\n', open(filename, 'r', encoding='utf-8').read()) best_checkpoint_filenames.append(os.path.realpath(filename)) files = [ os.path.realpath(os.path.join(self.temp_dir_obj.name, f)) for f in os.listdir(self.temp_dir_obj.name) ] if keep_only_last_best: self.assertEqual(1, len(files)) self.assertEqual(files[0], best_checkpoint_filenames[-1]) else: best_checkpoint_filenames = set(best_checkpoint_filenames) self.assertEqual(len(best_checkpoint_filenames), len(files)) self.assertEqual(best_checkpoint_filenames, set(files)) saver.on_train_end({})
def _test_restore_with_val_losses(self, checkpointer, val_losses, best_epoch): generator = some_data_generator(BestModelRestoreTest.batch_size) best_epoch_weights = None checkpointer.set_params({'epochs': len(val_losses), 'steps': 1}) checkpointer.set_model(self.model) checkpointer.on_train_begin({}) for epoch, val_loss in enumerate(val_losses, 1): checkpointer.on_epoch_begin(epoch, {}) checkpointer.on_train_batch_begin(1, {}) loss = self._update_model(generator) checkpointer.on_train_batch_end(1, {'batch': 1, 'size': BestModelRestoreTest.batch_size, 'loss': loss}) checkpointer.on_epoch_end(epoch, {'epoch': epoch, 'loss': loss, 'val_loss': val_loss}) if epoch == best_epoch: best_epoch_weights = torch_to_numpy(self.model.get_weight_copies()) checkpointer.on_train_end({}) final_weights = torch_to_numpy(self.model.get_weight_copies()) self.assertEqual(best_epoch_weights, final_weights)
def test_integration(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) model_restore = BestModelRestore(monitor='val_loss', verbose=True) self.model.fit_generator(train_gen, valid_gen, epochs=10, steps_per_epoch=5, callbacks=[model_restore])