class ModelDictOutputTest(ModelFittingTestCase):
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = DictOutputModel()
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(
self.pytorch_network,
self.optimizer,
lambda y_p, y_t: self.loss_function(y_p['out1'], y_t[0]) + self.loss_function(y_p['out2'], y_t[1]),
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def test_fitting_with_tensor_multi_output_dict(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelDictOutputTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = torch.rand(train_size, 1)
train_y = (torch.rand(train_size, 1), torch.rand(train_size, 1))
valid_real_steps_per_epoch = 10
# valid_batch_size will be the same as train_batch_size in the fit method.
valid_batch_size = train_batch_size
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = torch.rand(valid_size, 1)
valid_y = (torch.rand(valid_size, 1), torch.rand(valid_size, 1))
logs = self.model.fit(train_x,
train_y,
validation_data=(valid_x, valid_y),
epochs=ModelDictOutputTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {'epochs': ModelDictOutputTest.epochs, 'steps': train_real_steps_per_epoch}
self._test_callbacks_train(params, logs)
def test_ndarray_train_on_batch_dict_output(self):
x = np.random.rand(ModelDictOutputTest.batch_size, 1).astype(np.float32)
y1 = np.random.rand(ModelDictOutputTest.batch_size, 1).astype(np.float32)
y2 = np.random.rand(ModelDictOutputTest.batch_size, 1).astype(np.float32)
loss = self.model.train_on_batch(x, (y1, y2))
self.assertEqual(type(loss), float)
def test_evaluate_with_pred_dict_output(self):
y = (torch.rand(ModelDictOutputTest.evaluate_dataset_len,
1), torch.rand(ModelDictOutputTest.evaluate_dataset_len, 1))
x = torch.rand(ModelDictOutputTest.evaluate_dataset_len, 1)
# We also test the unpacking.
_, pred_y = self.model.evaluate(x, y, batch_size=ModelDictOutputTest.batch_size, return_pred=True)
for pred in pred_y.values():
self.assertEqual(pred.shape, (ModelDictOutputTest.evaluate_dataset_len, 1))
def setUp(self):
super().setUp()
self.train_generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
self.valid_generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
self.test_generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.batch_metrics = [
some_batch_metric_1, ('custom_name', some_batch_metric_2), repeat_batch_metric, repeat_batch_metric
]
self.batch_metrics_names = [
'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1', 'repeat_batch_metric2'
]
self.batch_metrics_values = [
some_metric_1_value, some_metric_2_value, repeat_batch_metric_value, repeat_batch_metric_value
]
self.epoch_metrics = [SomeConstantEpochMetric()]
self.epoch_metrics_names = ['some_constant_epoch_metric']
self.epoch_metrics_values = [some_constant_epoch_metric_value]
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
self._capture_output()
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.batch_metrics = [
some_batch_metric_1, ('custom_name', some_batch_metric_2),
repeat_batch_metric, repeat_batch_metric
]
self.batch_metrics_names = [
'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1',
'repeat_batch_metric2'
]
self.batch_metrics_values = [
some_metric_1_value, some_metric_2_value,
repeat_batch_metric_value, repeat_batch_metric_value
]
self.epoch_metrics = [SomeConstantEpochMetric()]
self.epoch_metrics_names = ['some_constant_epoch_metric']
self.epoch_metrics_values = [some_constant_epoch_metric_value]
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
self.default_main_device = ModelTestMultiGPU.cuda_device
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.model = Model(self.pytorch_network, self.optimizer,
self.loss_function)
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.Adam(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
self.temp_dir_obj = TemporaryDirectory()
self.checkpoint_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint_{epoch}.optim')
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Sequential(nn.Linear(1, 2), nn.Linear(2, 1))
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
self.temp_dir_obj = TemporaryDirectory()
self.csv_filename = os.path.join(self.temp_dir_obj.name, 'layer_{}.csv')
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=BaseCSVLoggerTest.lr)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
self.temp_dir_obj = TemporaryDirectory()
self.csv_filename = os.path.join(self.temp_dir_obj.name, 'my_log.csv')
def test_poutyne():
callback = PlotLossesPoutyne(outputs=(CheckOutput(), ))
network = Network()
optimizer = optim.Adam(params=network.parameters(), lr=0.001)
loss_fn = nn.CrossEntropyLoss()
train_dataloader = get_random_data()
model = Model(network, optimizer, loss_fn)
model.fit_generator(train_dataloader, epochs=2, callbacks=[callback])
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.model = Model(self.pytorch_network, self.optimizer,
self.loss_function)
self.train_gen = some_data_generator(20)
self.valid_gen = some_data_generator(20)
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
self.mock_callback = MagicMock(spec=Callback)
self.delay_callback = DelayCallback(self.mock_callback)
self.train_dict = {'loss': ANY, 'time': ANY}
self.log_dict = {'loss': ANY, 'val_loss': ANY, 'time': ANY}
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=BaseTensorBoardLoggerTest.lr)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
self.temp_dir_obj = TemporaryDirectory()
# pylint: disable=not-callable
self.writer = self.SummaryWriter(self.temp_dir_obj.name)
self.writer.add_scalars = MagicMock()
def test_epoch_metrics_with_multiple_names_returned_by_tensor_on_gpu(self):
with torch.cuda.device(MetricsModelIntegrationTest.cuda_device):
epoch_metric = ConstEpochMetric(
torch.tensor(self.metric_values).cuda())
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
epoch_metrics=[(self.metric_names, epoch_metric)])
model.cuda()
self._test_history(model, self.metric_names, self.metric_values)
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = DictIOModel(['x1', 'x2'], ['y1', 'y2'])
self.loss_function = dict_mse_loss
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def test_tracking_N_layers_model_with_bias(self):
self.num_layer = 4
self.pytorch_network = nn.Sequential(nn.Linear(1, 1), nn.Linear(1, 1), nn.Linear(1, 1), nn.Linear(1, 1))
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=self.lr)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
keep_bias = True
train_gen = some_data_generator(20)
valid_gen = some_data_generator(20)
tracker = TensorBoardGradientTracker(self.writer, keep_bias=keep_bias)
self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[tracker])
self._test_tracking(keep_bias)
class LRSchedulersTest(TestCase):
batch_size = 20
epochs = 10
steps_per_epoch = 5
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.model = Model(self.pytorch_network, self.optimizer,
self.loss_function)
self.train_gen = some_data_generator(20)
self.valid_gen = some_data_generator(20)
def test_lambda_lr_integration(self):
my_lambda = lambda epoch: 0.95**epoch
lambda_lr = LambdaLR(lr_lambda=[my_lambda])
self._fit_with_callback_integration(lambda_lr)
def test_step_lr_integration(self):
step_lr = StepLR(step_size=3)
self._fit_with_callback_integration(step_lr)
def test_multistep_lr_integration(self):
multistep_lr = MultiStepLR(milestones=[2, 5, 7])
self._fit_with_callback_integration(multistep_lr)
def test_exponential_lr_integration(self):
exponential_lr = ExponentialLR(gamma=0.01)
self._fit_with_callback_integration(exponential_lr)
def test_cosine_annealing_lr_integration(self):
cosine_annealing_lr = CosineAnnealingLR(T_max=8)
self._fit_with_callback_integration(cosine_annealing_lr)
def test_reduce_lr_on_plateau_integration(self):
reduce_lr = ReduceLROnPlateau(monitor='loss', patience=3)
self._fit_with_callback_integration(reduce_lr)
def _fit_with_callback_integration(self, callback):
self.model.fit_generator(
self.train_gen,
self.valid_gen,
epochs=LRSchedulersTest.epochs,
steps_per_epoch=LRSchedulersTest.steps_per_epoch,
callbacks=[callback],
)
def test_exception_is_thrown_on_optimizer_argument(self):
with self.assertRaises(ValueError):
StepLR(self.optimizer, step_size=3)
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = MultiIOModel(num_input=1, num_output=1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = MultiIOModel(num_input=2, num_output=2)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(
self.pytorch_network,
self.optimizer,
lambda y_pred, y_true: self.loss_function(y_pred[0], y_true[0]) + self.loss_function(y_pred[1], y_true[1]),
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = DictOutputModel()
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.model = Model(
self.pytorch_network,
self.optimizer,
lambda y_p, y_t: self.loss_function(y_p['out1'], y_t[0]) + self.loss_function(y_p['out2'], y_t[1]),
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def test_multiple_learning_rates(self):
train_gen = some_data_generator(20)
valid_gen = some_data_generator(20)
logger = self.CSVLogger(self.csv_filename)
lrs = [BaseCSVLoggerTest.lr, BaseCSVLoggerTest.lr / 2]
optimizer = torch.optim.SGD(
[dict(params=[self.pytorch_network.weight], lr=lrs[0]), dict(params=[self.pytorch_network.bias], lr=lrs[1])]
)
model = Model(self.pytorch_network, optimizer, self.loss_function)
history = model.fit_generator(
train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger]
)
self._test_logging(history, lrs=lrs)
def test_evaluate_with_only_one_metric(self):
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics[:1])
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, first_metric = model.evaluate(x,
y,
batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(first_metric), float)
self.assertEqual(first_metric, some_metric_1_value)
def test_epoch_metrics_with_multiple_names_returned_by_tuple(self):
epoch_metric = ConstEpochMetric(tuple(self.metric_values))
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
epoch_metrics=[(self.metric_names, epoch_metric)])
self._test_history(model, self.metric_names, self.metric_values)
def test_batch_metrics_with_multiple_names_returned_by_list(self):
batch_metric = get_const_batch_metric(list(self.metric_values))
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=[(self.metric_names, batch_metric)])
self._test_history(model, self.metric_names, self.metric_values)
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = nn.Sequential(nn.Flatten(),
nn.Linear(28 * 28, 10))
self.batch_metrics = ['accuracy']
self.batch_metrics_names = ['acc']
self.batch_metrics_values = [ANY]
self.epoch_metrics = ['f1']
self.epoch_metrics_names = ['fscore_micro']
self.epoch_metrics_values = [ANY]
self.model = Model(self.pytorch_network,
'sgd',
'cross_entropy',
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def test_correct_optim_calls_1_batch_per_step(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
mocked_optimizer = some_mocked_optimizer()
mocked_optim_model = Model(self.pytorch_network,
mocked_optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
mocked_optim_model.fit_generator(train_generator,
None,
epochs=1,
steps_per_epoch=1,
batches_per_step=1)
self.assertEqual(1, mocked_optimizer.step.call_count)
self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_epoch_metrics_integration(self):
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
epoch_metrics=[SomeEpochMetric()])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = model.fit_generator(train_generator,
valid_generator,
epochs=1,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch)
actual_value = logs[-1]['some_epoch_metric']
val_actual_value = logs[-1]['val_some_epoch_metric']
expected_value = 5
self.assertEqual(val_actual_value, expected_value)
self.assertEqual(actual_value, expected_value)
def test_epoch_metrics_with_multiple_names_returned_by_dict(self):
d = dict(zip(self.metric_names, self.metric_values))
epoch_metric = ConstEpochMetric(d)
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
epoch_metrics=[(self.metric_names, epoch_metric)])
self._test_history(model, d.keys(), d.values())
def test_repeated_batch_epoch_metrics_handling(self):
expected_names = ['some_metric_name1', 'some_metric_name2']
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=[get_batch_metric(1)],
epoch_metrics=[SomeMetricName(2)])
self._test_history(model, expected_names, [1, 2])
def test_metrics_integration(self):
num_steps = 10
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=[F.mse_loss])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, mse = model.evaluate_generator(generator, steps=num_steps)
self.assertEqual(type(loss), float)
self.assertEqual(type(mse), float)
def setUp(self) -> None:
super().setUp()
self.notification_callback_mock = MagicMock()
self.notificator_mock = MagicMock()
self.train_generator = some_data_tensor_generator(NotificationCallbackTest.batch_size)
self.valid_generator = some_data_tensor_generator(NotificationCallbackTest.batch_size)
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=NotificationCallbackTest.lr)
self.batch_metrics = [
some_batch_metric_1,
('custom_name', some_batch_metric_2),
repeat_batch_metric,
repeat_batch_metric,
]
self.batch_metrics_names = [
'some_batch_metric_1',
'custom_name',
'repeat_batch_metric1',
'repeat_batch_metric2',
]
self.batch_metrics_values = [
some_metric_1_value,
some_metric_2_value,
repeat_batch_metric_value,
repeat_batch_metric_value,
]
self.epoch_metrics = [SomeConstantEpochMetric()]
self.epoch_metrics_names = ['some_constant_epoch_metric']
self.epoch_metrics_values = [some_constant_epoch_metric_value]
self.model = Model(
self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics,
)
class BaseTensorBoardLoggerTest:
SummaryWriter = None
batch_size = 20
lr = 1e-3
num_epochs = 10
def setUp(self):
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=BaseTensorBoardLoggerTest.lr)
self.model = Model(self.pytorch_network, self.optimizer, self.loss_function)
self.temp_dir_obj = TemporaryDirectory()
# pylint: disable=not-callable
self.writer = self.SummaryWriter(self.temp_dir_obj.name)
self.writer.add_scalars = MagicMock()
def tearDown(self):
self.temp_dir_obj.cleanup()
def test_logging(self):
train_gen = some_data_generator(20)
valid_gen = some_data_generator(20)
logger = TensorBoardLogger(self.writer)
history = self.model.fit_generator(
train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger]
)
self._test_logging(history)
def test_multiple_learning_rates(self):
train_gen = some_data_generator(20)
valid_gen = some_data_generator(20)
logger = TensorBoardLogger(self.writer)
lrs = [BaseCSVLoggerTest.lr, BaseCSVLoggerTest.lr / 2]
optimizer = torch.optim.SGD(
[dict(params=[self.pytorch_network.weight], lr=lrs[0]), dict(params=[self.pytorch_network.bias], lr=lrs[1])]
)
model = Model(self.pytorch_network, optimizer, self.loss_function)
history = model.fit_generator(
train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger]
)
self._test_logging(history, lrs=lrs)
def _test_logging(self, history, lrs=None):
if lrs is None:
lrs = [BaseCSVLoggerTest.lr]
calls = []
for h in history:
calls.append(call('loss', {'loss': h['loss'], 'val_loss': h['val_loss']}, h['epoch']))
if len(lrs) == 1:
calls.append(call('lr', {'lr': self.lr}, h['epoch']))
else:
calls.append(call('lr', {f'lr_group_{i}': lr for i, lr in enumerate(lrs)}, h['epoch']))
self.writer.add_scalars.assert_has_calls(calls, any_order=True)
