def test_with_only_on_epoch_end_arg(self):
on_epoch_end = Mock()
lambda_callback = LambdaCallback(on_epoch_end=on_epoch_end)
train_generator = some_data_tensor_generator(LambdaTest.batch_size)
valid_generator = some_data_tensor_generator(LambdaTest.batch_size)
test_generator = some_data_tensor_generator(LambdaTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=LambdaTest.epochs,
steps_per_epoch=LambdaTest.steps_per_epoch,
validation_steps=LambdaTest.steps_per_epoch,
callbacks=[lambda_callback],
)
num_steps = 10
self.model.evaluate_generator(test_generator,
steps=num_steps,
callbacks=[lambda_callback])
expected_calls = [
call(epoch_number, log)
for epoch_number, log in enumerate(logs, 1)
]
actual_calls = on_epoch_end.mock_calls
self.assertEqual(len(expected_calls), len(actual_calls))
self.assertEqual(expected_calls, actual_calls)
def setUp(self):
super().setUp()
self.train_generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
self.valid_generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
self.test_generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3)
self.batch_metrics = [
some_batch_metric_1, ('custom_name', some_batch_metric_2), repeat_batch_metric, repeat_batch_metric
]
self.batch_metrics_names = [
'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1', 'repeat_batch_metric2'
]
self.batch_metrics_values = [
some_metric_1_value, some_metric_2_value, repeat_batch_metric_value, repeat_batch_metric_value
]
self.epoch_metrics = [SomeConstantEpochMetric()]
self.epoch_metrics_names = ['some_constant_epoch_metric']
self.epoch_metrics_values = [some_constant_epoch_metric_value]
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
self._capture_output()
def test_cpu_cuda(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
self._capture_output()
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda()
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
# The context manager is also used here because of this bug:
# https://github.com/pytorch/pytorch/issues/7320
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda(ModelTest.cuda_device)
self._test_device(
torch.device('cuda:' + str(ModelTest.cuda_device)))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.cpu()
self._test_device(torch.device('cpu'))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.to(torch.device('cuda:' + str(ModelTest.cuda_device)))
self._test_device(
torch.device('cuda:' + str(ModelTest.cuda_device)))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.to(torch.device('cpu'))
self._test_device(torch.device('cpu'))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
def test_predict_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator, steps=num_steps)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1))
def test_epoch_metrics_integration(self):
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
epoch_metrics=[SomeEpochMetric()])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = model.fit_generator(train_generator,
valid_generator,
epochs=1,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch)
actual_value = logs[-1]['some_epoch_metric']
val_actual_value = logs[-1]['val_some_epoch_metric']
expected_value = 5
self.assertEqual(val_actual_value, expected_value)
self.assertEqual(actual_value, expected_value)
def test_evaluate_generator_with_return_dict(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.evaluate_generator(generator,
steps=num_steps,
return_dict_format=True)
self._test_return_dict_logs(logs)
def test_fitting_tensor_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch,
'valid_steps': ModelTest.steps_per_epoch
}
self._test_callbacks_train(params,
logs,
valid_steps=ModelTest.steps_per_epoch)
def test_metrics_integration(self):
num_steps = 10
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=[F.mse_loss])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, mse = model.evaluate_generator(generator, steps=num_steps)
self.assertEqual(type(loss), float)
self.assertEqual(type(mse), float)
def test_evaluate_generator_with_callback(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
self.model.evaluate_generator(generator,
steps=num_steps,
callbacks=[self.mock_callback])
params = {'steps': ModelTest.epochs}
self._test_callbacks_test(params)
def test_lambda_train_calls(self):
lambda_callback, mock_calls = self._get_lambda_callback_with_mock_args(
)
train_generator = some_data_tensor_generator(LambdaTest.batch_size)
valid_generator = some_data_tensor_generator(LambdaTest.batch_size)
self.model.fit_generator(
train_generator,
valid_generator,
epochs=LambdaTest.epochs,
steps_per_epoch=LambdaTest.steps_per_epoch,
validation_steps=LambdaTest.steps_per_epoch,
callbacks=[lambda_callback, self.mock_callback],
)
expected_calls = self.mock_callback.method_calls[2:]
actual_calls = mock_calls.method_calls
self.assertEqual(len(expected_calls), len(actual_calls))
self.assertEqual(expected_calls, actual_calls)
def test_evaluate_generator_with_callback_and_progress_bar_coloring(self):
generator = some_data_tensor_generator(ModelFittingTestCase.batch_size)
_, _ = self.model.evaluate_generator(generator,
steps=ModelFittingTestCaseProgress.num_steps,
callbacks=[self.mock_callback],
verbose=True)
self.assertStdoutContains(["%", "[32m", "[35m", "[36m", "[94m", "\u2588"])
def test_integration_zero_args(self):
lambda_callback = LambdaCallback()
train_generator = some_data_tensor_generator(LambdaTest.batch_size)
valid_generator = some_data_tensor_generator(LambdaTest.batch_size)
test_generator = some_data_tensor_generator(LambdaTest.batch_size)
self.model.fit_generator(
train_generator,
valid_generator,
epochs=LambdaTest.epochs,
steps_per_epoch=LambdaTest.steps_per_epoch,
validation_steps=LambdaTest.steps_per_epoch,
callbacks=[lambda_callback],
)
num_steps = 10
self.model.evaluate_generator(test_generator,
steps=num_steps,
callbacks=[lambda_callback])
def setUp(self) -> None:
super().setUp()
self.notification_callback_mock = MagicMock()
self.notificator_mock = MagicMock()
self.train_generator = some_data_tensor_generator(NotificationCallbackTest.batch_size)
self.valid_generator = some_data_tensor_generator(NotificationCallbackTest.batch_size)
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=NotificationCallbackTest.lr)
self.batch_metrics = [
some_batch_metric_1,
('custom_name', some_batch_metric_2),
repeat_batch_metric,
repeat_batch_metric,
]
self.batch_metrics_names = [
'some_batch_metric_1',
'custom_name',
'repeat_batch_metric1',
'repeat_batch_metric2',
]
self.batch_metrics_values = [
some_metric_1_value,
some_metric_2_value,
repeat_batch_metric_value,
repeat_batch_metric_value,
]
self.epoch_metrics = [SomeConstantEpochMetric()]
self.epoch_metrics_names = ['some_constant_epoch_metric']
self.epoch_metrics_values = [some_constant_epoch_metric_value]
self.model = Model(
self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics,
)
def test_predict_generator_with_no_concatenation(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator,
steps=num_steps,
concatenate_returns=False)
self.assertEqual(type(pred_y), list)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
def test_evaluate_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
steps=num_steps,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1))
def test_fitting_without_valid_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
None,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_callbacks_train(params, logs, has_valid=False)
def test_lambda_test_calls(self):
lambda_callback, mock_calls = self._get_lambda_callback_with_mock_args(
)
num_steps = 10
generator = some_data_tensor_generator(LambdaTest.batch_size)
self.model.evaluate_generator(
generator,
steps=num_steps,
callbacks=[lambda_callback, self.mock_callback])
expected_calls = self.mock_callback.method_calls[2:]
actual_calls = mock_calls.method_calls
self.assertEqual(len(expected_calls), len(actual_calls))
self.assertEqual(expected_calls, actual_calls)
def test_correct_optim_calls_1_batch_per_step(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
mocked_optimizer = some_mocked_optimizer()
mocked_optim_model = Model(self.pytorch_network,
mocked_optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
mocked_optim_model.fit_generator(train_generator,
None,
epochs=1,
steps_per_epoch=1,
batches_per_step=1)
self.assertEqual(1, mocked_optimizer.step.call_count)
self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_evaluate_generator_with_no_concatenation(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y, true_y = self.model.evaluate_generator(
generator,
steps=num_steps,
return_pred=True,
return_ground_truth=True,
concatenate_returns=False)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(type(pred_y), list)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(type(true_y), list)
for true in true_y:
self.assertEqual(type(true), np.ndarray)
self.assertEqual(true.shape, (ModelTest.batch_size, 1))
