class ModelTest(ModelFittingTestCase):
# pylint: disable=too-many-public-methods
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.batch_metrics = [
some_batch_metric_1, ('custom_name', some_batch_metric_2),
repeat_batch_metric, repeat_batch_metric
]
self.batch_metrics_names = [
'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1',
'repeat_batch_metric2'
]
self.batch_metrics_values = [
some_metric_1_value, some_metric_2_value,
repeat_batch_metric_value, repeat_batch_metric_value
]
self.epoch_metrics = [SomeConstantEpochMetric()]
self.epoch_metrics_names = ['some_constant_epoch_metric']
self.epoch_metrics_values = [some_constant_epoch_metric_value]
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def test_fitting_tensor_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_without_valid_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
None,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_callbacks_train(params, logs, has_valid=False)
def test_correct_optim_calls_1_batch_per_step(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
mocked_optimizer = some_mocked_optimizer()
mocked_optim_model = Model(self.pytorch_network,
mocked_optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
mocked_optim_model.fit_generator(train_generator,
None,
epochs=1,
steps_per_epoch=1,
batches_per_step=1)
self.assertEqual(1, mocked_optimizer.step.call_count)
self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_correct_optim_calls__valid_n_batches_per_step(self):
n_batches = 5
items_per_batch = int(ModelTest.batch_size / n_batches)
x = torch.rand(n_batches, items_per_batch, 1)
y = torch.rand(n_batches, items_per_batch, 1)
mocked_optimizer = some_mocked_optimizer()
mocked_optim_model = Model(self.pytorch_network,
mocked_optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
mocked_optim_model.fit_generator(list(zip(x, y)),
None,
epochs=1,
batches_per_step=n_batches)
self.assertEqual(1, mocked_optimizer.step.call_count)
self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_fitting_generator_n_batches_per_step(self):
total_batch_size = 6
x = torch.rand(1, total_batch_size, 1)
y = torch.rand(1, total_batch_size, 1)
initial_params = self.model.get_weight_copies()
self.model.fit_generator(list(zip(x, y)),
None,
epochs=1,
batches_per_step=1)
expected_params = list(self.model.get_weight_copies().values())
for mini_batch_size in [1, 2, 5]:
self.model.set_weights(initial_params)
n_batches_per_step = int(total_batch_size / mini_batch_size)
x.resize_((n_batches_per_step, mini_batch_size, 1))
y.resize_((n_batches_per_step, mini_batch_size, 1))
self.model.fit_generator(list(zip(x, y)),
None,
epochs=1,
batches_per_step=n_batches_per_step)
returned_params = list(self.model.get_weight_copies().values())
np.testing.assert_almost_equal(returned_params,
expected_params,
decimal=4)
def test_fitting_generator_n_batches_per_step_higher_than_num_batches(
self):
total_batch_size = 6
x = torch.rand(1, total_batch_size, 1)
y = torch.rand(1, total_batch_size, 1)
initial_params = self.model.get_weight_copies()
self.model.fit_generator(list(zip(x, y)),
None,
epochs=1,
batches_per_step=1)
expected_params = list(self.model.get_weight_copies().values())
self.model.set_weights(initial_params)
self.model.fit_generator(list(zip(x, y)),
None,
epochs=1,
batches_per_step=2)
returned_params = list(self.model.get_weight_copies().values())
np.testing.assert_almost_equal(returned_params,
expected_params,
decimal=4)
def test_fitting_generator_n_batches_per_step_uneven_batches(self):
total_batch_size = 6
x = torch.rand(1, total_batch_size, 1)
y = torch.rand(1, total_batch_size, 1)
initial_params = self.model.get_weight_copies()
self.model.fit_generator(list(zip(x, y)),
None,
epochs=1,
batches_per_step=1)
expected_params = list(self.model.get_weight_copies().values())
x.squeeze_(dim=0)
y.squeeze_(dim=0)
uneven_chunk_sizes = [4, 5]
for chunk_size in uneven_chunk_sizes:
self.model.set_weights(initial_params)
splitted_x = x.split(chunk_size)
splitted_y = y.split(chunk_size)
n_batches_per_step = ceil(total_batch_size / chunk_size)
self.model.fit_generator(list(zip(splitted_x, splitted_y)),
None,
epochs=1,
batches_per_step=n_batches_per_step)
returned_params = list(self.model.get_weight_copies().values())
np.testing.assert_almost_equal(returned_params,
expected_params,
decimal=4)
def test_fitting_ndarray_generator(self):
train_generator = some_ndarray_generator(ModelTest.batch_size)
valid_generator = some_ndarray_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_with_data_loader(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = torch.rand(train_size, 1)
train_y = torch.rand(train_size, 1)
train_dataset = TensorDataset(train_x, train_y)
train_generator = DataLoader(train_dataset, train_batch_size)
valid_real_steps_per_epoch = 10
valid_batch_size = 15
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = torch.rand(valid_size, 1)
valid_y = torch.rand(valid_size, 1)
valid_dataset = TensorDataset(valid_x, valid_y)
valid_generator = DataLoader(valid_dataset, valid_batch_size)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_generator_calls(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = torch.rand(train_size, 1)
train_y = torch.rand(train_size, 1)
train_dataset = TensorDataset(train_x, train_y)
train_generator = DataLoader(train_dataset, train_batch_size)
valid_real_steps_per_epoch = 10
valid_batch_size = 15
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = torch.rand(valid_size, 1)
valid_y = torch.rand(valid_size, 1)
valid_dataset = TensorDataset(valid_x, valid_y)
valid_generator = DataLoader(valid_dataset, valid_batch_size)
class IterableMock:
def __init__(self, iterable):
self.iterable = iterable
self.iter = None
self.calls = []
def __iter__(self):
self.calls.append('__iter__')
self.iter = iter(self.iterable)
return self
def __next__(self):
self.calls.append('__next__')
return next(self.iter)
def __len__(self):
self.calls.append('__len__')
return len(self.iterable)
mock_train_generator = IterableMock(train_generator)
mock_valid_generator = IterableMock(valid_generator)
self.model.fit_generator(mock_train_generator,
mock_valid_generator,
epochs=ModelTest.epochs)
expected_train_calls = ['__len__'] + \
(['__iter__'] + ['__next__'] * train_real_steps_per_epoch) * ModelTest.epochs
expected_valid_calls = ['__len__'] + \
(['__iter__'] + ['__next__'] * valid_real_steps_per_epoch) * ModelTest.epochs
self.assertEqual(mock_train_generator.calls, expected_train_calls)
self.assertEqual(mock_valid_generator.calls, expected_valid_calls)
def test_fitting_with_tensor(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = torch.rand(train_size, 1)
train_y = torch.rand(train_size, 1)
valid_real_steps_per_epoch = 10
# valid_batch_size will be the same as train_batch_size in the fit method.
valid_batch_size = train_batch_size
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = torch.rand(valid_size, 1)
valid_y = torch.rand(valid_size, 1)
logs = self.model.fit(train_x,
train_y,
validation_data=(valid_x, valid_y),
epochs=ModelTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_with_np_array(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = np.random.rand(train_size, 1).astype(np.float32)
train_y = np.random.rand(train_size, 1).astype(np.float32)
valid_real_steps_per_epoch = 10
# valid_batch_size will be the same as train_batch_size in the fit method.
valid_batch_size = train_batch_size
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = np.random.rand(valid_size, 1).astype(np.float32)
valid_y = np.random.rand(valid_size, 1).astype(np.float32)
logs = self.model.fit(train_x,
train_y,
validation_data=(valid_x, valid_y),
epochs=ModelTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_with_generator_with_len(self):
train_real_steps_per_epoch = 30
train_generator = SomeDataGeneratorWithLen(
batch_size=ModelTest.batch_size,
length=train_real_steps_per_epoch,
num_missing_samples=7)
valid_generator = SomeDataGeneratorWithLen(batch_size=15,
length=10,
num_missing_samples=3)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_with_generator_with_stop_iteration(self):
train_real_steps_per_epoch = 30
train_generator = SomeDataGeneratorUsingStopIteration(
batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch)
valid_generator = SomeDataGeneratorUsingStopIteration(batch_size=15,
length=10)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': None}
self._test_callbacks_train(params,
logs,
steps=train_real_steps_per_epoch)
def test_tensor_train_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics = self.model.train_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), self.batch_metrics_values)
def test_train_on_batch_with_pred(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics, pred_y = self.model.train_on_batch(x,
y,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), self.batch_metrics_values)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_train_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
loss, metrics = self.model.train_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), self.batch_metrics_values)
def test_evaluate(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, metrics = self.model.evaluate(x,
y,
batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
def test_evaluate_with_pred(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
# We also test the unpacking.
_, _, pred_y = self.model.evaluate(x,
y,
batch_size=ModelTest.batch_size,
return_pred=True)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_with_callback(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
# We also test the unpacking.
_, _, pred_y = self.model.evaluate(x,
y,
batch_size=ModelTest.batch_size,
return_pred=True,
callbacks=[self.mock_callback])
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
y = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
loss, metrics, pred_y = self.model.evaluate(
x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
dataset = TensorDataset(x, y)
generator = DataLoader(dataset, ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
steps=num_steps,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1))
def test_evaluate_generator_with_callback(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
result_log = self.model.evaluate_generator(
generator,
steps=num_steps,
return_pred=True,
callbacks=[self.mock_callback])
params = {'batch': ModelTest.epochs}
self._test_callbacks_test(params, result_log)
def test_evaluate_generator_with_ground_truth(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y, true_y = self.model.evaluate_generator(
generator,
steps=num_steps,
return_pred=True,
return_ground_truth=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(type(true_y), np.ndarray)
self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1))
self.assertEqual(true_y.shape, (num_steps * ModelTest.batch_size, 1))
def test_evaluate_generator_with_no_concatenation(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y, true_y = self.model.evaluate_generator(
generator,
steps=num_steps,
return_pred=True,
return_ground_truth=True,
concatenate_returns=False)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
self.batch_metrics_values + self.epoch_metrics_values)
self.assertEqual(type(pred_y), list)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(type(true_y), list)
for true in true_y:
self.assertEqual(type(true), np.ndarray)
self.assertEqual(true.shape, (ModelTest.batch_size, 1))
def test_evaluate_with_only_one_metric(self):
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics[:1])
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, first_metric = model.evaluate(x,
y,
batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(first_metric), float)
self.assertEqual(first_metric, some_metric_1_value)
def test_metrics_integration(self):
num_steps = 10
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=[F.mse_loss])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, mse = model.evaluate_generator(generator, steps=num_steps)
self.assertEqual(type(loss), float)
self.assertEqual(type(mse), float)
def test_epoch_metrics_integration(self):
model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
epoch_metrics=[SomeEpochMetric()])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = model.fit_generator(train_generator,
valid_generator,
epochs=1,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch)
actual_value = logs[-1]['some_epoch_metric']
val_actual_value = logs[-1]['val_some_epoch_metric']
expected_value = 5
self.assertEqual(val_actual_value, expected_value)
self.assertEqual(actual_value, expected_value)
def test_evaluate_with_no_metric(self):
model = Model(self.pytorch_network, self.optimizer, self.loss_function)
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss = model.evaluate(x, y, batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
def test_tensor_evaluate_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics = self.model.evaluate_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), self.batch_metrics_values)
def test_evaluate_on_batch_with_pred(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics, pred_y = self.model.evaluate_on_batch(x,
y,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), self.batch_metrics_values)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_evaluate_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
loss, metrics = self.model.evaluate_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), self.batch_metrics_values)
def test_predict(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
generator = DataLoader(x, ModelTest.batch_size)
pred_y = self.model.predict_generator(generator)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator, steps=num_steps)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1))
def test_predict_generator_with_no_concatenation(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator,
steps=num_steps,
concatenate_returns=False)
self.assertEqual(type(pred_y), list)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
def test_tensor_predict_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
pred_y = self.model.predict_on_batch(x)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_predict_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
pred_y = self.model.predict_on_batch(x)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
@skipIf(not torch.cuda.is_available(), "no gpu available")
def test_cpu_cuda(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda()
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
# The context manager is also used here because of this bug:
# https://github.com/pytorch/pytorch/issues/7320
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda(ModelTest.cuda_device)
self._test_device(
torch.device('cuda:' + str(ModelTest.cuda_device)))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.cpu()
self._test_device(torch.device('cpu'))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.to(torch.device('cuda:' + str(ModelTest.cuda_device)))
self._test_device(
torch.device('cuda:' + str(ModelTest.cuda_device)))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.to(torch.device('cpu'))
self._test_device(torch.device('cpu'))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
def _test_device(self, device):
for p in self.pytorch_network.parameters():
self.assertEqual(p.device, device)
def test_get_batch_size(self):
batch_size = ModelTest.batch_size
x = np.random.rand(batch_size, 1).astype(np.float32)
y = np.random.rand(batch_size, 1).astype(np.float32)
batch_size2 = ModelTest.batch_size + 1
x2 = np.random.rand(batch_size2, 1).astype(np.float32)
y2 = np.random.rand(batch_size2, 1).astype(np.float32)
other_batch_size = batch_size2 + 1
inf_batch_size = self.model.get_batch_size(x, y)
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size(x2, y2)
self.assertEqual(inf_batch_size, batch_size2)
inf_batch_size = self.model.get_batch_size(x, y2)
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size(x2, y)
self.assertEqual(inf_batch_size, batch_size2)
inf_batch_size = self.model.get_batch_size((x, x2), y)
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size((x2, x), y)
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size((x, x2), (y, y2))
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size((x2, x), (y, y2))
self.assertEqual(inf_batch_size, batch_size2)
inf_batch_size = self.model.get_batch_size([x, x2], y)
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size([x2, x], y)
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size([x, x2], [y, y2])
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size([x2, x], [y, y2])
self.assertEqual(inf_batch_size, batch_size2)
inf_batch_size = self.model.get_batch_size(
{
'batch_size': other_batch_size,
'x': x
}, {'y': y})
self.assertEqual(inf_batch_size, other_batch_size)
inf_batch_size = self.model.get_batch_size({'x': x}, {
'batch_size': other_batch_size,
'y': y
})
self.assertEqual(inf_batch_size, other_batch_size)
inf_batch_size = self.model.get_batch_size({'x': x}, {'y': y})
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size(
OrderedDict([('x1', x), ('x2', x2)]), {'y': y})
self.assertEqual(inf_batch_size, batch_size)
inf_batch_size = self.model.get_batch_size(
OrderedDict([('x1', x2), ('x2', x)]), {'y': y})
self.assertEqual(inf_batch_size, batch_size2)
inf_batch_size = self.model.get_batch_size([1, 2, 3], {'y': y})
self.assertEqual(inf_batch_size, batch_size)
def test_get_batch_size_warning(self):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
inf_batch_size = self.model.get_batch_size([1, 2, 3], [4, 5, 6])
self.assertEqual(inf_batch_size, 1)
self.assertEqual(len(w), 1)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
warning_settings['batch_size'] = 'ignore'
inf_batch_size = self.model.get_batch_size([1, 2, 3], [4, 5, 6])
self.assertEqual(inf_batch_size, 1)
self.assertEqual(len(w), 0)
class ModelTest(TestCase):
# pylint: disable=too-many-public-methods
epochs = 10
steps_per_epoch = 5
batch_size = 20
evaluate_dataset_len = 107
cuda_device = int(os.environ.get('CUDA_DEVICE', 0))
def setUp(self):
torch.manual_seed(42)
self.pytorch_module = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(),
lr=1e-3)
self.metrics = [some_metric_1, some_metric_2]
self.metrics_names = ['some_metric_1', 'some_metric_2']
self.metrics_values = [some_metric_1_value, some_metric_2_value]
self.model = Model(self.pytorch_module,
self.optimizer,
self.loss_function,
metrics=self.metrics)
self.mock_callback = MagicMock()
def test_fitting_tensor_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_without_valid_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
None,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_fitting(params, logs, has_valid=False)
def test_fitting_ndarray_generator(self):
train_generator = some_ndarray_generator(ModelTest.batch_size)
valid_generator = some_ndarray_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_data_loader(self):
# pylint: disable=too-many-locals
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = torch.rand(train_size, 1)
train_y = torch.rand(train_size, 1)
train_dataset = TensorDataset(train_x, train_y)
train_generator = DataLoader(train_dataset, train_batch_size)
valid_real_steps_per_epoch = 10
valid_batch_size = 15
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = torch.rand(valid_size, 1)
valid_y = torch.rand(valid_size, 1)
valid_dataset = TensorDataset(valid_x, valid_y)
valid_generator = DataLoader(valid_dataset, valid_batch_size)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_tensor(self):
# pylint: disable=too-many-locals
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = torch.rand(train_size, 1)
train_y = torch.rand(train_size, 1)
valid_real_steps_per_epoch = 10
# valid_batch_size will be the same as train_batch_size in the fit method.
valid_batch_size = train_batch_size
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = torch.rand(valid_size, 1)
valid_y = torch.rand(valid_size, 1)
logs = self.model.fit(train_x,
train_y,
validation_x=valid_x,
validation_y=valid_y,
epochs=ModelTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_np_array(self):
# pylint: disable=too-many-locals
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = np.random.rand(train_size, 1).astype(np.float32)
train_y = np.random.rand(train_size, 1).astype(np.float32)
valid_real_steps_per_epoch = 10
# valid_batch_size will be the same as train_batch_size in the fit method.
valid_batch_size = train_batch_size
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = np.random.rand(valid_size, 1).astype(np.float32)
valid_y = np.random.rand(valid_size, 1).astype(np.float32)
logs = self.model.fit(train_x,
train_y,
validation_x=valid_x,
validation_y=valid_y,
epochs=ModelTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_generator_with_len(self):
train_real_steps_per_epoch = 30
train_generator = SomeDataGeneratorWithLen(
batch_size=ModelTest.batch_size,
length=train_real_steps_per_epoch,
num_missing_samples=7)
valid_generator = SomeDataGeneratorWithLen(batch_size=15,
length=10,
num_missing_samples=3)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_generator_with_stop_iteration(self):
train_real_steps_per_epoch = 30
train_generator = SomeDataGeneratorUsingStopIteration(
batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch)
valid_generator = SomeDataGeneratorUsingStopIteration(batch_size=15,
length=10)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': None}
self._test_fitting(params, logs, steps=train_real_steps_per_epoch)
def _test_fitting(self, params, logs, has_valid=True, steps=None):
if steps is None:
steps = params['steps']
self.assertEqual(len(logs), params['epochs'])
train_dict = dict(zip(self.metrics_names, self.metrics_values),
loss=ANY,
time=ANY)
if has_valid:
val_metrics_names = [
'val_' + metric_name for metric_name in self.metrics_names
]
val_dict = dict(zip(val_metrics_names, self.metrics_values),
val_loss=ANY)
log_dict = {**train_dict, **val_dict}
else:
log_dict = train_dict
for epoch, log in enumerate(logs, 1):
self.assertEqual(log, dict(log_dict, epoch=epoch))
call_list = []
call_list.append(call.on_train_begin({}))
for epoch in range(1, params['epochs'] + 1):
call_list.append(call.on_epoch_begin(epoch, {}))
for step in range(1, steps + 1):
call_list.append(call.on_batch_begin(step, {}))
call_list.append(call.on_backward_end(step))
call_list.append(
call.on_batch_end(step, {
'batch': step,
'size': ANY,
**train_dict
}))
call_list.append(
call.on_epoch_end(epoch, {
'epoch': epoch,
**log_dict
}))
call_list.append(call.on_train_end({}))
method_calls = self.mock_callback.method_calls
self.assertIn(call.set_model(self.model), method_calls[:2])
self.assertIn(call.set_params(params), method_calls[:2])
self.assertEqual(len(method_calls), len(call_list) + 2)
self.assertEqual(method_calls[2:], call_list)
def test_tensor_train_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics = self.model.train_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
def test_train_on_batch_with_pred(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics, pred_y = self.model.train_on_batch(x,
y,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_train_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
loss, metrics = self.model.train_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
def test_evaluate(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, metrics = self.model.evaluate(x,
y,
batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
def test_evaluate_with_pred(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
# We also test the unpacking.
# pylint: disable=unused-variable
loss, metrics, pred_y = self.model.evaluate(
x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
y = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
loss, metrics, pred_y = self.model.evaluate(
x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
dataset = TensorDataset(x, y)
generator = DataLoader(dataset, ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
self._test_predictions_for_evaluate_and_predict_generator(pred_y)
def test_evaluate_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
steps=num_steps,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(
np.concatenate(pred_y).shape,
(num_steps * ModelTest.batch_size, 1))
def test_evaluate_with_only_one_metric(self):
self.model = Model(self.pytorch_module,
self.optimizer,
self.loss_function,
metrics=self.metrics[:1])
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, first_metric = self.model.evaluate(
x, y, batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(first_metric), float)
self.assertEqual(first_metric, some_metric_1_value)
def test_metrics_integration(self):
num_steps = 10
self.model = Model(self.pytorch_module,
self.optimizer,
self.loss_function,
metrics=[F.mse_loss])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, mse = self.model.evaluate_generator(generator, steps=num_steps)
self.assertEqual(type(loss), float)
self.assertEqual(type(mse), float)
def test_evaluate_with_no_metric(self):
self.model = Model(self.pytorch_module, self.optimizer,
self.loss_function)
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss = self.model.evaluate(x, y, batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
def test_tensor_evaluate_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics = self.model.evaluate_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
def test_evaluate_on_batch_with_pred(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics, pred_y = self.model.evaluate_on_batch(x,
y,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_evaluate_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
loss, metrics = self.model.evaluate_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
def test_predict(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
generator = DataLoader(x, ModelTest.batch_size)
pred_y = self.model.predict_generator(generator)
self._test_predictions_for_evaluate_and_predict_generator(pred_y)
def test_predict_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator, steps=num_steps)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(
np.concatenate(pred_y).shape,
(num_steps * ModelTest.batch_size, 1))
def _test_predictions_for_evaluate_and_predict_generator(self, pred_y):
self.assertEqual(type(pred_y), list)
remaning_example = ModelTest.evaluate_dataset_len
cur_batch_size = ModelTest.batch_size
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
if remaning_example < ModelTest.batch_size:
cur_batch_size = remaning_example
remaning_example = 0
else:
remaning_example -= ModelTest.batch_size
self.assertEqual(pred.shape, (cur_batch_size, 1))
self.assertEqual(
np.concatenate(pred_y).shape, (ModelTest.evaluate_dataset_len, 1))
def test_tensor_predict_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
pred_y = self.model.predict_on_batch(x)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_predict_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
pred_y = self.model.predict_on_batch(x)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
@skipIf(not torch.cuda.is_available(), "no gpu available")
def test_cpu_cuda(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda()
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
# The context manager is also used here because of this bug:
# https://github.com/pytorch/pytorch/issues/7320
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda(ModelTest.cuda_device)
self._test_device(
torch.device('cuda:' + str(ModelTest.cuda_device)))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.cpu()
self._test_device(torch.device('cpu'))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.to(torch.device('cuda:' + str(ModelTest.cuda_device)))
self._test_device(
torch.device('cuda:' + str(ModelTest.cuda_device)))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
self.model.to(torch.device('cpu'))
self._test_device(torch.device('cpu'))
self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
def _test_device(self, device):
for p in self.pytorch_module.parameters():
self.assertEqual(p.device, device)
def test_disable_batch_size_warning(self):
import warnings
def tuple_generator(batch_size):
while True:
x1 = torch.rand(batch_size, 1)
x2 = torch.rand(batch_size, 1)
y1 = torch.rand(batch_size, 1)
y2 = torch.rand(batch_size, 1)
yield (x1, x2), (y1, y2)
class TupleModule(nn.Module):
def __init__(self):
super().__init__()
self.l1 = nn.Linear(1, 1)
self.l2 = nn.Linear(1, 1)
def forward(self, x):
# pylint: disable=arguments-differ
x1, x2 = x
return self.l1(x1), self.l2(x2)
def loss_function(y_pred, y_true):
return F.mse_loss(y_pred[0], y_true[0]) + F.mse_loss(
y_pred[1], y_true[1])
pytorch_module = TupleModule()
optimizer = torch.optim.SGD(pytorch_module.parameters(), lr=1e-3)
model = Model(pytorch_module, optimizer, loss_function)
train_generator = tuple_generator(ModelTest.batch_size)
valid_generator = tuple_generator(ModelTest.batch_size)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch)
num_warnings = ModelTest.steps_per_epoch * 2 * ModelTest.epochs
self.assertEqual(len(w), num_warnings)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
warning_settings['batch_size'] = 'ignore'
model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch)
self.assertEqual(len(w), 0)
def learn(o, X_spectral, X_image, y, objs, wavelengths, data_type='spectral_image', image_preprocess='resnet', epochs=50, batch_size=128, material_count=8, layers=[64]*2, dropout=0.0, lr=0.0005, seed=1000, test='looo', verbose=False):
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
torch.set_num_threads(1)
if 'test' in test:
train_idx, test_idx = o
X_spectral_train = X_spectral[train_idx]
X_image_train = X_image[train_idx]
y_train = y[train_idx]
X_spectral_test = X_spectral[test_idx]
X_image_test = X_image[test_idx]
y_test = y[test_idx]
objs_train = objs[train_idx]
elif 'looo' in test:
_, obj = o
# Set up leave-one-object-out training and test sets
X_spectral_train = X_spectral[objs != obj]
X_image_train = X_image[objs != obj]
y_train = y[objs != obj]
X_spectral_test = X_spectral[objs == obj]
X_image_test = X_image[objs == obj]
y_test = y[objs == obj]
objs_train = objs[objs != obj]
X_spectral_train, X_image_train, y_train, X_spectral_test, X_image_test, y_test = prepare_data(X_spectral_train, X_image_train, y_train, X_spectral_test, X_image_test, y_test, wavelengths, deriv=True, data_type=data_type, image_preprocess=image_preprocess, objs=objs)
if data_type == 'spectral':
X_spectral_train, y_train = shuffle(X_spectral_train, y_train)
elif data_type == 'image':
X_image_train, y_train = shuffle(X_image_train, y_train)
elif data_type == 'spectral_image':
X_spectral_train, X_image_train, y_train = shuffle(X_spectral_train, X_image_train, y_train)
y_train = torch.tensor(y_train, dtype=torch.long)
y_test = torch.tensor(y_test, dtype=torch.long)
# Image layers
if data_type == 'spectral_image':
spectral_layers = [64, 64, 32, 32]
spectral_dropout = 0.25
spectral_epochs = 50
image_layers = [128, 64, 32]
image_dropout = 0.1
image_epochs = 50
class ConcatPretrainedNetwork(torch.nn.Module):
def __init__(self):
# global spectral_accuracy, image_accuracy
super().__init__()
spectral_net = nn(np.shape(X_spectral_train)[-1], spectral_layers, spectral_dropout, material_count)
opt = torch.optim.Adam(spectral_net.parameters(), lr=lr)
spectral_model = Model(spectral_net, opt, 'cross_entropy', batch_metrics=['accuracy'])
spectral_model.fit(X_spectral_train, y_train, epochs=spectral_epochs, batch_size=batch_size, verbose=False)
image_net = nn(np.shape(X_image_train)[-1], image_layers, image_dropout, material_count)
opt = torch.optim.Adam(image_net.parameters(), lr=lr)
image_model = Model(image_net, opt, 'cross_entropy', batch_metrics=['accuracy'])
image_model.fit(X_image_train, y_train, epochs=image_epochs, batch_size=batch_size, verbose=False)
# Disable dropout, remove last layer and freeze network
self.trained_spectral_model = torch.nn.Sequential(*(list(spectral_net.children())[:-1]))
for p in self.trained_spectral_model.parameters():
p.requires_grad = False
self.trained_image_model = torch.nn.Sequential(*(list(image_net.children())[:-1]))
for p in self.trained_image_model.parameters():
p.requires_grad = False
self.concat_net = nn(spectral_layers[-1] + image_layers[-1], layers, dropout, material_count, batchnorm=False)
def forward(self, x_spectral, x_image):
y1 = self.trained_spectral_model(x_spectral)
y2 = self.trained_image_model(x_image)
concat = torch.cat((y1, y2), -1)
return self.concat_net(concat)
net = ConcatPretrainedNetwork()
X_train = [X_spectral_train, X_image_train]
X_test = [X_spectral_test, X_image_test]
elif data_type == 'image':
net = nn(np.shape(X_image_train)[-1], layers, dropout, material_count)
X_train = X_image_train
X_test = X_image_test
elif data_type == 'spectral':
net = nn(np.shape(X_spectral_train)[-1], layers, dropout, material_count)
X_train = X_spectral_train
X_test = X_spectral_test
opt = torch.optim.Adam(net.parameters(), lr=lr)
model = Model(net, opt, 'cross_entropy', batch_metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=batch_size, verbose=False)
cm = confusion_matrix(y_test, model.predict(X_test).argmax(axis=-1), labels=range(material_count))
# Return accuracy and confusion matrices
if 'backprop' in test:
return {'accuracy': model.evaluate(X_test, y_test)[-1], 'cm': cm, 'obj_cm': np.copy(cm[y_test[0]]), 'model': model, 'net': net, 'X_test': X_test, 'y_test': y_test}
else:
if verbose:
print(obj, model.evaluate(X_test, y_test)[-1])
return {'accuracy': model.evaluate(X_test, y_test)[-1], 'cm': cm, 'obj_cm': np.copy(cm[y_test[0]])}
class ModelMultiInputTest(ModelFittingTestCase):
def setUp(self):
super().setUp()
torch.manual_seed(42)
self.pytorch_network = MultiIOModel(num_input=1, num_output=1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(),
lr=1e-3)
self.model = Model(self.pytorch_network,
self.optimizer,
self.loss_function,
batch_metrics=self.batch_metrics,
epoch_metrics=self.epoch_metrics)
def test_fitting_tensor_generator_multi_input(self):
train_generator = some_data_tensor_generator_multi_input(
ModelMultiInputTest.batch_size)
valid_generator = some_data_tensor_generator_multi_input(
ModelMultiInputTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelMultiInputTest.epochs,
steps_per_epoch=ModelMultiInputTest.steps_per_epoch,
validation_steps=ModelMultiInputTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelMultiInputTest.epochs,
'steps': ModelMultiInputTest.steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_fitting_with_tensor_multi_input(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelMultiInputTest.batch_size
train_final_batch_missing_samples = 7
train_size = train_real_steps_per_epoch * train_batch_size - \
train_final_batch_missing_samples
train_x = (torch.rand(train_size, 1), torch.rand(train_size, 1))
train_y = torch.rand(train_size, 1)
valid_real_steps_per_epoch = 10
# valid_batch_size will be the same as train_batch_size in the fit method.
valid_batch_size = train_batch_size
valid_final_batch_missing_samples = 3
valid_size = valid_real_steps_per_epoch * valid_batch_size - \
valid_final_batch_missing_samples
valid_x = (torch.rand(valid_size, 1), torch.rand(valid_size, 1))
valid_y = torch.rand(valid_size, 1)
logs = self.model.fit(train_x,
train_y,
validation_data=(valid_x, valid_y),
epochs=ModelMultiInputTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelMultiInputTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_callbacks_train(params, logs)
def test_tensor_train_on_batch_multi_input(self):
x1 = torch.rand(ModelMultiInputTest.batch_size, 1)
x2 = torch.rand(ModelMultiInputTest.batch_size, 1)
y = torch.rand(ModelMultiInputTest.batch_size, 1)
loss = self.model.train_on_batch((x1, x2), y)
self.assertEqual(type(loss), float)
def test_train_on_batch_with_pred_multi_input(self):
x1 = torch.rand(ModelMultiInputTest.batch_size, 1)
x2 = torch.rand(ModelMultiInputTest.batch_size, 1)
y = torch.rand(ModelMultiInputTest.batch_size, 1)
loss, pred_y = self.model.train_on_batch((x1, x2), y, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(pred_y.shape, (ModelMultiInputTest.batch_size, 1))
def test_ndarray_train_on_batch_multi_input(self):
x1 = np.random.rand(ModelMultiInputTest.batch_size,
1).astype(np.float32)
x2 = np.random.rand(ModelMultiInputTest.batch_size,
1).astype(np.float32)
y = np.random.rand(ModelMultiInputTest.batch_size,
1).astype(np.float32)
loss = self.model.train_on_batch((x1, x2), y)
self.assertEqual(type(loss), float)
def test_evaluate_multi_input(self):
x = (torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1),
torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1))
y = torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1)
loss = self.model.evaluate(x,
y,
batch_size=ModelMultiInputTest.batch_size)
self.assertEqual(type(loss), float)
def test_evaluate_with_pred_multi_input(self):
x = (torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1),
torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1))
y = torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1)
# We also test the unpacking.
_, pred_y = self.model.evaluate(
x, y, batch_size=ModelMultiInputTest.batch_size, return_pred=True)
self.assertEqual(pred_y.shape,
(ModelMultiInputTest.evaluate_dataset_len, 1))
def test_evaluate_with_np_array_multi_input(self):
x1 = np.random.rand(ModelMultiInputTest.evaluate_dataset_len,
1).astype(np.float32)
x2 = np.random.rand(ModelMultiInputTest.evaluate_dataset_len,
1).astype(np.float32)
x = (x1, x2)
y = np.random.rand(ModelMultiInputTest.evaluate_dataset_len,
1).astype(np.float32)
loss, pred_y = self.model.evaluate(
x, y, batch_size=ModelMultiInputTest.batch_size, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(pred_y.shape,
(ModelMultiInputTest.evaluate_dataset_len, 1))
def test_evaluate_data_loader_multi_input(self):
x1 = torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1)
x2 = torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1)
y = torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1)
dataset = TensorDataset((x1, x2), y)
generator = DataLoader(dataset, ModelMultiInputTest.batch_size)
loss, pred_y = self.model.evaluate_generator(generator,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(pred_y.shape,
(ModelMultiInputTest.evaluate_dataset_len, 1))
def test_evaluate_generator_multi_input(self):
num_steps = 10
generator = some_data_tensor_generator_multi_input(
ModelMultiInputTest.batch_size)
loss, pred_y = self.model.evaluate_generator(generator,
steps=num_steps,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(pred_y.shape,
(num_steps * ModelMultiInputTest.batch_size, 1))
def test_tensor_evaluate_on_batch_multi_input(self):
x1 = torch.rand(ModelMultiInputTest.batch_size, 1)
x2 = torch.rand(ModelMultiInputTest.batch_size, 1)
y = torch.rand(ModelMultiInputTest.batch_size, 1)
loss = self.model.evaluate_on_batch((x1, x2), y)
self.assertEqual(type(loss), float)
def test_predict_multi_input(self):
x = (torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1),
torch.rand(ModelMultiInputTest.evaluate_dataset_len, 1))
pred_y = self.model.predict(x,
batch_size=ModelMultiInputTest.batch_size)
self.assertEqual(pred_y.shape,
(ModelMultiInputTest.evaluate_dataset_len, 1))
def test_predict_with_np_array_multi_input(self):
x1 = np.random.rand(ModelMultiInputTest.evaluate_dataset_len,
1).astype(np.float32)
x2 = np.random.rand(ModelMultiInputTest.evaluate_dataset_len,
1).astype(np.float32)
x = (x1, x2)
pred_y = self.model.predict(x,
batch_size=ModelMultiInputTest.batch_size)
self.assertEqual(pred_y.shape,
(ModelMultiInputTest.evaluate_dataset_len, 1))
def test_predict_generator_multi_input(self):
num_steps = 10
generator = some_data_tensor_generator_multi_input(
ModelMultiInputTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator, steps=num_steps)
self.assertEqual(type(pred_y), np.ndarray)
self.assertEqual(pred_y.shape,
(num_steps * ModelMultiInputTest.batch_size, 1))
def test_tensor_predict_on_batch_multi_input(self):
x1 = torch.rand(ModelMultiInputTest.batch_size, 1)
x2 = torch.rand(ModelMultiInputTest.batch_size, 1)
pred_y = self.model.predict_on_batch((x1, x2))
self.assertEqual(pred_y.shape, (ModelMultiInputTest.batch_size, 1))
class ANNClassifier:
_lr = ADAM_LEARNING_RATE
_encoding_dim = ANN_ENCODING_DIM
_dtype = 'float32'
def __init__(self, model_id, num_input_features, num_output_classes,
model_save_path, **aux_params):
self.ann_cls = NeuralNetworkClassifier(input_shape=num_input_features,
encoding_dim=self._encoding_dim,
classes=num_output_classes)
self.model, device = UtilsFactory.prepare_model(self.ann_cls)
self.model = Model(self.model,
Adam(self.model.parameters(), lr=self._lr),
BCELoss(),
batch_metrics=None)
self.model = self.model.to(device)
self.model_id = model_id
path = f"{model_save_path}/{model_id}"
os.makedirs(path, exist_ok=True)
self.model_path = path
self.modelfile_save_path = os.path.join(path, STANDARD_MODEL_NAME)
self.num_output_classes = num_output_classes
self.learning_parameters = {}
for key, value in aux_params.items():
self.learning_parameters[key] = value
def load(self):
self.model.load_weights(self.modelfile_save_path)
def fit(self, X_train, y_train, X_valid, y_valid):
X_trn = X_train.to_numpy().astype(self._dtype)
X_val = X_valid.to_numpy().astype(self._dtype)
y_trn = self._y_cat(y_train,
self.num_output_classes).astype(self._dtype)
y_val = self._y_cat(y_valid,
self.num_output_classes).astype(self._dtype)
self.model.fit(X_trn,
y_trn,
validation_data=(X_val, y_val),
callbacks=self._callbacks(),
**self.learning_parameters)
def save(self):
torch.save(self.model, self.modelfile_save_path)
def _callbacks(self):
return [
EarlyStopping(patience=ANN_PATIENCE),
ModelCheckpoint(filename=self.modelfile_save_path,
save_best_only=True,
restore_best=True)
]
def _y_cat(self, y, num_classes):
return to_categorical(y, num_classes=num_classes)
def predict(self, X, load=False):
if load:
self.load()
return self.model.predict(X.to_numpy().astype(self._dtype))
def explain(self, X_train, y_train, features, classes):
pass
# Metrics
top3 = TopKAccuracy(k=3)
top5 = TopKAccuracy(k=5)
metrics = ["acc", top3, top5]
model = Model(network=net,
optimizer="Adam",
loss_function=nn.CrossEntropyLoss(),
batch_metrics=metrics)
###################################################
checkname = os.path.join(MODEL_DIR + '/' + args.model + '.pth')
model.load_optimizer_state(checkname) #get the checkpoint model to predict
idx = 109000
Singledata = sX[idx - 1:idx]
idy = sy[idx - 1:idx]
s1 = torch.FloatTensor(Singledata).squeeze(0) ####important
if args.model == "vtcnn":
output = model.predict(s1)
print(MODULATIONS[int(np.argmax(output, axis=1))])
print("The true type is {} from vtcnn".format(sy[idx][0]))
##########realize the single sample prediction############
if args.model == "mrresnet":
outmodel = model.load_weights(checkname) ## it seems not work
################################################################################
