def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) self.mock_callback = MagicMock() self.delay_callback = DelayCallback(self.mock_callback) self.train_dict = {'loss': ANY, 'time': ANY} self.log_dict = {'loss': ANY, 'val_loss': ANY, 'time': ANY}
def setUp(self): torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=BaseTensorBoardLoggerTest.lr) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) self.temp_dir_obj = TemporaryDirectory() # pylint: disable=not-callable self.writer = self.SummaryWriter(self.temp_dir_obj.name) self.writer.add_scalars = MagicMock()
class LRSchedulersTest(TestCase): batch_size = 20 epochs = 10 steps_per_epoch = 5 def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) self.train_gen = some_data_generator(20) self.valid_gen = some_data_generator(20) def test_lambda_lr_integration(self): my_lambda = lambda epoch: 0.95**epoch lambda_lr = LambdaLR(lr_lambda=[my_lambda]) self._fit_with_callback_integration(lambda_lr) def test_step_lr_integration(self): step_lr = StepLR(step_size=3) self._fit_with_callback_integration(step_lr) def test_multistep_lr_integration(self): multistep_lr = MultiStepLR(milestones=[2, 5, 7]) self._fit_with_callback_integration(multistep_lr) def test_exponential_lr_integration(self): exponential_lr = ExponentialLR(gamma=0.01) self._fit_with_callback_integration(exponential_lr) def test_cosine_annealing_lr_integration(self): cosine_annealing_lr = CosineAnnealingLR(T_max=8) self._fit_with_callback_integration(cosine_annealing_lr) def test_reduce_lr_on_plateau_integration(self): reduce_lr = ReduceLROnPlateau(monitor='loss', patience=3) self._fit_with_callback_integration(reduce_lr) def _fit_with_callback_integration(self, callback): self.model.fit_generator( self.train_gen, self.valid_gen, epochs=LRSchedulersTest.epochs, steps_per_epoch=LRSchedulersTest.steps_per_epoch, callbacks=[callback]) def test_exception_is_thrown_on_optimizer_argument(self): with self.assertRaises(ValueError): StepLR(self.optimizer, step_size=3)
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = MultiIOModel(num_input=1, num_output=1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def test_evaluate_with_only_one_metric(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics[:1]) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss, first_metric = model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) self.assertEqual(type(first_metric), float) self.assertEqual(first_metric, some_metric_1_value)
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = MultiIOModel(num_input=1, num_output=2) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model( self.pytorch_network, self.optimizer, lambda y_pred, y_true: self.loss_function(y_pred[0], y_true[0]) + self.loss_function(y_pred[1], y_true[1]), batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def test_correct_optim_calls_1_batch_per_step(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) mocked_optimizer = some_mocked_optimizer() mocked_optim_model = Model(self.pytorch_network, mocked_optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics) mocked_optim_model.fit_generator(train_generator, None, epochs=1, steps_per_epoch=1, batches_per_step=1) self.assertEqual(1, mocked_optimizer.step.call_count) self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_epoch_metrics_integration(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[SomeEpochMetric()]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) logs = model.fit_generator(train_generator, valid_generator, epochs=1, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) actual_value = logs[-1]['some_epoch_metric'] val_actual_value = logs[-1]['val_some_epoch_metric'] expected_value = 5 self.assertEqual(val_actual_value, expected_value) self.assertEqual(actual_value, expected_value)
def __init__(self, directory, module, *, device=None, logging=True, optimizer='sgd', loss_function=None, metrics=[], monitor_metric=None, monitor_mode=None, type=None): self.directory = directory self.logging = logging if type is not None and not type.startswith( 'classif') and not type.startswith('reg'): raise ValueError("Invalid type '%s'" % type) loss_function = self._get_loss_function(loss_function, module, type) metrics = self._get_metrics(metrics, module, type) self._set_monitor(monitor_metric, monitor_mode, type) self.model = Model(module, optimizer, loss_function, metrics=metrics) if device is not None: self.model.to(device) join_dir = lambda x: os.path.join(directory, x) self.best_checkpoint_filename = join_dir( Experiment.BEST_CHECKPOINT_FILENAME) self.best_checkpoint_tmp_filename = join_dir( Experiment.BEST_CHECKPOINT_TMP_FILENAME) self.model_checkpoint_filename = join_dir( Experiment.MODEL_CHECKPOINT_FILENAME) self.model_checkpoint_tmp_filename = join_dir( Experiment.MODEL_CHECKPOINT_TMP_FILENAME) self.optimizer_checkpoint_filename = join_dir( Experiment.OPTIMIZER_CHECKPOINT_FILENAME) self.optimizer_checkpoint_tmp_filename = join_dir( Experiment.OPTIMIZER_CHECKPOINT_TMP_FILENAME) self.log_filename = join_dir(Experiment.LOG_FILENAME) self.tensorboard_directory = join_dir(Experiment.TENSORBOARD_DIRECTORY) self.epoch_filename = join_dir(Experiment.EPOCH_FILENAME) self.epoch_tmp_filename = join_dir(Experiment.EPOCH_TMP_FILENAME) self.lr_scheduler_filename = join_dir(Experiment.LR_SCHEDULER_FILENAME) self.lr_scheduler_tmp_filename = join_dir( Experiment.LR_SCHEDULER_TMP_FILENAME) self.test_log_filename = join_dir(Experiment.TEST_LOG_FILENAME)
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = DictOutputModel() self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model( self.pytorch_network, self.optimizer, lambda y_p, y_t: self.loss_function(y_p['out1'], y_t[ 0]) + self.loss_function(y_p['out2'], y_t[1]), batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.metrics = [some_metric_1, some_metric_2] self.metrics_names = ['some_metric_1', 'some_metric_2'] self.metrics_values = [some_metric_1_value, some_metric_2_value] self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=self.metrics) self.mock_callback = MagicMock()
def test_metrics_integration(self): num_steps = 10 model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[F.mse_loss]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) generator = some_data_tensor_generator(ModelTest.batch_size) loss, mse = model.evaluate_generator(generator, steps=num_steps) self.assertEqual(type(loss), float) self.assertEqual(type(mse), float)
def test_repeated_epoch_metrics_handling(self): expected_names = ['some_metric_name1', 'some_metric_name2'] model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[SomeMetricName(1), SomeMetricName(2)]) self._test_history(model, expected_names, [1, 2])
def test_epoch_metrics_with_multiple_names_returned_by_tuple(self): epoch_metric = ConstEpochMetric(tuple(self.metric_values)) model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[(self.metric_names, epoch_metric)]) self._test_history(model, self.metric_names, self.metric_values)
def test_batch_metrics_with_multiple_names_returned_by_list(self): batch_metric = get_const_batch_metric(list(self.metric_values)) model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[(self.metric_names, batch_metric)]) self._test_history(model, self.metric_names, self.metric_values)
def load_model(load_path, restore=False): load_path = Path(load_path) with open(load_path / "kwargs.json", "r") as kwargs_file: kwargs = json.load(kwargs_file) network = RNN(**kwargs) model = Model( network=network, optimizer=torch.optim.Adam(network.parameters()), loss_function=custom_loss, batch_metrics=[acc], ) if restore: model.load_weights(load_path / "model.pkl") return model
def test_epoch_metrics_with_name_with_multiple_names_returned_by_tensor(self): class EpochMetricWithName(ConstEpochMetric): __name__ = self.metric_names epoch_metric = EpochMetricWithName(torch.tensor(self.metric_values)) model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[epoch_metric]) self._test_history(model, self.metric_names, self.metric_values)
def test_batch_metrics_with_multiple_names_returned_by_dict(self): d = dict(zip(self.metric_names, self.metric_values)) batch_metric = get_const_batch_metric(d) model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[(self.metric_names, batch_metric)]) self._test_history(model, d.keys(), d.values())
def test_correct_optim_calls__valid_n_batches_per_step(self): n_batches = 5 items_per_batch = int(ModelTest.batch_size / n_batches) x = torch.rand(n_batches, items_per_batch, 1) y = torch.rand(n_batches, items_per_batch, 1) mocked_optimizer = some_mocked_optimizer() mocked_optim_model = Model(self.pytorch_network, mocked_optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics) mocked_optim_model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=n_batches) self.assertEqual(1, mocked_optimizer.step.call_count) self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_tracking_two_layers_shallow_model(self): self.num_layer = 2 self.pytorch_network = nn.Sequential(nn.Linear(1, 4), nn.Linear(4, 1)) self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=self.lr) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) keep_bias = False train_gen = some_data_generator(20) valid_gen = some_data_generator(20) tracker = TensorBoardGradientTracker(self.writer, keep_bias=keep_bias) self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[tracker]) self._test_tracking(keep_bias)
def __init__(self, model_id, num_input_features, num_output_classes, model_save_path, **aux_params): self.ann_cls = NeuralNetworkClassifier(input_shape=num_input_features, encoding_dim=self._encoding_dim, classes=num_output_classes) self.model, device = UtilsFactory.prepare_model(self.ann_cls) self.model = Model(self.model, Adam(self.model.parameters(), lr=self._lr), BCELoss(), batch_metrics=None) self.model = self.model.to(device) self.model_id = model_id path = f"{model_save_path}/{model_id}" os.makedirs(path, exist_ok=True) self.model_path = path self.modelfile_save_path = os.path.join(path, STANDARD_MODEL_NAME) self.num_output_classes = num_output_classes self.learning_parameters = {} for key, value in aux_params.items(): self.learning_parameters[key] = value
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.batch_metrics = [ some_batch_metric_1, ('custom_name', some_batch_metric_2), repeat_batch_metric, repeat_batch_metric ] self.batch_metrics_names = [ 'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1', 'repeat_batch_metric2' ] self.batch_metrics_values = [ some_metric_1_value, some_metric_2_value, repeat_batch_metric_value, repeat_batch_metric_value ] self.epoch_metrics = [SomeConstantEpochMetric()] self.epoch_metrics_names = ['some_constant_epoch_metric'] self.epoch_metrics_values = [some_constant_epoch_metric_value] self.model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def main(rnn_type, n_layers, dataset, embedding, device, save_path): train_iter, valid_iter, test_iter = dataset_factory(dataset, embedding=embedding) embedding_dim = int(embedding.split(".")[-1][:-1]) save_path = Path(save_path) / f"{rnn_type}_{n_layers}layer_{embedding_dim}" save_path.mkdir(parents=True, exist_ok=True) kwargs = dict( vocab_size=len(TEXT.vocab), embedding_dim=embedding_dim, hidden_dim=256, output_dim=1, n_layers=n_layers, dropout=0.5, pad_idx=TEXT.vocab.stoi[TEXT.pad_token], rnn_type="gru", ) with open(save_path / "kwargs.json", "w") as kwargs_file: json.dump(kwargs, kwargs_file) pretrained_embeddings = TEXT.vocab.vectors network = RNN(**kwargs) network.embedding.weight.data.copy_(pretrained_embeddings) UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token] PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token] network.embedding.weight.data[UNK_IDX] = torch.zeros(embedding_dim) network.embedding.weight.data[PAD_IDX] = torch.zeros(embedding_dim) optimizer = torch.optim.Adam(network.parameters()) model = Model( network=network, optimizer=optimizer, loss_function=custom_loss, batch_metrics=[acc], ) model.to(device) history = model.fit_generator( train_generator=train_iter, valid_generator=valid_iter, epochs=10, callbacks=[ ModelCheckpoint( filename=str(save_path / "model.pkl"), save_best_only=True, restore_best=True, ) ], ) print(f"Model saved to {save_path}") __import__("pudb").set_trace() test_loss, test_acc, y_pred, y_true = model.evaluate_generator( generator=test_iter, return_pred=True, return_ground_truth=True ) print(f"Test Loss: {test_loss:.4f}, Test Binary Accuracy: {test_acc:.4f}")
def test_disable_batch_size_warning(self): import warnings def tuple_generator(batch_size): while True: x1 = torch.rand(batch_size, 1) x2 = torch.rand(batch_size, 1) y1 = torch.rand(batch_size, 1) y2 = torch.rand(batch_size, 1) yield (x1, x2), (y1, y2) class TupleModule(nn.Module): def __init__(self): super().__init__() self.l1 = nn.Linear(1, 1) self.l2 = nn.Linear(1, 1) def forward(self, x): # pylint: disable=arguments-differ x1, x2 = x return self.l1(x1), self.l2(x2) def loss_function(y_pred, y_true): return F.mse_loss(y_pred[0], y_true[0]) + F.mse_loss( y_pred[1], y_true[1]) pytorch_module = TupleModule() optimizer = torch.optim.SGD(pytorch_module.parameters(), lr=1e-3) model = Model(pytorch_module, optimizer, loss_function) train_generator = tuple_generator(ModelTest.batch_size) valid_generator = tuple_generator(ModelTest.batch_size) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) num_warnings = ModelTest.steps_per_epoch * 2 * ModelTest.epochs self.assertEqual(len(w), num_warnings) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") warning_settings['batch_size'] = 'ignore' model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) self.assertEqual(len(w), 0)
class BaseTensorBoardLoggerTest: SummaryWriter = None batch_size = 20 lr = 1e-3 num_epochs = 10 def setUp(self): torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=BaseTensorBoardLoggerTest.lr) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) self.temp_dir_obj = TemporaryDirectory() # pylint: disable=not-callable self.writer = self.SummaryWriter(self.temp_dir_obj.name) self.writer.add_scalars = MagicMock() def tearDown(self): self.temp_dir_obj.cleanup() def test_logging(self): train_gen = some_data_generator(20) valid_gen = some_data_generator(20) logger = TensorBoardLogger(self.writer) history = self.model.fit_generator(train_gen, valid_gen, epochs=self.num_epochs, steps_per_epoch=5, callbacks=[logger]) self._test_logging(history) def _test_logging(self, history): calls = list() for h in history: calls.append( call('loss', { 'loss': h['loss'], 'val_loss': h['val_loss'] }, h['epoch'])) calls.append(call('lr', {'lr': self.lr}, h['epoch'])) self.writer.add_scalars.assert_has_calls(calls, any_order=True)
def start_record(models, datasets, device, batch_size, representations, epochs, metrics): for ds in datasets: for model_name, model in models.items(): print(f'{model_name} with {ds.name} :\n') ds.to(device) splitter = DataSplit(ds, test_train_split=TEST_TRAIN_SPLIT, val_train_split=VAL_TRAIN_SPLIT, shuffle=True) loaders = splitter.get_split(batch_size=batch_size) train_loader, valid_loader, test_loader = loaders net = model['network'](dataset=ds) optimizer = model['optimizer'][0](net.parameters(), **model['optimizer'][1]) base_callback = BaseCB(model_name=model_name, dataset_name=ds.name, records_path=RECORDS_PATH) callbacks = [ base_callback, DatasetCB(base_callback=base_callback, dataset=ds, batch_size=batch_size), MetricsCB(base_callback=base_callback, batch_metrics=metrics), LearningRateCB(base_callback=base_callback), DecisionBoundariesCB(base_callback=base_callback, dataset=ds, representations=representations), WeightsBiasesCB(base_callback=base_callback) ] if model['scheduler'] is not None: scheduler = model['scheduler'][0](**model['scheduler'][1]) callbacks.append(scheduler) batch_metrics = [metrics_list[metric] for metric in metrics] poutyne_model = Model(net, optimizer, model['loss function'], batch_metrics=batch_metrics) poutyne_model.to(device) poutyne_model.fit_generator(train_loader, valid_loader, epochs=epochs, callbacks=callbacks) test_loss, test_acc = poutyne_model.evaluate_generator(test_loader) print(f'Test:\n\tLoss: {test_loss}\n\tAccuracy: {test_acc}\n')
def __init__(self): # global spectral_accuracy, image_accuracy super().__init__() spectral_net = nn(np.shape(X_spectral_train)[-1], spectral_layers, spectral_dropout, material_count) opt = torch.optim.Adam(spectral_net.parameters(), lr=lr) spectral_model = Model(spectral_net, opt, 'cross_entropy', batch_metrics=['accuracy']) spectral_model.fit(X_spectral_train, y_train, epochs=spectral_epochs, batch_size=batch_size, verbose=False) image_net = nn(np.shape(X_image_train)[-1], image_layers, image_dropout, material_count) opt = torch.optim.Adam(image_net.parameters(), lr=lr) image_model = Model(image_net, opt, 'cross_entropy', batch_metrics=['accuracy']) image_model.fit(X_image_train, y_train, epochs=image_epochs, batch_size=batch_size, verbose=False) # Disable dropout, remove last layer and freeze network self.trained_spectral_model = torch.nn.Sequential(*(list(spectral_net.children())[:-1])) for p in self.trained_spectral_model.parameters(): p.requires_grad = False self.trained_image_model = torch.nn.Sequential(*(list(image_net.children())[:-1])) for p in self.trained_image_model.parameters(): p.requires_grad = False self.concat_net = nn(spectral_layers[-1] + image_layers[-1], layers, dropout, material_count, batchnorm=False)
def test_evaluate_with_no_metric(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss = model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float)
class ModelTest(ModelFittingTestCase): # pylint: disable=too-many-public-methods def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.batch_metrics = [ some_batch_metric_1, ('custom_name', some_batch_metric_2), repeat_batch_metric, repeat_batch_metric ] self.batch_metrics_names = [ 'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1', 'repeat_batch_metric2' ] self.batch_metrics_values = [ some_metric_1_value, some_metric_2_value, repeat_batch_metric_value, repeat_batch_metric_value ] self.epoch_metrics = [SomeConstantEpochMetric()] self.epoch_metrics_names = ['some_constant_epoch_metric'] self.epoch_metrics_values = [some_constant_epoch_metric_value] self.model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics) def test_fitting_tensor_generator(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) logs = self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch } self._test_callbacks_train(params, logs) def test_fitting_without_valid_generator(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) logs = self.model.fit_generator( train_generator, None, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch } self._test_callbacks_train(params, logs, has_valid=False) def test_correct_optim_calls_1_batch_per_step(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) mocked_optimizer = some_mocked_optimizer() mocked_optim_model = Model(self.pytorch_network, mocked_optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics) mocked_optim_model.fit_generator(train_generator, None, epochs=1, steps_per_epoch=1, batches_per_step=1) self.assertEqual(1, mocked_optimizer.step.call_count) self.assertEqual(1, mocked_optimizer.zero_grad.call_count) def test_correct_optim_calls__valid_n_batches_per_step(self): n_batches = 5 items_per_batch = int(ModelTest.batch_size / n_batches) x = torch.rand(n_batches, items_per_batch, 1) y = torch.rand(n_batches, items_per_batch, 1) mocked_optimizer = some_mocked_optimizer() mocked_optim_model = Model(self.pytorch_network, mocked_optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics) mocked_optim_model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=n_batches) self.assertEqual(1, mocked_optimizer.step.call_count) self.assertEqual(1, mocked_optimizer.zero_grad.call_count) def test_fitting_generator_n_batches_per_step(self): total_batch_size = 6 x = torch.rand(1, total_batch_size, 1) y = torch.rand(1, total_batch_size, 1) initial_params = self.model.get_weight_copies() self.model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=1) expected_params = list(self.model.get_weight_copies().values()) for mini_batch_size in [1, 2, 5]: self.model.set_weights(initial_params) n_batches_per_step = int(total_batch_size / mini_batch_size) x.resize_((n_batches_per_step, mini_batch_size, 1)) y.resize_((n_batches_per_step, mini_batch_size, 1)) self.model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=n_batches_per_step) returned_params = list(self.model.get_weight_copies().values()) np.testing.assert_almost_equal(returned_params, expected_params, decimal=4) def test_fitting_generator_n_batches_per_step_higher_than_num_batches( self): total_batch_size = 6 x = torch.rand(1, total_batch_size, 1) y = torch.rand(1, total_batch_size, 1) initial_params = self.model.get_weight_copies() self.model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=1) expected_params = list(self.model.get_weight_copies().values()) self.model.set_weights(initial_params) self.model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=2) returned_params = list(self.model.get_weight_copies().values()) np.testing.assert_almost_equal(returned_params, expected_params, decimal=4) def test_fitting_generator_n_batches_per_step_uneven_batches(self): total_batch_size = 6 x = torch.rand(1, total_batch_size, 1) y = torch.rand(1, total_batch_size, 1) initial_params = self.model.get_weight_copies() self.model.fit_generator(list(zip(x, y)), None, epochs=1, batches_per_step=1) expected_params = list(self.model.get_weight_copies().values()) x.squeeze_(dim=0) y.squeeze_(dim=0) uneven_chunk_sizes = [4, 5] for chunk_size in uneven_chunk_sizes: self.model.set_weights(initial_params) splitted_x = x.split(chunk_size) splitted_y = y.split(chunk_size) n_batches_per_step = ceil(total_batch_size / chunk_size) self.model.fit_generator(list(zip(splitted_x, splitted_y)), None, epochs=1, batches_per_step=n_batches_per_step) returned_params = list(self.model.get_weight_copies().values()) np.testing.assert_almost_equal(returned_params, expected_params, decimal=4) def test_fitting_ndarray_generator(self): train_generator = some_ndarray_generator(ModelTest.batch_size) valid_generator = some_ndarray_generator(ModelTest.batch_size) logs = self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch } self._test_callbacks_train(params, logs) def test_fitting_with_data_loader(self): train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = torch.rand(train_size, 1) train_y = torch.rand(train_size, 1) train_dataset = TensorDataset(train_x, train_y) train_generator = DataLoader(train_dataset, train_batch_size) valid_real_steps_per_epoch = 10 valid_batch_size = 15 valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = torch.rand(valid_size, 1) valid_y = torch.rand(valid_size, 1) valid_dataset = TensorDataset(valid_x, valid_y) valid_generator = DataLoader(valid_dataset, valid_batch_size) logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_callbacks_train(params, logs) def test_fitting_generator_calls(self): train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = torch.rand(train_size, 1) train_y = torch.rand(train_size, 1) train_dataset = TensorDataset(train_x, train_y) train_generator = DataLoader(train_dataset, train_batch_size) valid_real_steps_per_epoch = 10 valid_batch_size = 15 valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = torch.rand(valid_size, 1) valid_y = torch.rand(valid_size, 1) valid_dataset = TensorDataset(valid_x, valid_y) valid_generator = DataLoader(valid_dataset, valid_batch_size) class IterableMock: def __init__(self, iterable): self.iterable = iterable self.iter = None self.calls = [] def __iter__(self): self.calls.append('__iter__') self.iter = iter(self.iterable) return self def __next__(self): self.calls.append('__next__') return next(self.iter) def __len__(self): self.calls.append('__len__') return len(self.iterable) mock_train_generator = IterableMock(train_generator) mock_valid_generator = IterableMock(valid_generator) self.model.fit_generator(mock_train_generator, mock_valid_generator, epochs=ModelTest.epochs) expected_train_calls = ['__len__'] + \ (['__iter__'] + ['__next__'] * train_real_steps_per_epoch) * ModelTest.epochs expected_valid_calls = ['__len__'] + \ (['__iter__'] + ['__next__'] * valid_real_steps_per_epoch) * ModelTest.epochs self.assertEqual(mock_train_generator.calls, expected_train_calls) self.assertEqual(mock_valid_generator.calls, expected_valid_calls) def test_fitting_with_tensor(self): train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = torch.rand(train_size, 1) train_y = torch.rand(train_size, 1) valid_real_steps_per_epoch = 10 # valid_batch_size will be the same as train_batch_size in the fit method. valid_batch_size = train_batch_size valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = torch.rand(valid_size, 1) valid_y = torch.rand(valid_size, 1) logs = self.model.fit(train_x, train_y, validation_data=(valid_x, valid_y), epochs=ModelTest.epochs, batch_size=train_batch_size, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_callbacks_train(params, logs) def test_fitting_with_np_array(self): train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = np.random.rand(train_size, 1).astype(np.float32) train_y = np.random.rand(train_size, 1).astype(np.float32) valid_real_steps_per_epoch = 10 # valid_batch_size will be the same as train_batch_size in the fit method. valid_batch_size = train_batch_size valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = np.random.rand(valid_size, 1).astype(np.float32) valid_y = np.random.rand(valid_size, 1).astype(np.float32) logs = self.model.fit(train_x, train_y, validation_data=(valid_x, valid_y), epochs=ModelTest.epochs, batch_size=train_batch_size, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_callbacks_train(params, logs) def test_fitting_with_generator_with_len(self): train_real_steps_per_epoch = 30 train_generator = SomeDataGeneratorWithLen( batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch, num_missing_samples=7) valid_generator = SomeDataGeneratorWithLen(batch_size=15, length=10, num_missing_samples=3) logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_callbacks_train(params, logs) def test_fitting_with_generator_with_stop_iteration(self): train_real_steps_per_epoch = 30 train_generator = SomeDataGeneratorUsingStopIteration( batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch) valid_generator = SomeDataGeneratorUsingStopIteration(batch_size=15, length=10) logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = {'epochs': ModelTest.epochs, 'steps': None} self._test_callbacks_train(params, logs, steps=train_real_steps_per_epoch) def test_tensor_train_on_batch(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics = self.model.train_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values) def test_train_on_batch_with_pred(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics, pred_y = self.model.train_on_batch(x, y, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) def test_ndarray_train_on_batch(self): x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) loss, metrics = self.model.train_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values) def test_evaluate(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss, metrics = self.model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values + self.epoch_metrics_values) def test_evaluate_with_pred(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) # We also test the unpacking. _, _, pred_y = self.model.evaluate(x, y, batch_size=ModelTest.batch_size, return_pred=True) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_evaluate_with_callback(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) # We also test the unpacking. _, _, pred_y = self.model.evaluate(x, y, batch_size=ModelTest.batch_size, return_pred=True, callbacks=[self.mock_callback]) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_evaluate_with_np_array(self): x = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32) y = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32) loss, metrics, pred_y = self.model.evaluate( x, y, batch_size=ModelTest.batch_size, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values + self.epoch_metrics_values) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_evaluate_data_loader(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) dataset = TensorDataset(x, y) generator = DataLoader(dataset, ModelTest.batch_size) loss, metrics, pred_y = self.model.evaluate_generator(generator, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values + self.epoch_metrics_values) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_evaluate_generator(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) loss, metrics, pred_y = self.model.evaluate_generator(generator, steps=num_steps, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values + self.epoch_metrics_values) self.assertEqual(type(pred_y), np.ndarray) self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1)) def test_evaluate_generator_with_callback(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) result_log = self.model.evaluate_generator( generator, steps=num_steps, return_pred=True, callbacks=[self.mock_callback]) params = {'batch': ModelTest.epochs} self._test_callbacks_test(params, result_log) def test_evaluate_generator_with_ground_truth(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) loss, metrics, pred_y, true_y = self.model.evaluate_generator( generator, steps=num_steps, return_pred=True, return_ground_truth=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values + self.epoch_metrics_values) self.assertEqual(type(pred_y), np.ndarray) self.assertEqual(type(true_y), np.ndarray) self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1)) self.assertEqual(true_y.shape, (num_steps * ModelTest.batch_size, 1)) def test_evaluate_generator_with_no_concatenation(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) loss, metrics, pred_y, true_y = self.model.evaluate_generator( generator, steps=num_steps, return_pred=True, return_ground_truth=True, concatenate_returns=False) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values + self.epoch_metrics_values) self.assertEqual(type(pred_y), list) for pred in pred_y: self.assertEqual(type(pred), np.ndarray) self.assertEqual(pred.shape, (ModelTest.batch_size, 1)) self.assertEqual(type(true_y), list) for true in true_y: self.assertEqual(type(true), np.ndarray) self.assertEqual(true.shape, (ModelTest.batch_size, 1)) def test_evaluate_with_only_one_metric(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics[:1]) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss, first_metric = model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) self.assertEqual(type(first_metric), float) self.assertEqual(first_metric, some_metric_1_value) def test_metrics_integration(self): num_steps = 10 model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[F.mse_loss]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) generator = some_data_tensor_generator(ModelTest.batch_size) loss, mse = model.evaluate_generator(generator, steps=num_steps) self.assertEqual(type(loss), float) self.assertEqual(type(mse), float) def test_epoch_metrics_integration(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[SomeEpochMetric()]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) logs = model.fit_generator(train_generator, valid_generator, epochs=1, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) actual_value = logs[-1]['some_epoch_metric'] val_actual_value = logs[-1]['val_some_epoch_metric'] expected_value = 5 self.assertEqual(val_actual_value, expected_value) self.assertEqual(actual_value, expected_value) def test_evaluate_with_no_metric(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss = model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) def test_tensor_evaluate_on_batch(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics = self.model.evaluate_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values) def test_evaluate_on_batch_with_pred(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics, pred_y = self.model.evaluate_on_batch(x, y, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) def test_ndarray_evaluate_on_batch(self): x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) loss, metrics = self.model.evaluate_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), self.batch_metrics_values) def test_predict(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) pred_y = self.model.predict(x, batch_size=ModelTest.batch_size) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_predict_with_np_array(self): x = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32) pred_y = self.model.predict(x, batch_size=ModelTest.batch_size) self.assertEqual(type(pred_y), np.ndarray) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_predict_data_loader(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) generator = DataLoader(x, ModelTest.batch_size) pred_y = self.model.predict_generator(generator) self.assertEqual(type(pred_y), np.ndarray) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_predict_generator(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) generator = (x for x, _ in generator) pred_y = self.model.predict_generator(generator, steps=num_steps) self.assertEqual(type(pred_y), np.ndarray) self.assertEqual(pred_y.shape, (num_steps * ModelTest.batch_size, 1)) def test_predict_generator_with_no_concatenation(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) generator = (x for x, _ in generator) pred_y = self.model.predict_generator(generator, steps=num_steps, concatenate_returns=False) self.assertEqual(type(pred_y), list) for pred in pred_y: self.assertEqual(type(pred), np.ndarray) self.assertEqual(pred.shape, (ModelTest.batch_size, 1)) def test_tensor_predict_on_batch(self): x = torch.rand(ModelTest.batch_size, 1) pred_y = self.model.predict_on_batch(x) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) def test_ndarray_predict_on_batch(self): x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) pred_y = self.model.predict_on_batch(x) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) @skipIf(not torch.cuda.is_available(), "no gpu available") def test_cpu_cuda(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) with torch.cuda.device(ModelTest.cuda_device): self.model.cuda() self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) # The context manager is also used here because of this bug: # https://github.com/pytorch/pytorch/issues/7320 with torch.cuda.device(ModelTest.cuda_device): self.model.cuda(ModelTest.cuda_device) self._test_device( torch.device('cuda:' + str(ModelTest.cuda_device))) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) self.model.cpu() self._test_device(torch.device('cpu')) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) self.model.to(torch.device('cuda:' + str(ModelTest.cuda_device))) self._test_device( torch.device('cuda:' + str(ModelTest.cuda_device))) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) self.model.to(torch.device('cpu')) self._test_device(torch.device('cpu')) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) def _test_device(self, device): for p in self.pytorch_network.parameters(): self.assertEqual(p.device, device) def test_get_batch_size(self): batch_size = ModelTest.batch_size x = np.random.rand(batch_size, 1).astype(np.float32) y = np.random.rand(batch_size, 1).astype(np.float32) batch_size2 = ModelTest.batch_size + 1 x2 = np.random.rand(batch_size2, 1).astype(np.float32) y2 = np.random.rand(batch_size2, 1).astype(np.float32) other_batch_size = batch_size2 + 1 inf_batch_size = self.model.get_batch_size(x, y) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size(x2, y2) self.assertEqual(inf_batch_size, batch_size2) inf_batch_size = self.model.get_batch_size(x, y2) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size(x2, y) self.assertEqual(inf_batch_size, batch_size2) inf_batch_size = self.model.get_batch_size((x, x2), y) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size((x2, x), y) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size((x, x2), (y, y2)) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size((x2, x), (y, y2)) self.assertEqual(inf_batch_size, batch_size2) inf_batch_size = self.model.get_batch_size([x, x2], y) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size([x2, x], y) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size([x, x2], [y, y2]) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size([x2, x], [y, y2]) self.assertEqual(inf_batch_size, batch_size2) inf_batch_size = self.model.get_batch_size( { 'batch_size': other_batch_size, 'x': x }, {'y': y}) self.assertEqual(inf_batch_size, other_batch_size) inf_batch_size = self.model.get_batch_size({'x': x}, { 'batch_size': other_batch_size, 'y': y }) self.assertEqual(inf_batch_size, other_batch_size) inf_batch_size = self.model.get_batch_size({'x': x}, {'y': y}) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size( OrderedDict([('x1', x), ('x2', x2)]), {'y': y}) self.assertEqual(inf_batch_size, batch_size) inf_batch_size = self.model.get_batch_size( OrderedDict([('x1', x2), ('x2', x)]), {'y': y}) self.assertEqual(inf_batch_size, batch_size2) inf_batch_size = self.model.get_batch_size([1, 2, 3], {'y': y}) self.assertEqual(inf_batch_size, batch_size) def test_get_batch_size_warning(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") inf_batch_size = self.model.get_batch_size([1, 2, 3], [4, 5, 6]) self.assertEqual(inf_batch_size, 1) self.assertEqual(len(w), 1) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") warning_settings['batch_size'] = 'ignore' inf_batch_size = self.model.get_batch_size([1, 2, 3], [4, 5, 6]) self.assertEqual(inf_batch_size, 1) self.assertEqual(len(w), 0)
class ModelTest(TestCase): # pylint: disable=too-many-public-methods epochs = 10 steps_per_epoch = 5 batch_size = 20 evaluate_dataset_len = 107 cuda_device = int(os.environ.get('CUDA_DEVICE', 0)) def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.metrics = [some_metric_1, some_metric_2] self.metrics_names = ['some_metric_1', 'some_metric_2'] self.metrics_values = [some_metric_1_value, some_metric_2_value] self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=self.metrics) self.mock_callback = MagicMock() def test_fitting_tensor_generator(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) logs = self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch } self._test_fitting(params, logs) def test_fitting_without_valid_generator(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) logs = self.model.fit_generator( train_generator, None, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch } self._test_fitting(params, logs, has_valid=False) def test_fitting_ndarray_generator(self): train_generator = some_ndarray_generator(ModelTest.batch_size) valid_generator = some_ndarray_generator(ModelTest.batch_size) logs = self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch } self._test_fitting(params, logs) def test_fitting_with_data_loader(self): # pylint: disable=too-many-locals train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = torch.rand(train_size, 1) train_y = torch.rand(train_size, 1) train_dataset = TensorDataset(train_x, train_y) train_generator = DataLoader(train_dataset, train_batch_size) valid_real_steps_per_epoch = 10 valid_batch_size = 15 valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = torch.rand(valid_size, 1) valid_y = torch.rand(valid_size, 1) valid_dataset = TensorDataset(valid_x, valid_y) valid_generator = DataLoader(valid_dataset, valid_batch_size) logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_fitting(params, logs) def test_fitting_with_tensor(self): # pylint: disable=too-many-locals train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = torch.rand(train_size, 1) train_y = torch.rand(train_size, 1) valid_real_steps_per_epoch = 10 # valid_batch_size will be the same as train_batch_size in the fit method. valid_batch_size = train_batch_size valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = torch.rand(valid_size, 1) valid_y = torch.rand(valid_size, 1) logs = self.model.fit(train_x, train_y, validation_x=valid_x, validation_y=valid_y, epochs=ModelTest.epochs, batch_size=train_batch_size, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_fitting(params, logs) def test_fitting_with_np_array(self): # pylint: disable=too-many-locals train_real_steps_per_epoch = 30 train_batch_size = ModelTest.batch_size train_final_batch_missing_samples = 7 train_size = train_real_steps_per_epoch * train_batch_size - \ train_final_batch_missing_samples train_x = np.random.rand(train_size, 1).astype(np.float32) train_y = np.random.rand(train_size, 1).astype(np.float32) valid_real_steps_per_epoch = 10 # valid_batch_size will be the same as train_batch_size in the fit method. valid_batch_size = train_batch_size valid_final_batch_missing_samples = 3 valid_size = valid_real_steps_per_epoch * valid_batch_size - \ valid_final_batch_missing_samples valid_x = np.random.rand(valid_size, 1).astype(np.float32) valid_y = np.random.rand(valid_size, 1).astype(np.float32) logs = self.model.fit(train_x, train_y, validation_x=valid_x, validation_y=valid_y, epochs=ModelTest.epochs, batch_size=train_batch_size, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_fitting(params, logs) def test_fitting_with_generator_with_len(self): train_real_steps_per_epoch = 30 train_generator = SomeDataGeneratorWithLen( batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch, num_missing_samples=7) valid_generator = SomeDataGeneratorWithLen(batch_size=15, length=10, num_missing_samples=3) logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = { 'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch } self._test_fitting(params, logs) def test_fitting_with_generator_with_stop_iteration(self): train_real_steps_per_epoch = 30 train_generator = SomeDataGeneratorUsingStopIteration( batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch) valid_generator = SomeDataGeneratorUsingStopIteration(batch_size=15, length=10) logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback]) params = {'epochs': ModelTest.epochs, 'steps': None} self._test_fitting(params, logs, steps=train_real_steps_per_epoch) def _test_fitting(self, params, logs, has_valid=True, steps=None): if steps is None: steps = params['steps'] self.assertEqual(len(logs), params['epochs']) train_dict = dict(zip(self.metrics_names, self.metrics_values), loss=ANY, time=ANY) if has_valid: val_metrics_names = [ 'val_' + metric_name for metric_name in self.metrics_names ] val_dict = dict(zip(val_metrics_names, self.metrics_values), val_loss=ANY) log_dict = {**train_dict, **val_dict} else: log_dict = train_dict for epoch, log in enumerate(logs, 1): self.assertEqual(log, dict(log_dict, epoch=epoch)) call_list = [] call_list.append(call.on_train_begin({})) for epoch in range(1, params['epochs'] + 1): call_list.append(call.on_epoch_begin(epoch, {})) for step in range(1, steps + 1): call_list.append(call.on_batch_begin(step, {})) call_list.append(call.on_backward_end(step)) call_list.append( call.on_batch_end(step, { 'batch': step, 'size': ANY, **train_dict })) call_list.append( call.on_epoch_end(epoch, { 'epoch': epoch, **log_dict })) call_list.append(call.on_train_end({})) method_calls = self.mock_callback.method_calls self.assertIn(call.set_model(self.model), method_calls[:2]) self.assertIn(call.set_params(params), method_calls[:2]) self.assertEqual(len(method_calls), len(call_list) + 2) self.assertEqual(method_calls[2:], call_list) def test_tensor_train_on_batch(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics = self.model.train_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) def test_train_on_batch_with_pred(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics, pred_y = self.model.train_on_batch(x, y, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) def test_ndarray_train_on_batch(self): x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) loss, metrics = self.model.train_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) def test_evaluate(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss, metrics = self.model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) def test_evaluate_with_pred(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) # We also test the unpacking. # pylint: disable=unused-variable loss, metrics, pred_y = self.model.evaluate( x, y, batch_size=ModelTest.batch_size, return_pred=True) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_evaluate_with_np_array(self): x = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32) y = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32) loss, metrics, pred_y = self.model.evaluate( x, y, batch_size=ModelTest.batch_size, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_evaluate_data_loader(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) dataset = TensorDataset(x, y) generator = DataLoader(dataset, ModelTest.batch_size) loss, metrics, pred_y = self.model.evaluate_generator(generator, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) self._test_predictions_for_evaluate_and_predict_generator(pred_y) def test_evaluate_generator(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) loss, metrics, pred_y = self.model.evaluate_generator(generator, steps=num_steps, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) for pred in pred_y: self.assertEqual(type(pred), np.ndarray) self.assertEqual(pred.shape, (ModelTest.batch_size, 1)) self.assertEqual( np.concatenate(pred_y).shape, (num_steps * ModelTest.batch_size, 1)) def test_evaluate_with_only_one_metric(self): self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=self.metrics[:1]) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss, first_metric = self.model.evaluate( x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) self.assertEqual(type(first_metric), float) self.assertEqual(first_metric, some_metric_1_value) def test_metrics_integration(self): num_steps = 10 self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=[F.mse_loss]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) generator = some_data_tensor_generator(ModelTest.batch_size) loss, mse = self.model.evaluate_generator(generator, steps=num_steps) self.assertEqual(type(loss), float) self.assertEqual(type(mse), float) def test_evaluate_with_no_metric(self): self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss = self.model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) def test_tensor_evaluate_on_batch(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics = self.model.evaluate_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) def test_evaluate_on_batch_with_pred(self): x = torch.rand(ModelTest.batch_size, 1) y = torch.rand(ModelTest.batch_size, 1) loss, metrics, pred_y = self.model.evaluate_on_batch(x, y, return_pred=True) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) def test_ndarray_evaluate_on_batch(self): x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) loss, metrics = self.model.evaluate_on_batch(x, y) self.assertEqual(type(loss), float) self.assertEqual(type(metrics), np.ndarray) self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value]) def test_predict(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) pred_y = self.model.predict(x, batch_size=ModelTest.batch_size) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_predict_with_np_array(self): x = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32) pred_y = self.model.predict(x, batch_size=ModelTest.batch_size) self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1)) def test_predict_data_loader(self): x = torch.rand(ModelTest.evaluate_dataset_len, 1) generator = DataLoader(x, ModelTest.batch_size) pred_y = self.model.predict_generator(generator) self._test_predictions_for_evaluate_and_predict_generator(pred_y) def test_predict_generator(self): num_steps = 10 generator = some_data_tensor_generator(ModelTest.batch_size) generator = (x for x, _ in generator) pred_y = self.model.predict_generator(generator, steps=num_steps) for pred in pred_y: self.assertEqual(type(pred), np.ndarray) self.assertEqual(pred.shape, (ModelTest.batch_size, 1)) self.assertEqual( np.concatenate(pred_y).shape, (num_steps * ModelTest.batch_size, 1)) def _test_predictions_for_evaluate_and_predict_generator(self, pred_y): self.assertEqual(type(pred_y), list) remaning_example = ModelTest.evaluate_dataset_len cur_batch_size = ModelTest.batch_size for pred in pred_y: self.assertEqual(type(pred), np.ndarray) if remaning_example < ModelTest.batch_size: cur_batch_size = remaning_example remaning_example = 0 else: remaning_example -= ModelTest.batch_size self.assertEqual(pred.shape, (cur_batch_size, 1)) self.assertEqual( np.concatenate(pred_y).shape, (ModelTest.evaluate_dataset_len, 1)) def test_tensor_predict_on_batch(self): x = torch.rand(ModelTest.batch_size, 1) pred_y = self.model.predict_on_batch(x) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) def test_ndarray_predict_on_batch(self): x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32) pred_y = self.model.predict_on_batch(x) self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1)) @skipIf(not torch.cuda.is_available(), "no gpu available") def test_cpu_cuda(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) with torch.cuda.device(ModelTest.cuda_device): self.model.cuda() self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) # The context manager is also used here because of this bug: # https://github.com/pytorch/pytorch/issues/7320 with torch.cuda.device(ModelTest.cuda_device): self.model.cuda(ModelTest.cuda_device) self._test_device( torch.device('cuda:' + str(ModelTest.cuda_device))) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) self.model.cpu() self._test_device(torch.device('cpu')) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) self.model.to(torch.device('cuda:' + str(ModelTest.cuda_device))) self._test_device( torch.device('cuda:' + str(ModelTest.cuda_device))) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) self.model.to(torch.device('cpu')) self._test_device(torch.device('cpu')) self.model.fit_generator( train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) def _test_device(self, device): for p in self.pytorch_module.parameters(): self.assertEqual(p.device, device) def test_disable_batch_size_warning(self): import warnings def tuple_generator(batch_size): while True: x1 = torch.rand(batch_size, 1) x2 = torch.rand(batch_size, 1) y1 = torch.rand(batch_size, 1) y2 = torch.rand(batch_size, 1) yield (x1, x2), (y1, y2) class TupleModule(nn.Module): def __init__(self): super().__init__() self.l1 = nn.Linear(1, 1) self.l2 = nn.Linear(1, 1) def forward(self, x): # pylint: disable=arguments-differ x1, x2 = x return self.l1(x1), self.l2(x2) def loss_function(y_pred, y_true): return F.mse_loss(y_pred[0], y_true[0]) + F.mse_loss( y_pred[1], y_true[1]) pytorch_module = TupleModule() optimizer = torch.optim.SGD(pytorch_module.parameters(), lr=1e-3) model = Model(pytorch_module, optimizer, loss_function) train_generator = tuple_generator(ModelTest.batch_size) valid_generator = tuple_generator(ModelTest.batch_size) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) num_warnings = ModelTest.steps_per_epoch * 2 * ModelTest.epochs self.assertEqual(len(w), num_warnings) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") warning_settings['batch_size'] = 'ignore' model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) self.assertEqual(len(w), 0)