def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.batch_metrics = [ some_batch_metric_1, ('custom_name', some_batch_metric_2), repeat_batch_metric, repeat_batch_metric ] self.batch_metrics_names = [ 'some_batch_metric_1', 'custom_name', 'repeat_batch_metric1', 'repeat_batch_metric2' ] self.batch_metrics_values = [ some_metric_1_value, some_metric_2_value, repeat_batch_metric_value, repeat_batch_metric_value ] self.epoch_metrics = [SomeConstantEpochMetric()] self.epoch_metrics_names = ['some_constant_epoch_metric'] self.epoch_metrics_values = [some_constant_epoch_metric_value] self.model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.metrics = [some_metric_1, some_metric_2] self.metrics_names = ['some_metric_1', 'some_metric_2'] self.metrics_values = [some_metric_1_value, some_metric_2_value] self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=self.metrics) self.multi_input_model = Model(MultiInputModel(), self.optimizer, self.loss_function, metrics=self.metrics) self.mock_callback = MagicMock()
def test_evaluate_with_no_metric(self): self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss = self.model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float)
def test_epoch_metrics_with_name_with_multiple_names_returned_by_tensor(self): class EpochMetricWithName(ConstEpochMetric): __name__ = self.metric_names epoch_metric = EpochMetricWithName(torch.tensor(self.metric_values)) model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[epoch_metric]) self._test_history(model, self.metric_names, self.metric_values)
def test_repeated_epoch_metrics_handling(self): expected_names = ['some_metric_name1', 'some_metric_name2'] model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[SomeMetricName(1), SomeMetricName(2)]) self._test_history(model, expected_names, [1, 2])
def test_batch_metrics_with_multiple_names_returned_by_list(self): batch_metric = get_const_batch_metric(list(self.metric_values)) model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[(self.metric_names, batch_metric)]) self._test_history(model, self.metric_names, self.metric_values)
def test_epoch_metrics_with_multiple_names_returned_by_tuple(self): epoch_metric = ConstEpochMetric(tuple(self.metric_values)) model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[(self.metric_names, epoch_metric)]) self._test_history(model, self.metric_names, self.metric_values)
def train(self, features: np.ndarray, labels: np.ndarray, validation_features: Optional[np.ndarray] = None, validation_labels: Optional[np.ndarray] = None, epochs: int = 50, shuffle: bool = True, batch_size: int = 32) -> None: """ Trains the model :param features: consists of padded training segments and dummy tokens :param labels: consists of labels to predict :param validation_features: consists of padded training segments and dummy tokens :param validation_labels: consists of labels to predict """ # Wrap pytorch with pytoune (very keras-like) for easier training # TODO define adam manually with different learning rates net = Model(self.model, optimizer='adam', loss_function='cross_entropy', metrics=['accuracy']) if self.gpu_available: net = net.cuda() net.fit(features, labels, validation_x=validation_features, validation_y=validation_labels, batch_size=batch_size, epochs=epochs)
def __init__(self, directory, network, *, device=None, logging=True, optimizer='sgd', loss_function=None, batch_metrics=None, epoch_metrics=None, monitor_metric=None, monitor_mode=None, task=None): self.directory = directory self.logging = logging if task is not None and not task.startswith( 'classif') and not task.startswith('reg'): raise ValueError("Invalid task '%s'" % task) batch_metrics = [] if batch_metrics is None else batch_metrics epoch_metrics = [] if epoch_metrics is None else epoch_metrics loss_function = self._get_loss_function(loss_function, network, task) batch_metrics = self._get_batch_metrics(batch_metrics, network, task) epoch_metrics = self._get_epoch_metrics(epoch_metrics, network, task) self._set_monitor(monitor_metric, monitor_mode, task) self.model = Model(network, optimizer, loss_function, batch_metrics=batch_metrics, epoch_metrics=epoch_metrics) if device is not None: self.model.to(device) self.best_checkpoint_filename = self.get_path( Experiment.BEST_CHECKPOINT_FILENAME) self.best_checkpoint_tmp_filename = self.get_path( Experiment.BEST_CHECKPOINT_TMP_FILENAME) self.model_checkpoint_filename = self.get_path( Experiment.MODEL_CHECKPOINT_FILENAME) self.model_checkpoint_tmp_filename = self.get_path( Experiment.MODEL_CHECKPOINT_TMP_FILENAME) self.optimizer_checkpoint_filename = self.get_path( Experiment.OPTIMIZER_CHECKPOINT_FILENAME) self.optimizer_checkpoint_tmp_filename = self.get_path( Experiment.OPTIMIZER_CHECKPOINT_TMP_FILENAME) self.log_filename = self.get_path(Experiment.LOG_FILENAME) self.log_tmp_filename = self.get_path(Experiment.LOG_TMP_FILENAME) self.tensorboard_directory = self.get_path( Experiment.TENSORBOARD_DIRECTORY) self.epoch_filename = self.get_path(Experiment.EPOCH_FILENAME) self.epoch_tmp_filename = self.get_path(Experiment.EPOCH_TMP_FILENAME) self.lr_scheduler_filename = self.get_path( Experiment.LR_SCHEDULER_FILENAME) self.lr_scheduler_tmp_filename = self.get_path( Experiment.LR_SCHEDULER_TMP_FILENAME) self.test_log_filename = self.get_path(Experiment.TEST_LOG_FILENAME)
def setUp(self): torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) self.temp_dir_obj = TemporaryDirectory() self.checkpoint_filename = os.path.join(self.temp_dir_obj.name, 'my_checkpoint_{epoch}.ckpt')
def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.model = Model(self.pytorch_module, self.optimizer, self.loss_function)
def test_evaluate_with_only_one_metric(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics[:1]) x = torch.rand(ModelTest.evaluate_dataset_len, 1) y = torch.rand(ModelTest.evaluate_dataset_len, 1) loss, first_metric = model.evaluate(x, y, batch_size=ModelTest.batch_size) self.assertEqual(type(loss), float) self.assertEqual(type(first_metric), float) self.assertEqual(first_metric, some_metric_1_value)
def test_batch_metrics_with_multiple_names_returned_by_dict(self): d = dict(zip(self.metric_names, self.metric_values)) batch_metric = get_const_batch_metric(d) model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[(self.metric_names, batch_metric)]) self._test_history(model, d.keys(), d.values())
def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3) self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) self.train_gen = some_data_generator(20) self.valid_gen = some_data_generator(20)
def test_epoch_metrics_with_multiple_names_returned_by_tensor_on_gpu(self): with torch.cuda.device(MetricsModelIntegrationTest.cuda_device): epoch_metric = ConstEpochMetric(torch.tensor(self.metric_values).cuda()) model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[(self.metric_names, epoch_metric)]) model.cuda() self._test_history(model, self.metric_names, self.metric_values)
def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=CSVLoggerTest.lr) self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) self.temp_dir_obj = TemporaryDirectory() self.csv_filename = os.path.join(self.temp_dir_obj.name, 'my_log.csv')
def setUp(self): torch.manual_seed(42) self.pytorch_network = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) self.mock_callback = MagicMock(spec=Callback) self.delay_callback = DelayCallback(self.mock_callback) self.train_dict = {'loss': ANY, 'time': ANY} self.log_dict = {'loss': ANY, 'val_loss': ANY, 'time': ANY}
def main(rnn_type, n_layers, dataset, embedding, device, save_path): train_iter, valid_iter, test_iter = dataset_factory(dataset, embedding=embedding) embedding_dim = int(embedding.split(".")[-1][:-1]) save_path = Path(save_path) / f"{rnn_type}_{n_layers}layer_{embedding_dim}" save_path.mkdir(parents=True, exist_ok=True) kwargs = dict( vocab_size=len(TEXT.vocab), embedding_dim=embedding_dim, hidden_dim=256, output_dim=1, n_layers=n_layers, dropout=0.5, pad_idx=TEXT.vocab.stoi[TEXT.pad_token], rnn_type="gru", ) with open(save_path / "kwargs.json", "w") as kwargs_file: json.dump(kwargs, kwargs_file) pretrained_embeddings = TEXT.vocab.vectors network = RNN(**kwargs) network.embedding.weight.data.copy_(pretrained_embeddings) UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token] PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token] network.embedding.weight.data[UNK_IDX] = torch.zeros(embedding_dim) network.embedding.weight.data[PAD_IDX] = torch.zeros(embedding_dim) optimizer = torch.optim.Adam(network.parameters()) model = Model( network=network, optimizer=optimizer, loss_function=custom_loss, batch_metrics=[acc], ) model.to(device) history = model.fit_generator( train_generator=train_iter, valid_generator=valid_iter, epochs=10, callbacks=[ ModelCheckpoint( filename=str(save_path / "model.pkl"), save_best_only=True, restore_best=True, ) ], ) print(f"Model saved to {save_path}") __import__("pudb").set_trace() test_loss, test_acc, y_pred, y_true = model.evaluate_generator( generator=test_iter, return_pred=True, return_ground_truth=True ) print(f"Test Loss: {test_loss:.4f}, Test Binary Accuracy: {test_acc:.4f}")
def setUp(self): torch.manual_seed(42) self.pytorch_network = nn.Sequential(nn.Linear(1, 2), nn.Linear(2, 1)) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function) self.temp_dir_obj = TemporaryDirectory() self.csv_filename = os.path.join(self.temp_dir_obj.name, 'layer_{}.csv')
def setUp(self): torch.manual_seed(42) self.pytorch_module = nn.Linear(1, 1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=BaseTensorBoardLoggerTest.lr) self.model = Model(self.pytorch_module, self.optimizer, self.loss_function) self.temp_dir_obj = TemporaryDirectory() self.writer = self.SummaryWriter(self.temp_dir_obj.name) self.writer.add_scalars = MagicMock()
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = DictIOModel(['x1', 'x2'], ['y1', 'y2']) self.loss_function = dict_mse_loss self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = MultiIOModel(num_input=1, num_output=2) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model( self.pytorch_network, self.optimizer, lambda y_pred, y_true: self.loss_function(y_pred[0], y_true[0]) + self.loss_function(y_pred[1], y_true[1]), batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def __init__(self): # global spectral_accuracy, image_accuracy super().__init__() spectral_net = nn(np.shape(X_spectral_train)[-1], spectral_layers, spectral_dropout, material_count) opt = torch.optim.Adam(spectral_net.parameters(), lr=lr) spectral_model = Model(spectral_net, opt, 'cross_entropy', batch_metrics=['accuracy']) spectral_model.fit(X_spectral_train, y_train, epochs=spectral_epochs, batch_size=batch_size, verbose=False) image_net = nn(np.shape(X_image_train)[-1], image_layers, image_dropout, material_count) opt = torch.optim.Adam(image_net.parameters(), lr=lr) image_model = Model(image_net, opt, 'cross_entropy', batch_metrics=['accuracy']) image_model.fit(X_image_train, y_train, epochs=image_epochs, batch_size=batch_size, verbose=False) # Disable dropout, remove last layer and freeze network self.trained_spectral_model = torch.nn.Sequential(*(list(spectral_net.children())[:-1])) for p in self.trained_spectral_model.parameters(): p.requires_grad = False self.trained_image_model = torch.nn.Sequential(*(list(image_net.children())[:-1])) for p in self.trained_image_model.parameters(): p.requires_grad = False self.concat_net = nn(spectral_layers[-1] + image_layers[-1], layers, dropout, material_count, batchnorm=False)
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = MultiIOModel(num_input=1, num_output=1) self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def test_correct_optim_calls_1_batch_per_step(self): train_generator = some_data_tensor_generator(ModelTest.batch_size) mocked_optimizer = some_mocked_optimizer() mocked_optim_model = Model(self.pytorch_network, mocked_optimizer, self.loss_function, batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics) mocked_optim_model.fit_generator(train_generator, None, epochs=1, steps_per_epoch=1, batches_per_step=1) self.assertEqual(1, mocked_optimizer.step.call_count) self.assertEqual(1, mocked_optimizer.zero_grad.call_count)
def test_disable_batch_size_warning(self): import warnings def tuple_generator(batch_size): while True: x1 = torch.rand(batch_size, 1) x2 = torch.rand(batch_size, 1) y1 = torch.rand(batch_size, 1) y2 = torch.rand(batch_size, 1) yield (x1, x2), (y1, y2) class TupleModule(nn.Module): def __init__(self): super().__init__() self.l1 = nn.Linear(1, 1) self.l2 = nn.Linear(1, 1) def forward(self, x): # pylint: disable=arguments-differ x1, x2 = x return self.l1(x1), self.l2(x2) def loss_function(y_pred, y_true): return F.mse_loss(y_pred[0], y_true[0]) + F.mse_loss( y_pred[1], y_true[1]) pytorch_module = TupleModule() optimizer = torch.optim.SGD(pytorch_module.parameters(), lr=1e-3) model = Model(pytorch_module, optimizer, loss_function) train_generator = tuple_generator(ModelTest.batch_size) valid_generator = tuple_generator(ModelTest.batch_size) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) num_warnings = ModelTest.steps_per_epoch * 2 * ModelTest.epochs self.assertEqual(len(w), num_warnings) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") warning_settings['batch_size'] = 'ignore' model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) self.assertEqual(len(w), 0)
def test_epoch_metrics_integration(self): model = Model(self.pytorch_network, self.optimizer, self.loss_function, epoch_metrics=[SomeEpochMetric()]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) logs = model.fit_generator(train_generator, valid_generator, epochs=1, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch) actual_value = logs[-1]['some_epoch_metric'] val_actual_value = logs[-1]['val_some_epoch_metric'] expected_value = 5 self.assertEqual(val_actual_value, expected_value) self.assertEqual(actual_value, expected_value)
def __init__(self, directory, module, *, device=None, logging=True, optimizer='sgd', loss_function=None, metrics=[], monitor_metric=None, monitor_mode=None, type=None): self.directory = directory self.logging = logging if type is not None and not type.startswith( 'classif') and not type.startswith('reg'): raise ValueError("Invalid type '%s'" % type) loss_function = self._get_loss_function(loss_function, module, type) metrics = self._get_metrics(metrics, module, type) self._set_monitor(monitor_metric, monitor_mode, type) self.model = Model(module, optimizer, loss_function, metrics=metrics) if device is not None: self.model.to(device) join_dir = lambda x: os.path.join(directory, x) self.best_checkpoint_filename = join_dir( Experiment.BEST_CHECKPOINT_FILENAME) self.best_checkpoint_tmp_filename = join_dir( Experiment.BEST_CHECKPOINT_TMP_FILENAME) self.model_checkpoint_filename = join_dir( Experiment.MODEL_CHECKPOINT_FILENAME) self.model_checkpoint_tmp_filename = join_dir( Experiment.MODEL_CHECKPOINT_TMP_FILENAME) self.optimizer_checkpoint_filename = join_dir( Experiment.OPTIMIZER_CHECKPOINT_FILENAME) self.optimizer_checkpoint_tmp_filename = join_dir( Experiment.OPTIMIZER_CHECKPOINT_TMP_FILENAME) self.log_filename = join_dir(Experiment.LOG_FILENAME) self.tensorboard_directory = join_dir(Experiment.TENSORBOARD_DIRECTORY) self.epoch_filename = join_dir(Experiment.EPOCH_FILENAME) self.epoch_tmp_filename = join_dir(Experiment.EPOCH_TMP_FILENAME) self.lr_scheduler_filename = join_dir(Experiment.LR_SCHEDULER_FILENAME) self.lr_scheduler_tmp_filename = join_dir( Experiment.LR_SCHEDULER_TMP_FILENAME) self.test_log_filename = join_dir(Experiment.TEST_LOG_FILENAME)
def setUp(self): super().setUp() torch.manual_seed(42) self.pytorch_network = DictOutputModel() self.loss_function = nn.MSELoss() self.optimizer = torch.optim.SGD(self.pytorch_network.parameters(), lr=1e-3) self.model = Model( self.pytorch_network, self.optimizer, lambda y_p, y_t: self.loss_function(y_p['out1'], y_t[ 0]) + self.loss_function(y_p['out2'], y_t[1]), batch_metrics=self.batch_metrics, epoch_metrics=self.epoch_metrics)
def test_metrics_integration(self): num_steps = 10 model = Model(self.pytorch_network, self.optimizer, self.loss_function, batch_metrics=[F.mse_loss]) train_generator = some_data_tensor_generator(ModelTest.batch_size) valid_generator = some_data_tensor_generator(ModelTest.batch_size) model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback]) generator = some_data_tensor_generator(ModelTest.batch_size) loss, mse = model.evaluate_generator(generator, steps=num_steps) self.assertEqual(type(loss), float) self.assertEqual(type(mse), float)