class ModelTest(TestCase):
epochs = 10
steps_per_epoch = 5
batch_size = 20
evaluate_dataset_len = 107
cuda_device = int(os.environ.get('CUDA_DEVICE', 0))
def setUp(self):
torch.manual_seed(42)
self.pytorch_module = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(), lr=1e-3)
self.metrics = [some_metric_1, some_metric_2]
self.metrics_names = ['some_metric_1', 'some_metric_2']
self.metrics_values = [some_metric_1_value, some_metric_2_value]
self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=self.metrics)
self.mock_callback = MagicMock()
def test_fitting_tensor_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch}
self._test_fitting(params, logs)
def test_fitting_without_valid_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(train_generator, None, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch}
self._test_fitting(params, logs, has_valid=False)
def test_fitting_ndarray_generator(self):
train_generator = some_ndarray_generator(ModelTest.batch_size)
valid_generator = some_ndarray_generator(ModelTest.batch_size)
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': ModelTest.steps_per_epoch}
self._test_fitting(params, logs)
def test_fitting_with_data_loader(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_x = torch.rand(train_real_steps_per_epoch * train_batch_size - train_final_batch_missing_samples, 1)
train_y = torch.rand(train_real_steps_per_epoch * train_batch_size - train_final_batch_missing_samples, 1)
train_dataset = TensorDataset(train_x, train_y)
train_generator = DataLoader(train_dataset, train_batch_size)
valid_real_steps_per_epoch = 10
valid_batch_size = 15
valid_final_batch_missing_samples = 3
valid_x = torch.rand(valid_real_steps_per_epoch * valid_batch_size - valid_final_batch_missing_samples, 1)
valid_y = torch.rand(valid_real_steps_per_epoch * valid_batch_size - valid_final_batch_missing_samples, 1)
valid_dataset = TensorDataset(valid_x, valid_y)
valid_generator = DataLoader(valid_dataset, valid_batch_size)
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch}
self._test_fitting(params, logs)
def test_fitting_with_tensor(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_x = torch.rand(train_real_steps_per_epoch * train_batch_size - train_final_batch_missing_samples, 1)
train_y = torch.rand(train_real_steps_per_epoch * train_batch_size - train_final_batch_missing_samples, 1)
valid_real_steps_per_epoch = 10
valid_batch_size = train_batch_size # valid_batch_size will be the same as train_batch_size in the fit method.
valid_final_batch_missing_samples = 3
valid_x = torch.rand(valid_real_steps_per_epoch * valid_batch_size - valid_final_batch_missing_samples, 1)
valid_y = torch.rand(valid_real_steps_per_epoch * valid_batch_size - valid_final_batch_missing_samples, 1)
logs = self.model.fit(train_x, train_y, validation_x=valid_x, validation_y=valid_y, epochs=ModelTest.epochs, batch_size=train_batch_size, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch}
self._test_fitting(params, logs)
def test_fitting_with_np_array(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_x = np.random.rand(train_real_steps_per_epoch * train_batch_size - train_final_batch_missing_samples, 1).astype(np.float32)
train_y = np.random.rand(train_real_steps_per_epoch * train_batch_size - train_final_batch_missing_samples, 1).astype(np.float32)
valid_real_steps_per_epoch = 10
valid_batch_size = train_batch_size # valid_batch_size will be the same as train_batch_size in the fit method.
valid_final_batch_missing_samples = 3
valid_x = np.random.rand(valid_real_steps_per_epoch * valid_batch_size - valid_final_batch_missing_samples, 1).astype(np.float32)
valid_y = np.random.rand(valid_real_steps_per_epoch * valid_batch_size - valid_final_batch_missing_samples, 1).astype(np.float32)
logs = self.model.fit(train_x, train_y, validation_x=valid_x, validation_y=valid_y, epochs=ModelTest.epochs, batch_size=train_batch_size, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch}
self._test_fitting(params, logs)
def test_fitting_with_generator_with_len(self):
train_real_steps_per_epoch = 30
train_generator = SomeDataGeneratorWithLen(batch_size=ModelTest.batch_size, length=train_real_steps_per_epoch, num_missing_samples=7)
valid_generator = SomeDataGeneratorWithLen(batch_size=15, length=10, num_missing_samples=3)
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=None, validation_steps=None, callbacks=[self.mock_callback])
params = {'epochs': ModelTest.epochs, 'steps': train_real_steps_per_epoch}
self._test_fitting(params, logs)
def _test_fitting(self, params, logs, has_valid=True):
self.assertEqual(len(logs), params['epochs'])
train_dict = dict(zip(self.metrics_names, self.metrics_values), loss=ANY)
if has_valid:
val_metrics_names = ['val_' + metric_name for metric_name in self.metrics_names]
val_dict = dict(zip(val_metrics_names, self.metrics_values), val_loss=ANY)
log_dict = {**train_dict, **val_dict}
else:
log_dict = train_dict
for epoch, log in enumerate(logs, 1):
self.assertEqual(log, dict(log_dict, epoch=epoch))
call_list = []
call_list.append(call.on_train_begin({}))
for epoch in range(1, params['epochs']+1):
call_list.append(call.on_epoch_begin(epoch, {}))
for step in range(1, params['steps']+1):
call_list.append(call.on_batch_begin(step, {}))
call_list.append(call.on_backward_end(step))
call_list.append(call.on_batch_end(step, {'batch': step, 'size': ANY, **train_dict}))
call_list.append(call.on_epoch_end(epoch, {'epoch': epoch, **log_dict}))
call_list.append(call.on_train_end({}))
method_calls = self.mock_callback.method_calls
self.assertIn(call.set_model(self.model), method_calls[:2])
self.assertIn(call.set_params(params), method_calls[:2])
self.assertEqual(len(method_calls), len(call_list) + 2)
self.assertEqual(method_calls[2:], call_list)
def test_tensor_train_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics = self.model.train_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
def test_train_on_batch_with_pred(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics, pred_y = self.model.train_on_batch(x, y, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_train_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
loss, metrics = self.model.train_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
def test_evaluate(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, metrics = self.model.evaluate(x, y, batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
def test_evaluate_with_pred(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, metrics, pred_y = self.model.evaluate(x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32)
y = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32)
loss, metrics, pred_y = self.model.evaluate(x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
dataset = TensorDataset(x, y)
generator = DataLoader(dataset, ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
self._test_predictions_for_evaluate_and_predict_generator(pred_y)
def test_evaluate_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator, steps=num_steps, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(np.concatenate(pred_y).shape, (num_steps * ModelTest.batch_size, 1))
def test_evaluate_with_only_one_metric(self):
self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=self.metrics[:1])
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, first_metric = self.model.evaluate(x, y, batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(first_metric), float)
self.assertEqual(first_metric, some_metric_1_value)
def test_metrics_integration(self):
num_steps = 10
import torch.nn.functional as F
self.model = Model(self.pytorch_module, self.optimizer, self.loss_function, metrics=[F.mse_loss])
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, mse = self.model.evaluate_generator(generator, steps=num_steps)
self.assertEqual(type(loss), float)
self.assertEqual(type(mse), float)
def test_evaluate_with_no_metric(self):
self.model = Model(self.pytorch_module, self.optimizer, self.loss_function)
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss = self.model.evaluate(x, y, batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
def test_tensor_evaluate_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics = self.model.evaluate_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
def test_evaluate_on_batch_with_pred(self):
x = torch.rand(ModelTest.batch_size, 1)
y = torch.rand(ModelTest.batch_size, 1)
loss, metrics, pred_y = self.model.evaluate_on_batch(x, y, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_evaluate_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
y = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
loss, metrics = self.model.evaluate_on_batch(x, y)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(), [some_metric_1_value, some_metric_2_value])
def test_predict(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len, 1).astype(np.float32)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
generator = DataLoader(x, ModelTest.batch_size)
pred_y = self.model.predict_generator(generator)
self._test_predictions_for_evaluate_and_predict_generator(pred_y)
def test_predict_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator, steps=num_steps)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(np.concatenate(pred_y).shape, (num_steps * ModelTest.batch_size, 1))
def _test_predictions_for_evaluate_and_predict_generator(self, pred_y):
self.assertEqual(type(pred_y), list)
remaning_example = ModelTest.evaluate_dataset_len
cur_batch_size = ModelTest.batch_size
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
if remaning_example < ModelTest.batch_size:
cur_batch_size = remaning_example
remaning_example = 0
else:
remaning_example -= ModelTest.batch_size
self.assertEqual(pred.shape, (cur_batch_size, 1))
self.assertEqual(np.concatenate(pred_y).shape, (ModelTest.evaluate_dataset_len, 1))
def test_tensor_predict_on_batch(self):
x = torch.rand(ModelTest.batch_size, 1)
pred_y = self.model.predict_on_batch(x)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
def test_ndarray_predict_on_batch(self):
x = np.random.rand(ModelTest.batch_size, 1).astype(np.float32)
pred_y = self.model.predict_on_batch(x)
self.assertEqual(pred_y.shape, (ModelTest.batch_size, 1))
@skipIf(not torch.cuda.is_available(), "no gpu available")
def test_cpu_cuda(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda()
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
# The context manager is also used here because of this bug: https://github.com/pytorch/pytorch/issues/7320
with torch.cuda.device(ModelTest.cuda_device):
self.model.cuda(ModelTest.cuda_device)
self._test_device(torch.device('cuda:' + str(ModelTest.cuda_device)))
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
self.model.cpu()
self._test_device(torch.device('cpu'))
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
self.model.to(torch.device('cuda:' + str(ModelTest.cuda_device)))
self._test_device(torch.device('cuda:' + str(ModelTest.cuda_device)))
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
self.model.to(torch.device('cpu'))
self._test_device(torch.device('cpu'))
logs = self.model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch, callbacks=[self.mock_callback])
def _test_device(self, device):
for p in self.pytorch_module.parameters():
self.assertEqual(p.device, device)
def test_disable_batch_size_warning(self):
import warnings
def tuple_generator(batch_size):
while True:
x1 = torch.rand(batch_size, 1)
x2 = torch.rand(batch_size, 1)
y1 = torch.rand(batch_size, 1)
y2 = torch.rand(batch_size, 1)
yield (x1, x2), (y1, y2)
class TupleModule(nn.Module):
def __init__(self):
super().__init__()
self.l1 = nn.Linear(1, 1)
self.l2 = nn.Linear(1, 1)
def forward(self, x):
x1, x2 = x
return self.l1(x1), self.l2(x2)
def loss_function(y_pred, y_true):
return F.mse_loss(y_pred[0], y_true[0]) + F.mse_loss(y_pred[1], y_true[1])
pytorch_module = TupleModule()
optimizer = torch.optim.SGD(pytorch_module.parameters(), lr=1e-3)
model = Model(pytorch_module, optimizer, loss_function)
train_generator = tuple_generator(ModelTest.batch_size)
valid_generator = tuple_generator(ModelTest.batch_size)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch)
num_warnings = ModelTest.steps_per_epoch * 2 * ModelTest.epochs
self.assertEqual(len(w), num_warnings)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
warning_settings['batch_size'] = 'ignore'
model.fit_generator(train_generator, valid_generator, epochs=ModelTest.epochs, steps_per_epoch=ModelTest.steps_per_epoch, validation_steps=ModelTest.steps_per_epoch)
self.assertEqual(len(w), 0)
def launch_train(embeddings, model_name, device, debug):
if debug:
epochs = 1
else:
epochs = 40
train_sentences, train_tags = parse_conll_file('./data/conll/train.txt')
valid_sentences, valid_tags = parse_conll_file('./data/conll/valid.txt')
test_sentences, test_tags = parse_conll_file('./data/conll/test.txt')
words_vocab, words_to_idx = make_vocab_and_idx(train_sentences +
valid_sentences +
test_sentences)
tags_vocab, tags_to_idx = make_vocab_and_idx(train_tags + valid_tags +
test_tags)
train_sentences = [[words_to_idx[word] for word in sentence]
for sentence in train_sentences]
train_tags = [[tags_to_idx[word] for word in sentence]
for sentence in train_tags]
valid_sentences = [[words_to_idx[word] for word in sentence]
for sentence in valid_sentences]
valid_tags = [[tags_to_idx[word] for word in sentence]
for sentence in valid_tags]
test_sentences = [[words_to_idx[word] for word in sentence]
for sentence in test_sentences]
test_tags = [[tags_to_idx[word] for word in sentence]
for sentence in test_tags]
train_dataset = list(zip(train_sentences, train_tags))
valid_dataset = list(zip(valid_sentences, valid_tags))
test_dataset = list(zip(test_sentences, test_tags))
def cuda_collate(samples):
words_tensor, labels_tensor = collate_examples(samples)
return words_tensor.cuda(), labels_tensor.cuda()
use_gpu = torch.cuda.is_available()
if use_gpu:
cuda_device = device
torch.cuda.set_device(cuda_device)
logging.info('Using GPU')
if use_gpu:
collate_fn = cuda_collate
else:
collate_fn = collate_examples
train_loader = DataLoader(train_dataset,
batch_size=32,
shuffle=True,
collate_fn=collate_fn)
valid_loader = DataLoader(valid_dataset,
batch_size=32,
shuffle=True,
collate_fn=collate_fn)
test_loader = DataLoader(test_dataset,
batch_size=32,
shuffle=True,
collate_fn=collate_fn)
net = LSTMTagger(100, 50, words_to_idx, len(tags_to_idx), use_gpu)
net.load_words_embeddings(embeddings)
if use_gpu:
net.cuda()
lrscheduler = ReduceLROnPlateau(patience=2)
early_stopping = EarlyStopping(patience=5)
model_path = './models/'
checkpoint = ModelCheckpoint(model_path + 'ner_' + model_name + '.torch',
save_best_only=True,
restore_best=True,
temporary_filename=model_path + 'tmp_ner_' +
model_name + '.torch',
verbose=True)
csv_logger = CSVLogger('./train_logs/ner_{}.csv'.format(model_name))
model = Model(net,
Adam(net.parameters(), lr=0.001),
sequence_cross_entropy,
metrics=[f1])
model.fit_generator(
train_loader,
valid_loader,
epochs=epochs,
callbacks=[lrscheduler, checkpoint, early_stopping, csv_logger])
loss, metric = model.evaluate_generator(test_loader)
logging.info("Test loss: {}".format(loss))
logging.info("Test metric: {}".format(metric))
class ModelTest(TestCase):
epochs = 10
steps_per_epoch = 5
batch_size = 20
evaluate_dataset_len = 107
def setUp(self):
torch.manual_seed(42)
self.pytorch_module = nn.Linear(1, 1)
self.loss_function = nn.MSELoss()
self.optimizer = torch.optim.SGD(self.pytorch_module.parameters(),
lr=1e-3)
self.metrics = [some_metric_1, some_metric_2]
self.metrics_names = ['some_metric_1', 'some_metric_2']
self.metrics_values = [some_metric_1_value, some_metric_2_value]
self.model = Model(self.pytorch_module,
self.optimizer,
self.loss_function,
metrics=self.metrics)
self.mock_callback = MagicMock()
def test_fitting_tensor_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
valid_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_without_valid_generator(self):
train_generator = some_data_tensor_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
None,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_fitting(params, logs, has_valid=False)
def test_fitting_variable_generator(self):
train_generator = some_data_variable_generator(ModelTest.batch_size)
valid_generator = some_data_variable_generator(ModelTest.batch_size)
logs = self.model.fit_generator(
train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=ModelTest.steps_per_epoch,
validation_steps=ModelTest.steps_per_epoch,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': ModelTest.steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_data_loader(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_x = torch.rand(
train_real_steps_per_epoch * train_batch_size -
train_final_batch_missing_samples, 1)
train_y = torch.rand(
train_real_steps_per_epoch * train_batch_size -
train_final_batch_missing_samples, 1)
train_dataset = TensorDataset(train_x, train_y)
train_generator = DataLoader(train_dataset, train_batch_size)
valid_real_steps_per_epoch = 10
valid_batch_size = 15
valid_final_batch_missing_samples = 3
valid_x = torch.rand(
valid_real_steps_per_epoch * valid_batch_size -
valid_final_batch_missing_samples, 1)
valid_y = torch.rand(
valid_real_steps_per_epoch * valid_batch_size -
valid_final_batch_missing_samples, 1)
valid_dataset = TensorDataset(valid_x, valid_y)
valid_generator = DataLoader(valid_dataset, valid_batch_size)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_tensor(self):
train_real_steps_per_epoch = 30
train_batch_size = ModelTest.batch_size
train_final_batch_missing_samples = 7
train_x = torch.rand(
train_real_steps_per_epoch * train_batch_size -
train_final_batch_missing_samples, 1)
train_y = torch.rand(
train_real_steps_per_epoch * train_batch_size -
train_final_batch_missing_samples, 1)
valid_real_steps_per_epoch = 10
valid_batch_size = train_batch_size # valid_batch_size will be the same as train_batch_size in the fit method.
valid_final_batch_missing_samples = 3
valid_x = torch.rand(
valid_real_steps_per_epoch * valid_batch_size -
valid_final_batch_missing_samples, 1)
valid_y = torch.rand(
valid_real_steps_per_epoch * valid_batch_size -
valid_final_batch_missing_samples, 1)
logs = self.model.fit(train_x,
train_y,
validation_x=valid_x,
validation_y=valid_y,
epochs=ModelTest.epochs,
batch_size=train_batch_size,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def test_fitting_with_generator_with_len(self):
train_real_steps_per_epoch = 30
train_generator = SomeDataGeneratorWithLen(
batch_size=ModelTest.batch_size,
length=train_real_steps_per_epoch,
num_missing_samples=7)
valid_generator = SomeDataGeneratorWithLen(batch_size=15,
length=10,
num_missing_samples=3)
logs = self.model.fit_generator(train_generator,
valid_generator,
epochs=ModelTest.epochs,
steps_per_epoch=None,
validation_steps=None,
callbacks=[self.mock_callback])
params = {
'epochs': ModelTest.epochs,
'steps': train_real_steps_per_epoch
}
self._test_fitting(params, logs)
def _test_fitting(self, params, logs, has_valid=True):
self.assertEqual(len(logs), params['epochs'])
train_dict = dict(zip(self.metrics_names, self.metrics_values),
loss=ANY)
if has_valid:
val_metrics_names = [
'val_' + metric_name for metric_name in self.metrics_names
]
val_dict = dict(zip(val_metrics_names, self.metrics_values),
val_loss=ANY)
log_dict = {**train_dict, **val_dict}
else:
log_dict = train_dict
for epoch, log in enumerate(logs, 1):
self.assertEqual(log, dict(log_dict, epoch=epoch))
call_list = []
call_list.append(call.on_train_begin({}))
for epoch in range(1, params['epochs'] + 1):
call_list.append(call.on_epoch_begin(epoch, {}))
for step in range(1, params['steps'] + 1):
call_list.append(call.on_batch_begin(step, {}))
call_list.append(
call.on_batch_end(step, {
'batch': step,
'size': ANY,
**train_dict
}))
call_list.append(
call.on_epoch_end(epoch, {
'epoch': epoch,
**log_dict
}))
call_list.append(call.on_train_end({}))
method_calls = self.mock_callback.method_calls
self.assertIn(call.set_model(self.model), method_calls[:2])
self.assertIn(call.set_params(params), method_calls[:2])
self.assertEqual(len(method_calls), len(call_list) + 2)
self.assertEqual(method_calls[2:], call_list)
def test_evaluate(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, metrics = self.model.evaluate(x,
y,
batch_size=ModelTest.batch_size)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
def test_evaluate_with_pred(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
loss, metrics, pred_y = self.model.evaluate(
x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
y = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
loss, metrics, pred_y = self.model.evaluate(
x, y, batch_size=ModelTest.batch_size, return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_evaluate_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
y = torch.rand(ModelTest.evaluate_dataset_len, 1)
dataset = TensorDataset(x, y)
generator = DataLoader(dataset, ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
self._test_predictions_for_evaluate_and_predict_generator(pred_y)
def test_evaluate_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
loss, metrics, pred_y = self.model.evaluate_generator(generator,
steps=num_steps,
return_pred=True)
self.assertEqual(type(loss), float)
self.assertEqual(type(metrics), np.ndarray)
self.assertEqual(metrics.tolist(),
[some_metric_1_value, some_metric_2_value])
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(
np.concatenate(pred_y).shape,
(num_steps * ModelTest.batch_size, 1))
def test_predict(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_with_np_array(self):
x = np.random.rand(ModelTest.evaluate_dataset_len,
1).astype(np.float32)
pred_y = self.model.predict(x, batch_size=ModelTest.batch_size)
self.assertEqual(pred_y.shape, (ModelTest.evaluate_dataset_len, 1))
def test_predict_data_loader(self):
x = torch.rand(ModelTest.evaluate_dataset_len, 1)
generator = DataLoader(x, ModelTest.batch_size)
pred_y = self.model.predict_generator(generator)
self._test_predictions_for_evaluate_and_predict_generator(pred_y)
def test_predict_generator(self):
num_steps = 10
generator = some_data_tensor_generator(ModelTest.batch_size)
generator = (x for x, _ in generator)
pred_y = self.model.predict_generator(generator, steps=num_steps)
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
self.assertEqual(pred.shape, (ModelTest.batch_size, 1))
self.assertEqual(
np.concatenate(pred_y).shape,
(num_steps * ModelTest.batch_size, 1))
def _test_predictions_for_evaluate_and_predict_generator(self, pred_y):
self.assertEqual(type(pred_y), list)
remaning_example = ModelTest.evaluate_dataset_len
cur_batch_size = ModelTest.batch_size
for pred in pred_y:
self.assertEqual(type(pred), np.ndarray)
if remaning_example < ModelTest.batch_size:
cur_batch_size = remaning_example
remaning_example = 0
else:
remaning_example -= ModelTest.batch_size
self.assertEqual(pred.shape, (cur_batch_size, 1))
self.assertEqual(
np.concatenate(pred_y).shape, (ModelTest.evaluate_dataset_len, 1))
class Experiment:
BEST_CHECKPOINT_FILENAME = 'checkpoint_epoch_{epoch}.ckpt'
BEST_CHECKPOINT_TMP_FILENAME = 'checkpoint_epoch.tmp.ckpt'
MODEL_CHECKPOINT_FILENAME = 'checkpoint.ckpt'
MODEL_CHECKPOINT_TMP_FILENAME = 'checkpoint.tmp.ckpt'
OPTIMIZER_CHECKPOINT_FILENAME = 'checkpoint.optim'
OPTIMIZER_CHECKPOINT_TMP_FILENAME = 'checkpoint.tmp.optim'
LOG_FILENAME = 'log.tsv'
TENSORBOARD_DIRECTORY = 'tensorboard'
EPOCH_FILENAME = 'last.epoch'
EPOCH_TMP_FILENAME = 'last.tmp.epoch'
LR_SCHEDULER_FILENAME = 'lr_sched_%d.lrsched'
LR_SCHEDULER_TMP_FILENAME = 'lr_sched_%d.tmp.lrsched'
TEST_LOG_FILENAME = 'test_log.tsv'
def __init__(self,
directory,
module,
*,
device=None,
logging=True,
optimizer='sgd',
loss_function=None,
metrics=[],
monitor_metric=None,
monitor_mode=None,
type=None):
self.directory = directory
self.logging = logging
if type is not None and not type.startswith(
'classif') and not type.startswith('reg'):
raise ValueError("Invalid type '%s'" % type)
loss_function = self._get_loss_function(loss_function, module, type)
metrics = self._get_metrics(metrics, module, type)
self._set_monitor(monitor_metric, monitor_mode, type)
self.model = Model(module, optimizer, loss_function, metrics=metrics)
if device is not None:
self.model.to(device)
join_dir = lambda x: os.path.join(directory, x)
self.best_checkpoint_filename = join_dir(
Experiment.BEST_CHECKPOINT_FILENAME)
self.best_checkpoint_tmp_filename = join_dir(
Experiment.BEST_CHECKPOINT_TMP_FILENAME)
self.model_checkpoint_filename = join_dir(
Experiment.MODEL_CHECKPOINT_FILENAME)
self.model_checkpoint_tmp_filename = join_dir(
Experiment.MODEL_CHECKPOINT_TMP_FILENAME)
self.optimizer_checkpoint_filename = join_dir(
Experiment.OPTIMIZER_CHECKPOINT_FILENAME)
self.optimizer_checkpoint_tmp_filename = join_dir(
Experiment.OPTIMIZER_CHECKPOINT_TMP_FILENAME)
self.log_filename = join_dir(Experiment.LOG_FILENAME)
self.tensorboard_directory = join_dir(Experiment.TENSORBOARD_DIRECTORY)
self.epoch_filename = join_dir(Experiment.EPOCH_FILENAME)
self.epoch_tmp_filename = join_dir(Experiment.EPOCH_TMP_FILENAME)
self.lr_scheduler_filename = join_dir(Experiment.LR_SCHEDULER_FILENAME)
self.lr_scheduler_tmp_filename = join_dir(
Experiment.LR_SCHEDULER_TMP_FILENAME)
self.test_log_filename = join_dir(Experiment.TEST_LOG_FILENAME)
def _get_loss_function(self, loss_function, module, type):
if loss_function is None:
if hasattr(module, 'loss_function'):
return module.loss_function
elif type is not None:
if type.startswith('classif'):
return 'cross_entropy'
elif type.startswith('reg'):
return 'mse'
return loss_function
def _get_metrics(self, metrics, module, type):
if metrics is None or len(metrics) == 0:
if hasattr(module, 'metrics'):
return module.metrics
elif type is not None and type.startswith('classif'):
return ['accuracy']
return metrics
def _set_monitor(self, monitor_metric, monitor_mode, type):
if monitor_mode is not None and monitor_mode not in ['min', 'max']:
raise ValueError("Invalid mode '%s'" % monitor_mode)
self.monitor_metric = 'val_loss'
self.monitor_mode = 'min'
if monitor_metric is not None:
self.monitor_metric = monitor_metric
if monitor_mode is not None:
self.monitor_mode = monitor_mode
elif type is not None and type.startswith('classif'):
self.monitor_metric = 'val_acc'
self.monitor_mode = 'max'
def get_best_epoch_stats(self):
if pd is None:
warnings.warn("pandas needs to be installed to use this function.")
history = pd.read_csv(self.log_filename, sep='\t')
if self.monitor_mode == 'min':
best_epoch_index = history[self.monitor_metric].idxmin()
else:
best_epoch_index = history[self.monitor_metric].idxmax()
return history.iloc[best_epoch_index:best_epoch_index + 1]
def _warn_missing_file(self, filename):
warnings.warn("Missing checkpoint: %s." % filename)
def _load_epoch_state(self, lr_schedulers):
# pylint: disable=broad-except
initial_epoch = 1
if os.path.isfile(self.epoch_filename):
try:
with open(self.epoch_filename, 'r') as f:
initial_epoch = int(f.read()) + 1
except Exception as e:
print(e)
if os.path.isfile(self.model_checkpoint_filename):
try:
print("Loading weights from %s and starting at epoch %d." %
(self.model_checkpoint_filename, initial_epoch))
self.model.load_weights(self.model_checkpoint_filename)
except Exception as e:
print(e)
else:
self._warn_missing_file(self.model_checkpoint_filename)
if os.path.isfile(self.optimizer_checkpoint_filename):
try:
print(
"Loading optimizer state from %s and starting at epoch %d."
% (self.optimizer_checkpoint_filename, initial_epoch))
self.model.load_optimizer_state(
self.optimizer_checkpoint_filename)
except Exception as e:
print(e)
else:
self._warn_missing_file(self.optimizer_checkpoint_filename)
for i, lr_scheduler in enumerate(lr_schedulers):
filename = self.lr_scheduler_filename % i
if os.path.isfile(filename):
try:
print(
"Loading LR scheduler state from %s and starting at epoch %d."
% (filename, initial_epoch))
lr_scheduler.load_state(filename)
except Exception as e:
print(e)
else:
self._warn_missing_file(filename)
return initial_epoch
def _init_model_restoring_callbacks(self, initial_epoch, save_every_epoch):
callbacks = []
best_checkpoint = ModelCheckpoint(
self.best_checkpoint_filename,
monitor=self.monitor_metric,
mode=self.monitor_mode,
save_best_only=not save_every_epoch,
restore_best=not save_every_epoch,
verbose=not save_every_epoch,
temporary_filename=self.best_checkpoint_tmp_filename)
callbacks.append(best_checkpoint)
if save_every_epoch:
best_restore = BestModelRestore(monitor=self.monitor_metric,
mode=self.monitor_mode,
verbose=True)
callbacks.append(best_restore)
if initial_epoch > 1:
# We set the current best metric score in the ModelCheckpoint so that
# it does not save checkpoint it would not have saved if the
# optimization was not stopped.
best_epoch_stats = self.get_best_epoch_stats()
best_epoch = best_epoch_stats['epoch'].item()
best_filename = self.best_checkpoint_filename.format(
epoch=best_epoch)
if not save_every_epoch:
best_checkpoint.best_filename = best_filename
best_checkpoint.current_best = best_epoch_stats[
self.monitor_metric].item()
else:
best_restore.best_weights = torch.load(best_filename,
map_location='cpu')
best_restore.current_best = best_epoch_stats[
self.monitor_metric].item()
return callbacks
def _init_tensorboard_callbacks(self, disable_tensorboard):
tensorboard_writer = None
callbacks = []
if not disable_tensorboard:
if SummaryWriter is None:
warnings.warn(
"tensorboardX does not seem to be installed. "
"To remove this warning, set the 'disable_tensorboard' "
"flag to True.")
else:
tensorboard_writer = SummaryWriter(self.tensorboard_directory)
callbacks += [TensorBoardLogger(tensorboard_writer)]
return tensorboard_writer, callbacks
def _init_lr_scheduler_callbacks(self, lr_schedulers):
callbacks = []
if self.logging:
for i, lr_scheduler in enumerate(lr_schedulers):
filename = self.lr_scheduler_filename % i
tmp_filename = self.lr_scheduler_tmp_filename % i
callbacks += [
LRSchedulerCheckpoint(lr_scheduler,
filename,
verbose=False,
temporary_filename=tmp_filename)
]
else:
callbacks += lr_schedulers
callbacks += [
BestModelRestore(monitor=self.monitor_metric,
mode=self.monitor_mode,
verbose=True)
]
return callbacks
def train(self,
train_loader,
valid_loader=None,
*,
callbacks=[],
lr_schedulers=[],
save_every_epoch=False,
disable_tensorboard=False,
epochs=1000,
steps_per_epoch=None,
validation_steps=None,
seed=42):
if seed is not None:
# Make training deterministic.
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
# Copy callback list.
callbacks = list(callbacks)
tensorboard_writer = None
initial_epoch = 1
if self.logging:
if not os.path.exists(self.directory):
os.makedirs(self.directory)
# Restarting optimization if needed.
initial_epoch = self._load_epoch_state(lr_schedulers)
callbacks += [
CSVLogger(self.log_filename,
separator='\t',
append=initial_epoch != 1)
]
callbacks += self._init_model_restoring_callbacks(
initial_epoch, save_every_epoch)
callbacks += [
ModelCheckpoint(
self.model_checkpoint_filename,
verbose=False,
temporary_filename=self.model_checkpoint_tmp_filename)
]
callbacks += [
OptimizerCheckpoint(
self.optimizer_checkpoint_filename,
verbose=False,
temporary_filename=self.optimizer_checkpoint_tmp_filename)
]
# We save the last epoch number after the end of the epoch so that the
# _load_epoch_state() knows which epoch to restart the optimization.
callbacks += [
PeriodicSaveLambda(
lambda fd, epoch, logs: print(epoch, file=fd),
self.epoch_filename,
temporary_filename=self.epoch_tmp_filename,
open_mode='w')
]
tensorboard_writer, cb_list = self._init_tensorboard_callbacks(
disable_tensorboard)
callbacks += cb_list
# This method returns callbacks that checkpoints the LR scheduler if logging is enabled.
# Otherwise, it just returns the list of LR schedulers with a BestModelRestore callback.
callbacks += self._init_lr_scheduler_callbacks(lr_schedulers)
try:
self.model.fit_generator(train_loader,
valid_loader,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps,
initial_epoch=initial_epoch,
callbacks=callbacks)
finally:
if tensorboard_writer is not None:
tensorboard_writer.close()
def load_best_checkpoint(self, *, verbose=False):
best_epoch_stats = self.get_best_epoch_stats()
best_epoch = best_epoch_stats['epoch'].item()
if verbose:
metrics_str = ', '.join(
'%s: %g' % (metric_name, best_epoch_stats[metric_name].item())
for metric_name in best_epoch_stats.columns[2:])
print("Found best checkpoint at epoch: {}".format(best_epoch))
print(metrics_str)
self.load_checkpoint(best_epoch)
return best_epoch_stats
def load_checkpoint(self, epoch):
ckpt_filename = self.best_checkpoint_filename.format(epoch=epoch)
self.model.load_weights(ckpt_filename)
def load_last_checkpoint(self):
self.model.load_weights(self.model_checkpoint_filename)
def test(self,
test_loader,
*,
steps=None,
load_best_checkpoint=True,
load_last_checkpoint=False,
seed=42):
if seed is not None:
# Make training deterministic.
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
best_epoch_stats = None
if load_best_checkpoint:
best_epoch_stats = self.load_best_checkpoint(verbose=True)
elif load_last_checkpoint:
best_epoch_stats = self.load_last_checkpoint()
test_loss, test_metrics = self.model.evaluate_generator(test_loader,
steps=steps)
if not isinstance(test_metrics, np.ndarray):
test_metrics = np.array([test_metrics])
test_metrics_names = ['test_loss'] + \
['test_' + metric_name for metric_name in self.model.metrics_names]
test_metrics_values = np.concatenate(([test_loss], test_metrics))
test_metrics_str = ', '.join(
'%s: %g' % (col, val)
for col, val in zip(test_metrics_names, test_metrics_values))
print("On best model: %s" % test_metrics_str)
if self.logging:
test_stats = pd.DataFrame([test_metrics_values],
columns=test_metrics_names)
if best_epoch_stats is not None:
best_epoch_stats = best_epoch_stats.reset_index(drop=True)
test_stats = best_epoch_stats.join(test_stats)
test_stats.to_csv(self.test_log_filename, sep='\t', index=False)
def train(embeddings, model_name='vanilla'):
train_sentences, train_tags = parse_20newsgroup_file('./data/20newsgroup/train.pickle')
valid_sentences, valid_tags = parse_20newsgroup_file('./data/20newsgroup/dev.pickle')
test_sentences, test_tags = parse_20newsgroup_file('./data/20newsgroup/test.pickle')
labels = set(train_tags + valid_tags + test_tags)
words_vocab, words_to_idx = make_vocab_and_idx(train_sentences + valid_sentences + test_sentences)
tags_to_idx = {v: v for v in labels}
train_sentences = [[words_to_idx[word] for word in sentence] for sentence in train_sentences]
valid_sentences = [[words_to_idx[word] for word in sentence] for sentence in valid_sentences]
test_sentences = [[words_to_idx[word] for word in sentence] for sentence in test_sentences]
train_dataset = list(zip(train_sentences, train_tags))
valid_dataset = list(zip(valid_sentences, valid_tags))
test_dataset = list(zip(test_sentences, test_tags))
train_loader = DataLoader(
train_dataset,
batch_size=32,
shuffle=True,
collate_fn=collate_examples
)
valid_loader = DataLoader(
valid_dataset,
batch_size=32,
shuffle=True,
collate_fn=collate_examples
)
test_loader = DataLoader(
test_dataset,
batch_size=32,
shuffle=True,
collate_fn=collate_examples
)
net = LSTMClassifier(
100,
50,
words_to_idx,
len(tags_to_idx)
)
net.load_words_embeddings(embeddings)
lrscheduler = ReduceLROnPlateau(patience=5)
early_stopping = EarlyStopping(patience=10)
model_path = './models/'
checkpoint = ModelCheckpoint(model_path+'newsclassif_'+model_name+'.torch',
save_best_only=True,
restore_best=True,
temporary_filename=model_path+'tmp_newsclassif_'+model_name+'.torch')
csv_logger = CSVLogger('./train_logs/newsclassif_{}.csv'.format(model_name))
loss = CrossEntropyLoss()
model = Model(net, Adam(net.parameters(), lr=0.001), loss, metrics=[acc])
model.fit_generator(train_loader, valid_loader, epochs=40, callbacks=[lrscheduler, checkpoint, early_stopping, csv_logger])
loss, metric = model.evaluate_generator(test_loader)
logging.info("Test loss: {}".format(loss))
logging.info("Test metric: {}".format(metric))
class GroupSparseNN:
def __init__(self,
input_dim,
output_dim,
loss_function=None,
activation_function='ReLu',
layers_sizes=[],
lr=0.001):
super(GroupSparseNN, self).__init__()
self.lr = lr
self.fc_network = FullyConnectedNN(input_dim, output_dim,
activation_function, layers_sizes)
if torch.cuda.is_available():
self.fc_network.cuda()
if loss_function is None:
self.loss = nn.MultiLabelSoftMarginLoss()
else:
self.loss = loss_function
optimizer = optim.Adam(self.fc_network.parameters(), lr=self.lr)
self.model = Model(self.fc_network, optimizer,
self.loss) # Pytoune Encapsulation
def fit(self,
x_train,
y_train,
x_valid,
y_valid,
n_epochs=100,
batch_size=32,
log_filename=None,
checkpoint_filename=None,
with_early_stopping=True):
"""
:param x_train: training set examples
:param y_train: training set labels
:param x_valid: testing set examples
:param y_valid: testing set labels
:param n_epochs: int, number of epoch default value 100
:param batch_size: int, size of the batch default value 32, must be multiple of 2
:param log_filename: optional, to output the training informations
:param checkpoint_filename: optional, to save the model
:param with_early_stopping: to activate the early stopping or not
:return: self, the model
"""
callbacks = []
if with_early_stopping:
early_stopping = EarlyStopping(monitor='val_loss',
patience=3,
verbose=0)
callbacks += [early_stopping]
reduce_lr = ReduceLROnPlateau(monitor='loss',
patience=2,
factor=1 / 10,
min_lr=1e-6)
best_model_restore = BestModelRestore()
callbacks += [reduce_lr, best_model_restore]
if log_filename:
logger = CSVLogger(log_filename,
batch_granularity=False,
separator='\t')
callbacks += [logger]
if checkpoint_filename:
checkpointer = ModelCheckpoint(checkpoint_filename,
monitor='val_loss',
save_best_only=True)
callbacks += [checkpointer]
# self.model.fit(x_train, y_train, x_valid, y_valid,
# batch_size=batch_size, epochs=n_epochs,
# callbacks=callbacks)
nb_steps_train, nb_step_valid = int(
len(x_train) / batch_size), int(len(x_valid) / batch_size)
self.model.fit_generator(
generator(x_train, y_train, batch_size),
steps_per_epoch=nb_steps_train,
valid_generator=generator(x_valid, y_valid, batch_size),
validation_steps=nb_step_valid,
epochs=n_epochs,
callbacks=callbacks,
)
return self
def evaluate(self, x, y, batch_size=16, metrics=None):
if metrics is None:
metrics = [hamming_loss]
valid_gen = generator(x, y, batch=batch_size)
nsteps = ceil(len(x) / (batch_size * 1.0))
_, y_pred = self.model.evaluate_generator(valid_gen,
steps=nsteps,
return_pred=True)
y_pred = np.concatenate(y_pred, axis=0)
if torch.cuda.is_available():
y_true = y.cpu().numpy()
else:
y_true = y.numpy()
res = {
metric.__name__: metric(y_true, y_pred.round())
for metric in metrics
}
print('The metrics of the model is: {}'.format(res))
return res
def load(self, checkpoint_filename):
self.model.load_weights(checkpoint_filename)
