def train(
model: LanguageModel,
train_data: np.ndarray,
valid_data: np.ndarray,
batch_size: int,
max_epochs: int,
patience: int,
save_file: str,
):
best_valid_loss, _ = model.run_one_epoch(
get_minibatch_iterator(valid_data, batch_size, is_training=False),
training=False,
)
print(f"Initial valid loss: {best_valid_loss:.3f}.")
model.save(save_file)
best_valid_epoch = 0
train_time_start = time.time()
for epoch in range(1, max_epochs + 1):
print(f"== Epoch {epoch}")
train_loss, train_acc = model.run_one_epoch(
get_minibatch_iterator(train_data, batch_size, is_training=True),
training=True,
)
print(f" Train: Loss {train_loss:.4f}, Acc {train_acc:.3f}")
valid_loss, valid_acc = model.run_one_epoch(
get_minibatch_iterator(valid_data, batch_size, is_training=False),
training=False,
)
print(f" Valid: Loss {valid_loss:.4f}, Acc {valid_acc:.3f}")
# Save if good enough.
if valid_loss < best_valid_loss:
print(
f" (Best epoch so far, loss decreased {valid_loss:.4f} from {best_valid_loss:.4f})",
)
model.save(save_file)
print(f" (Saved model to {save_file})")
best_valid_loss = valid_loss
best_valid_epoch = epoch
elif epoch - best_valid_epoch >= patience:
total_time = time.time() - train_time_start
print(
f"Stopping training after {patience} epochs without "
f"improvement on validation loss.", )
print(
f"Training took {total_time:.0f}s. Best validation loss: {best_valid_loss:.4f}",
)
break
time_per_batch += '{} \n'.format(t1 - t0)
wps = np.round((batch_size * seq_len) / (t1 - t0))
wps_list += '{} \n'.format(wps)
progbar.add(len(X_batch),
values=[("loss", loss), ("perplexity", perp),
("words/sec", wps)])
t_epoch_end = time.time()
total_epoch = t_epoch_end - t_epoch_start
time_per_epoch += '{} \n'.format(total_epoch)
total_time_training += total_epoch
model.save(directoryCkpt)
print('save epoch stats...')
with open(directoryData + 'loss.txt', 'a') as outstream:
outstream.write(loss_list)
with open(directoryData + 'perp.txt', 'a') as outstream:
outstream.write(perp_list)
with open(directoryData + 'wps.txt', 'a') as outstream:
outstream.write(wps_list)
with open(directoryData + 'time_per_batch.txt', 'a') as outstream:
outstream.write(time_per_batch)
with open(directoryData + 'time_per_epoch.txt', 'a') as outstream:
outstream.write(time_per_epoch)
loss_list = ''
model.compile()
eval_softmax = 5
for epoch in range(num_epochs):
dataset.set_data_dir(data_dir)
dataset.set_batch_size(batch_size)
progbar = generic_utils.Progbar(dataset.token.document_count)
for X_batch, Y_batch in dataset:
t0 = time.time()
loss = model.train_on_batch(X_batch, Y_batch)
perp = np.exp(np.float32(loss))
t1 = time.time()
wps = np.round((batch_size * seq_len) / (t1 - t0))
progbar.add(len(X_batch),
values=[("loss", loss), ("perplexity", perp),
("words/sec", wps)])
model.save(save_dir)
dataset.set_data_dir(valid_data_dir)
dataset.set_batch_size(valid_batch_size)
valid_logprob = 0.
tokens = 0.
count = 0
if epoch % eval_softmax == 0:
print '\n\nEstimating validation perplexity...'
if epoch == 0:
n_valid_batches = 0
else:
progbar = generic_utils.Progbar(n_valid_batches)
for X_batch, Y_batch in dataset:
if epoch == 0:
n_valid_batches += 1
else: