def validate_batches(voc, val_pairs, encoder, decoder):
encoder.eval()
decoder.eval()
random.shuffle(val_pairs)
batches = [
val_pairs[i:i + config.batch_size]
for i in range(0, len(val_pairs), config.batch_size)
]
val_batches = [
prepare_data.batch2TrainData(voc, batches[i])
for i in range(len(batches))
]
val_loss = 0
for iteration in range(1, config.n_val_iteration + 1):
val_batch = val_batches[iteration - 1]
input_variable, lengths, target_variable, mask, max_target_len = val_batch
loss = validate(input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, config.batch_size)
val_loss += loss
return val_loss / config.n_val_iteration
def train_iters(voc, pairs, encoder, decoder, encoder_optimizer,
decoder_optimizer, embedding, save_dir, checkpoint, epoch):
encoder.train()
decoder.train()
random.shuffle(pairs)
batches = [
pairs[i:i + config.batch_size]
for i in range(0, len(pairs), config.batch_size)
]
training_batches = [
prepare_data.batch2TrainData(voc, batches[i])
for i in range(len(batches))
]
start_iteration = 1
print_loss = 0
epoch_loss = 0
if config.loadFilename:
start_iteration = checkpoint['iteration'] + 1
for iteration in range(start_iteration, config.n_iteration + 1):
s = time.time()
training_batch = training_batches[iteration - 1]
input_variable, lengths, target_variable, mask, max_target_len = training_batch
loss = train(input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, encoder_optimizer,
decoder_optimizer, config.batch_size, config.clip)
print_loss += loss
epoch_loss += loss
config.train_time += time.time() - s
if iteration % config.print_every == 0:
print_loss_avg = print_loss / config.print_every
print(
"Epoch: {} Iteration: {}; Percent complete: {:.1f}%; Average loss: {:.4f}; Train time: {}"
.format(epoch, iteration, iteration / config.n_iteration * 100,
print_loss_avg, config.train_time))
print_loss = 0
if iteration % config.save_every == 0:
directory = os.path.join(
save_dir, config.model_name, config.corpus_name,
'{}-{}_{}'.format(config.encoder_n_layers,
config.decoder_n_layers, config.hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iteration,
'epoch': epoch,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'voc_dict': voc.__dict__,
'embedding': embedding.state_dict()
},
os.path.join(
directory, '{}_{}_{}.tar'.format(epoch, iteration,
'checkpoint')))
return epoch_loss / config.n_iteration
def trainIters(device,
pairs,
input_lang,
output_lang,
encoder,
decoder,
batch_size,
n_iters,
print_every=250,
plot_every=250,
save_every=2000,
learning_rate=0.01,
save_dir='checkpoints'):
start = time.time()
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
training_pairs = [
tensorsFromPair(random.choice(pairs), input_lang, output_lang, device)
for i in range(n_iters)
]
training_batches = [
batch2TrainData(input_lang, output_lang,
[random.choice(pairs) for _ in range(batch_size)])
for _ in range(n_iters)
]
for iter in range(1, n_iters + 1):
training_batch = training_batches[iter - 1]
# Extract fields from batch
input_variable, lengths, target_variable, mask, max_target_len = training_batch
loss = train(device, input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, encoder_optimizer,
decoder_optimizer, batch_size)
print_loss_total += loss
plot_loss_total += loss
if iter % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('%s (%d %d%%) %.4f' %
(timeSince(start, iter / n_iters), iter,
iter / n_iters * 100, print_loss_avg))
if iter % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
# Save checkpoint
if iter % save_every == 0:
directory = save_dir
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iter,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'input_lang': input_lang.__dict__,
'output_lang': output_lang.__dict__
},
os.path.join(directory, '{}_{}.tar'.format(iter,
'checkpoint')))
showPlot(plot_losses)
def trainIters(model_name, voc, train_pairs, dev_pairs, encoder, decoder,
encoder_optimizer, decoder_optimizer, embedding,
encoder_n_layers, decoder_n_layers, save_dir, n_iteration,
batch_size, print_every, save_every, evaluate_every, clip,
corpus_name, loadFilename, searcher):
# Load batches for each iteration
training_batches = [
batch2TrainData(
voc, [random.choice(train_pairs) for _ in range(batch_size)])
for _ in range(n_iteration)
]
# Initializations
print('Initializing ...')
start_iteration = 1
print_loss = 0
if loadFilename:
checkpoint = torch.load(loadFilename)
start_iteration = checkpoint['iteration'] + 1
# Training loop
print("Training...")
for iteration in range(start_iteration, n_iteration + 1):
training_batch = training_batches[iteration - 1]
# Extract fields from batch
input_variable, lengths, target_variable, mask, max_target_len, answer_mask = training_batch
#append one to lengths because of extra answer bit at embedding
#ones = torch.from_numpy(np.ones(batch_size))
#ones = ones.long()
#lengths = lengths + ones
# Run a training iteration with batch
loss = train(voc, input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, embedding,
encoder_optimizer, decoder_optimizer, batch_size, clip,
answer_mask)
print_loss += loss
# Print progress
if iteration % print_every == 0:
print_loss_avg = print_loss / print_every
print(
"Iteration: {}; Percent complete: {:.1f}%; Average loss: {:.4f}"
.format(iteration, iteration / n_iteration * 100,
print_loss_avg))
print_loss = 0
# Save checkpoint
if (iteration % save_every == 0):
directory = os.path.join(
save_dir, model_name, corpus_name,
'{}-{}_{}-{}'.format(encoder_n_layers, decoder_n_layers,
embedding_size, hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iteration,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'voc_dict': voc.__dict__,
'embedding': embedding.state_dict()
},
os.path.join(directory,
'{}_{}.tar'.format(iteration, 'checkpoint')))
# Calculate average BLEU scores on devleopment set
if (iteration % evaluate_every == 0):
print("*" * 70)
print("iteration {}".format(iteration))
dev_evaluate(encoder, decoder, dev_pairs, searcher, voc)
print("*" * 70)
def trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer,
decoder_optimizer, embedding, encoder_n_layers,
decoder_n_layers, save_dir, n_iteration, batch_size,
print_every, save_every, clip, corpus_name, checkpoint,
hidden_size, teacher_forcing_ratio, device):
# Load batches for each iteration
training_batches = [
batch2TrainData(voc, [random.choice(pairs) for _ in range(batch_size)])
for _ in range(n_iteration)
]
# Initializations
print('Initializing ...')
start_iteration = 1
print_loss = 0
if checkpoint:
start_iteration = checkpoint['iteration'] + 1
# Training loop
print("Training...")
for iteration in range(start_iteration, n_iteration + 1):
training_batch = training_batches[iteration - 1]
# Extract fields from batch
input_variable, lengths, target_variable, mask, max_target_len = training_batch
# Run a training iteration with batch
loss = train(input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, encoder_optimizer,
decoder_optimizer, batch_size, clip, device,
teacher_forcing_ratio)
print_loss += loss
# Print progress
if iteration % print_every == 0:
print_loss_avg = print_loss / print_every
print(
"Iteration: {}; Percent complete: {:.1f}%; Average loss: {:.4f}"
.format(iteration, iteration / n_iteration * 100,
print_loss_avg))
print_loss = 0
# Save checkpoint
if iteration % save_every == 0:
directory = os.path.join(
save_dir, model_name, corpus_name,
'{}-{}_{}'.format(encoder_n_layers, decoder_n_layers,
hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iteration,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'voc_dict': voc.__dict__,
'embedding': embedding.state_dict()
},
os.path.join(directory,
'{}_{}.tar'.format(iteration, 'checkpoint')))