def generateResults(encoder_decoder: EncoderDecoder, data_loader,
resultFilename, input_tokens_list):
idx_to_tok = encoder_decoder.lang.idx_to_tok
all_output_seqs = []
all_target_seqs = []
for batch_idx, (input_idxs, target_idxs, _,
_) in enumerate(tqdm(data_loader)):
input_lengths = (input_idxs != 0).long().sum(dim=1)
sorted_lengths, order = torch.sort(input_lengths, descending=True)
input_variable = Variable(input_idxs[order, :][:, :max(input_lengths)])
target_variable = Variable(target_idxs[order, :])
output_log_probs, output_seqs = encoder_decoder(
input_variable, list(sorted_lengths))
print(output_seqs.size())
all_output_seqs.extend(trim_seqs(output_seqs))
all_target_seqs.extend([list(seq[seq > 0])]
for seq in to_np(target_variable))
with open(resultFilename, 'w') as fo:
for seq, input_tokens in zip(all_output_seqs, input_tokens_list):
print(type(seq))
#seq = seq.data.view(-1)
eos_idx = seq.index(2) if 2 in seq else seq
string = seq_to_string(seq[:eos_idx + 1],
idx_to_tok,
input_tokens=None)
fo.write(string + '\n')
return None
def evaluate(encoder_decoder: EncoderDecoder, data_loader):
loss_function = torch.nn.NLLLoss(
ignore_index=0, reduce=False
) # what does this return for ignored idxs? same length output?
losses = []
all_output_seqs = []
all_target_seqs = []
for batch_idx, (input_idxs, target_idxs, _,
_) in enumerate(tqdm(data_loader)):
input_lengths = (input_idxs != 0).long().sum(dim=1)
sorted_lengths, order = torch.sort(input_lengths, descending=True)
input_variable = Variable(input_idxs[order, :][:, :max(input_lengths)],
volatile=True)
target_variable = Variable(target_idxs[order, :], volatile=True)
batch_size = input_variable.shape[0]
output_log_probs, output_seqs = encoder_decoder(
input_variable, list(sorted_lengths))
all_output_seqs.extend(trim_seqs(output_seqs))
all_target_seqs.extend([list(seq[seq > 0])]
for seq in to_np(target_variable))
flattened_log_probs = output_log_probs.view(
batch_size * encoder_decoder.decoder.max_length, -1)
batch_losses = loss_function(flattened_log_probs,
target_variable.contiguous().view(-1))
losses.extend(list(to_np(batch_losses)))
mean_loss = len(losses) / sum(losses)
"""
for i in range(20):
print(all_target_seqs[i])
print(all_output_seqs[i])
print('*'*80)
"""
bleu_score = corpus_bleu(all_target_seqs,
all_output_seqs,
smoothing_function=SmoothingFunction().method2)
print('BLEU SCORE: ' + str(bleu_score))
return mean_loss, bleu_score
def validate(args, val_loader, model):
batch_time = AverageMeter()
data_time = AverageMeter()
val_logger = LogCollector()
# switch to evaluate mode
model.val_start()
model.logger = val_logger
end = time.time()
max_length=50
for i, val_data in enumerate(val_loader):
decoder_outputs, sampled_idxs, mean, logvar, z = model.forward_emb(*val_data)
if torch.cuda.is_available():
val_data[1]=val_data[1].cuda()
batch_size = val_data[1].size(0)
max_length=50
flattened_outputs = decoder_outputs.view(batch_size * max_length, -1)
loss = model.loss_function(flattened_outputs, val_data[1].contiguous().view(-1))
kl_loss = (-0.5 * torch.sum(logvar - torch.pow(mean, 2) - torch.exp(logvar) + 1, 1)).mean().squeeze()
model.logger.update('KL Loss', kl_loss.item(), 1)
model.logger.update('MLE Loss', loss.item(), 1)
batch_outputs = trim_seqs(sampled_idxs)
np_targets=trim_seqs(val_data[1].unsqueeze(-1))
batch_targets = [[seq] for seq in np_targets]
corpus_bleu_score = corpus_bleu(batch_targets, batch_outputs, smoothing_function=SmoothingFunction().method1)
model.logger.update('C-BLEU', corpus_bleu_score, batch_size)
batch_bleu_score=0
for j in range(batch_size):
batch_bleu_score += sentence_bleu(batch_targets[j], batch_outputs[j], weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=SmoothingFunction().method1)
batch_bleu_score=batch_bleu_score/batch_size
model.logger.update('S-BLEU', batch_bleu_score, batch_size)
batch_time.update(time.time() - end)
end = time.time()
# Print log info
model.Eiters += 1
if model.Eiters % args.logging_step == 0:
print('Test: [{0}/{1}]\t'
'{e_log}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
.format(
i, len(val_loader), batch_time=batch_time,
e_log=str(model.logger)))
print('Test: [{0}/{1}]\t'
'{e_log}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
.format(
i, len(val_loader), batch_time=batch_time,
e_log=str(model.logger)))
return batch_bleu_score
def validate(args, val_loader, model):
batch_time = AverageMeter()
data_time = AverageMeter()
val_logger = LogCollector()
# switch to evaluate mode
model.val_start()
model.logger = val_logger
end = time.time()
max_length=50
for i, val_data in enumerate(val_loader):
decoder_outputs, sampled_idxs, mean, logvar, z = model.forward_emb(*val_data)
if torch.cuda.is_available():
val_data[1]=val_data[1].cuda()
batch_size = val_data[1].size(0)
max_length=50
flattened_outputs = decoder_outputs.view(batch_size * max_length, -1)
batch_outputs = trim_seqs(sampled_idxs)
np_targets=trim_seqs(val_data[1].unsqueeze(-1))
batch_targets = [[seq] for seq in np_targets]
corpus_bleu_score = corpus_bleu(batch_targets, batch_outputs, smoothing_function=SmoothingFunction().method1)
model.logger.update('C-BLEU', corpus_bleu_score, batch_size)
corpus_nist_score = corpus_nist(batch_targets, batch_outputs, n=4)
model.logger.update('C-NIST', corpus_nist_score, batch_size)
corpus_meteor_score=0
rouge_scores=0
for j in range(batch_size):
reference=[]
for tid in range(len(batch_targets[j][0])):
tok=batch_targets[j][0][tid]
reference.append(vocab_inv[str(tok)])
ref=[str(' '.join(reference))]
hypothesis=[]
for tid in range(len(batch_outputs[j])):
tok=batch_outputs[j][tid]
hypothesis.append(vocab_inv[str(tok)])
hypo=str(' '.join(hypothesis))
corpus_meteor_score+=meteor_score(ref, hypo)
rouge_scores += rouge.score(ref[0],hypo)['rougeL'][2]
rouge_scores=rouge_scores/batch_size
model.logger.update('ROUGH-L', rouge_scores, batch_size)
corpus_meteor_score=corpus_meteor_score/batch_size
model.logger.update('C-METEOR', corpus_meteor_score, batch_size)
batch_time.update(time.time() - end)
end = time.time()
# Print log info
model.Eiters += 1
if model.Eiters % args.logging_step == 0:
print('Test: [{0}/{1}]\t'
'{e_log}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
.format(
i, len(val_loader), batch_time=batch_time,
e_log=str(model.logger)))
print('Test: [{0}/{1}]\t'
'{e_log}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
.format(
i, len(val_loader), batch_time=batch_time,
e_log=str(model.logger)))
return 0
def train(encoder_decoder: EncoderDecoder, train_data_loader: DataLoader,
model_name, val_data_loader: DataLoader, keep_prob,
teacher_forcing_schedule, lr, max_length, use_decay, data_path):
global_step = 0
loss_function = torch.nn.NLLLoss(ignore_index=0)
optimizer = optim.Adam(encoder_decoder.parameters(), lr=lr)
model_path = './saved/' + model_name + '/'
if (use_decay == False):
gamma = 1.0
else:
gamma = 0.5
scheduler = StepLR(optimizer, step_size=1, gamma=gamma)
#val_loss, val_bleu_score = evaluate(encoder_decoder, val_data_loader)
best_bleu = 0.0
for epoch, teacher_forcing in enumerate(teacher_forcing_schedule):
#scheduler.step()
print('epoch %i' % (epoch), flush=True)
print('lr: ' + str(scheduler.get_lr()))
for batch_idx, (input_idxs, target_idxs, input_tokens,
target_tokens) in enumerate(tqdm(train_data_loader)):
# input_idxs and target_idxs have dim (batch_size x max_len)
# they are NOT sorted by length
'''
print(input_idxs[0])
print(input_tokens[0])
print(target_idxs[0])
print(target_tokens[0])
'''
lengths = (input_idxs != 0).long().sum(dim=1)
sorted_lengths, order = torch.sort(lengths, descending=True)
input_variable = Variable(input_idxs[order, :][:, :max(lengths)])
target_variable = Variable(target_idxs[order, :])
optimizer.zero_grad()
output_log_probs, output_seqs = encoder_decoder(
input_variable,
list(sorted_lengths),
targets=target_variable,
keep_prob=keep_prob,
teacher_forcing=teacher_forcing)
batch_size = input_variable.shape[0]
flattened_outputs = output_log_probs.view(batch_size * max_length,
-1)
batch_loss = loss_function(flattened_outputs,
target_variable.contiguous().view(-1))
batch_loss.backward()
optimizer.step()
batch_outputs = trim_seqs(output_seqs)
batch_targets = [[list(seq[seq > 0])]
for seq in list(to_np(target_variable))]
#batch_bleu_score = corpus_bleu(batch_targets, batch_outputs, smoothing_function=SmoothingFunction().method2)
batch_bleu_score = corpus_bleu(batch_targets, batch_outputs)
'''
if global_step < 10 or (global_step % 10 == 0 and global_step < 100) or (global_step % 100 == 0 and epoch < 2):
input_string = "Amy, Please schedule a meeting with Marcos on Tuesday April 3rd. Adam Kleczewski"
output_string = encoder_decoder.get_response(input_string)
writer.add_text('schedule', output_string, global_step=global_step)
input_string = "Amy, Please cancel this meeting. Adam Kleczewski"
output_string = encoder_decoder.get_response(input_string)
writer.add_text('cancel', output_string, global_step=global_step)
'''
if global_step % 100 == 0:
writer.add_scalar('train_batch_loss', batch_loss, global_step)
writer.add_scalar('train_batch_bleu_score', batch_bleu_score,
global_step)
for tag, value in encoder_decoder.named_parameters():
tag = tag.replace('.', '/')
writer.add_histogram('weights/' + tag,
value,
global_step,
bins='doane')
writer.add_histogram('grads/' + tag,
to_np(value.grad),
global_step,
bins='doane')
global_step += 1
debug = False
if (debug):
if batch_idx == 5:
break
val_loss, val_bleu_score = evaluate(encoder_decoder, val_data_loader)
writer.add_scalar('val_loss', val_loss, global_step=global_step)
writer.add_scalar('val_bleu_score',
val_bleu_score,
global_step=global_step)
encoder_embeddings = encoder_decoder.encoder.embedding.weight.data
encoder_vocab = encoder_decoder.lang.tok_to_idx.keys()
writer.add_embedding(encoder_embeddings,
metadata=encoder_vocab,
global_step=0,
tag='encoder_embeddings')
decoder_embeddings = encoder_decoder.decoder.embedding.weight.data
decoder_vocab = encoder_decoder.lang.tok_to_idx.keys()
writer.add_embedding(decoder_embeddings,
metadata=decoder_vocab,
global_step=0,
tag='decoder_embeddings')
'''
input_string = "Amy, Please schedule a meeting with Marcos on Tuesday April 3rd. Adam Kleczewski"
output_string = encoder_decoder.get_response(input_string)
writer.add_text('schedule', output_string, global_step=global_step)
input_string = "Amy, Please cancel this meeting. Adam Kleczewski"
output_string = encoder_decoder.get_response(input_string)
writer.add_text('cancel', output_string, global_step=global_step)
'''
calc_bleu_score = get_bleu(encoder_decoder, data_path, None, 'dev')
print('val loss: %.5f, val BLEU score: %.5f' %
(val_loss, calc_bleu_score),
flush=True)
if (calc_bleu_score > best_bleu):
print("Best BLEU score! Saving model...")
best_bleu = calc_bleu_score
torch.save(
encoder_decoder, "%s%s_%i_%.3f.pt" %
(model_path, model_name, epoch, calc_bleu_score))
print('-' * 100, flush=True)
scheduler.step()
def train(encoder_decoder: EncoderDecoder, train_data_loader: DataLoader,
model_name, val_data_loader: DataLoader, keep_prob,
teacher_forcing_schedule, lr, max_length, device,
test_data_loader: DataLoader):
global_step = 0
loss_function = torch.nn.NLLLoss(ignore_index=0)
optimizer = optim.Adam(encoder_decoder.parameters(), lr=lr)
model_path = './model/' + model_name + '/'
trained_model = encoder_decoder
for epoch, teacher_forcing in enumerate(teacher_forcing_schedule):
print('epoch %i' % epoch, flush=True)
correct_predictions = 0.0
all_predictions = 0.0
for batch_idx, (input_idxs, target_idxs, input_tokens,
target_tokens) in enumerate(tqdm(train_data_loader)):
# Empty the cache at each batch
torch.cuda.empty_cache()
# input_idxs and target_idxs have dim (batch_size x max_len)
# they are NOT sorted by length
lengths = (input_idxs != 0).long().sum(dim=1)
sorted_lengths, order = torch.sort(lengths, descending=True)
input_variable = input_idxs[order, :][:, :max(lengths)]
input_variable = input_variable.to(device)
target_variable = target_idxs[order, :]
target_variable = target_variable.to(device)
optimizer.zero_grad()
output_log_probs, output_seqs = encoder_decoder(
input_variable,
list(sorted_lengths),
targets=target_variable,
keep_prob=keep_prob,
teacher_forcing=teacher_forcing)
batch_size = input_variable.shape[0]
output_sentences = output_seqs.squeeze(2)
flattened_outputs = output_log_probs.view(batch_size * max_length,
-1)
batch_loss = loss_function(flattened_outputs,
target_variable.contiguous().view(-1))
batch_outputs = trim_seqs(output_seqs)
batch_inputs = [[list(seq[seq > 0])]
for seq in list(to_np(input_variable))]
batch_targets = [[list(seq[seq > 0])]
for seq in list(to_np(target_variable))]
for i in range(len(batch_outputs)):
y_i = batch_outputs[i]
tgt_i = batch_targets[i][0]
if y_i == tgt_i:
correct_predictions += 1.0
all_predictions += 1.0
batch_loss.backward()
optimizer.step()
batch_bleu_score = corpus_bleu(
batch_targets,
batch_outputs,
smoothing_function=SmoothingFunction().method1)
if global_step % 100 == 0:
writer.add_scalar('train_batch_loss', batch_loss, global_step)
writer.add_scalar('train_batch_bleu_score', batch_bleu_score,
global_step)
for tag, value in encoder_decoder.named_parameters():
tag = tag.replace('.', '/')
writer.add_histogram('weights/' + tag,
value,
global_step,
bins='doane')
writer.add_histogram('grads/' + tag,
to_np(value.grad),
global_step,
bins='doane')
global_step += 1
encoder_embeddings = encoder_decoder.encoder.embedding.weight.data
encoder_vocab = encoder_decoder.lang.tok_to_idx.keys()
writer.add_embedding(encoder_embeddings,
metadata=encoder_vocab,
global_step=0,
tag='encoder_embeddings')
decoder_embeddings = encoder_decoder.decoder.embedding.weight.data
decoder_vocab = encoder_decoder.lang.tok_to_idx.keys()
writer.add_embedding(decoder_embeddings,
metadata=decoder_vocab,
global_step=0,
tag='decoder_embeddings')
print('training accuracy %.5f' %
(100.0 * (correct_predictions / all_predictions)))
torch.save(encoder_decoder,
"%s%s_%i.pt" % (model_path, model_name, epoch))
trained_model = encoder_decoder
print('-' * 100, flush=True)
torch.save(encoder_decoder, "%s%s_final.pt" % (model_path, model_name))
return trained_model
def test(encoder_decoder: EncoderDecoder,
test_data_loader: DataLoader,
max_length,
device,
log_files=None):
correct_predictions = 0.0
struct_correct_only = 0.0
all_predictions = 0.0
for batch_idx, (input_idxs, target_idxs, input_tokens,
target_tokens) in enumerate(tqdm(test_data_loader)):
print('memory usage {}'.format(torch.cuda.memory_allocated()))
# input_idxs and target_idxs have dim (batch_size x max_len)
# they are NOT sorted by length
lengths = (input_idxs != 0).long().sum(dim=1)
sorted_lengths, order = torch.sort(lengths, descending=True)
input_variable = input_idxs[order, :][:, :max(lengths)]
input_variable = input_variable.to(device)
input_tokens = [input_tokens[o] for o in order]
target_variable = target_idxs[order, :]
target_variable = target_variable.to(device)
target_tokens = [target_tokens[o] for o in order]
output_log_probs, output_seqs = encoder_decoder(
input_variable, list(sorted_lengths), targets=target_variable)
batch_size = input_variable.shape[0]
output_sentences = output_seqs.squeeze(2)
flattened_outputs = output_log_probs.view(batch_size * max_length, -1)
batch_outputs = trim_seqs(output_seqs)
batch_inputs = [[list(seq[seq > 0])]
for seq in list(to_np(input_variable))]
batch_targets = [[list(seq[seq > 0])]
for seq in list(to_np(target_variable))]
for i in range(len(batch_outputs)):
# Get the input and output tokens
y_i = batch_outputs[i]
tgt_i = batch_targets[i][0]
src_i = batch_inputs[i][0]
# Make dictionaries of unknown words and their tokens
src_token_list = input_tokens[i].split()
tar_token_list = target_tokens[i].split()
src_unk_to_tok = {
src_i[j]: src_token_list[j]
for j in range(len(src_i))
if not src_i[j] in encoder_decoder.lang.idx_to_tok
}
tar_unk_to_tok = {
tgt_i[j]: tar_token_list[j]
for j in range(len(tgt_i))
if not tgt_i[j] in encoder_decoder.lang.idx_to_tok
}
# Translate tokens to words
correct_seq = [
encoder_decoder.lang.idx_to_tok[n]
if n in encoder_decoder.lang.idx_to_tok else tar_unk_to_tok[n]
for n in tgt_i
]
incorrect_seq = [
encoder_decoder.lang.idx_to_tok[n]
if n in encoder_decoder.lang.idx_to_tok else src_unk_to_tok[n]
for n in y_i
]
input_seq = [
encoder_decoder.lang.idx_to_tok[n]
if n in encoder_decoder.lang.idx_to_tok else src_unk_to_tok[n]
for n in src_i
]
if y_i == tgt_i:
correct_predictions += 1.0
if not log_files == None:
log_files[1].write("INPUT PHRASE: {}\n".format(
' '.join(input_seq)))
log_files[1].write("CORRECT LABEL: {}\n".format(
' '.join(correct_seq)))
log_files[1].write("PREDICTED LABEL: {}\n".format(
' '.join(incorrect_seq)))
log_files[1].write("\n\n")
else:
if not log_files == None:
log_files[0].write("INPUT PHRASE: {}\n".format(
' '.join(input_seq)))
log_files[0].write("CORRECT LABEL: {}\n".format(
' '.join(correct_seq)))
log_files[0].write("PREDICTED LABEL: {}\n".format(
' '.join(incorrect_seq)))
log_files[0].write("\n\n")
all_predictions += 1.0
return 100.0 * (correct_predictions / all_predictions)
def train_emb(self, sources, qs_tar, mask_tar, temp_tar, sqls_lens,
question_lens, mask_lens, template_lens, ids, *args):
self.Eiters += 1
self.logger.update('Eit', self.Eiters)
self.logger.update('lr', self.optimizer.param_groups[0]['lr'])
sources = Variable(sources)
qs_tar = Variable(qs_tar)
mask_tar = Variable(mask_tar)
temp_tar = Variable(temp_tar)
if torch.cuda.is_available():
sources = sources.cuda()
temp_tar = temp_tar.cuda()
mask_tar = mask_tar.cuda()
qs_tar = qs_tar.cuda()
data = [
sources, qs_tar, mask_tar, temp_tar, sqls_lens, question_lens,
mask_lens, template_lens, ids
]
# compute the embeddings
decoder_outputs, sampled_idxs, mean, logvar, z = self.forward_emb(
*data)
self.optimizer.zero_grad()
batch_size = sources.shape[0]
max_length = 50
flattened_outputs = decoder_outputs.view(batch_size * max_length, -1)
sos_tar = qs_tar.clone()
sos_tar[sos_tar == 5] = 0
loss = self.loss_function(flattened_outputs,
sos_tar.contiguous().view(-1))
kl_loss = (
-0.5 *
torch.sum(logvar - torch.pow(mean, 2) - torch.exp(logvar) + 1,
1)).mean().squeeze()
self.logger.update('KL Loss', kl_loss.item(), 1)
self.logger.update('CE Loss', loss.item(), 1)
loss = loss + kl_loss
loss.backward()
self.optimizer.step()
batch_outputs = trim_seqs(sampled_idxs)
np_targets = trim_seqs(qs_tar.unsqueeze(-1))
batch_targets = [[seq] for seq in np_targets]
corpus_bleu_score = corpus_bleu(
batch_targets,
batch_outputs,
smoothing_function=SmoothingFunction().method1)
self.logger.update('C-BLEU', corpus_bleu_score, qs_tar.size(0))
