def translate(dataset, fields, model):
already_1, hypothesis_1, references_1 = 0, [], []
already_2_3, hypothesis_2, hypothesis_3 = 0, [], []
for batch, flag in dataset:
if flag:
predictions = beam_search(opt, model, batch.src, fields, flag)
hypothesis_1 += [
fields["task1_tgt"].decode(p) for p in predictions
]
already_1 += len(predictions)
logging.info("Task 1: %7d/%7d" %
(already_1, dataset.task1_dataset.num_examples))
else:
predictions_1, predictions_2 = beam_search(opt, model, batch.src,
fields, flag)
hypothesis_2 += [
fields["task2_tgt"].decode(p) for p in predictions_1
]
hypothesis_3 += [
fields["task3_tgt"].decode(p) for p in predictions_2
]
already_2_3 += len(predictions_1)
logging.info("Finished: %7d/%7d" %
(already_2_3, dataset.task2_3_dataset.num_examples))
origin_1 = sorted(zip(hypothesis_1, dataset.task1_dataset.seed),
key=lambda t: t[1])
hypothesis_1 = [h for h, _ in origin_1]
with open(opt.output[0], "w", encoding="UTF-8") as out_file:
out_file.write("\n".join(hypothesis_1))
out_file.write("\n")
origin_2_3 = sorted(zip(hypothesis_2, hypothesis_3,
dataset.task2_3_dataset.seed),
key=lambda t: t[2])
hypothesis_2 = [h for h, _, _ in origin_2_3]
hypothesis_3 = [h for _, h, _ in origin_2_3]
with open(opt.output[1], "w", encoding="UTF-8") as out_file:
out_file.write("\n".join(hypothesis_2))
out_file.write("\n")
with open(opt.output[2], "w", encoding="UTF-8") as out_file:
out_file.write("\n".join(hypothesis_3))
out_file.write("\n")
logging.info("All finished. ")
def valid(model, criterion, valid_dataset, step):
model.eval()
total_loss, total = 0.0, 0
hypothesis, references = [], []
for batch in valid_dataset:
scores = model(batch.src, batch.tgt, batch.probs, batch.idxes)
loss = criterion(scores, batch.tgt)
total_loss += loss.data
total += 1
if opt.tf:
_, predictions = scores.topk(k=1, dim=-1)
else:
predictions = beam_search(opt, model, batch.src,
valid_dataset.fields)
hypothesis += [
valid_dataset.fields["tgt"].decode(p) for p in predictions
]
references += [
valid_dataset.fields["tgt"].decode(t) for t in batch.tgt
]
bleu = calculate_bleu(hypothesis, references)
logging.info("Valid loss: %.2f\tValid BLEU: %3.2f" %
(total_loss / total, bleu))
checkpoint = {"model": model.state_dict(), "opt": opt}
saver.save(checkpoint, step, bleu, total_loss / total)
def translate(dataset, fields, model):
already, hypothesis, references = 0, [], []
for batch in dataset:
if opt.tf:
scores = model(batch.src, batch.tgt)
_, predictions = scores.topk(k=1, dim=-1)
else:
predictions = beam_search(opt, model, batch.src, fields)
hypothesis += [fields["tgt"].decode(p) for p in predictions]
already += len(predictions)
logging.info("Translated: %7d/%7d" % (already, dataset.num_examples))
references += [fields["tgt"].decode(t) for t in batch.tgt]
if opt.bleu:
bleu = calculate_bleu(hypothesis, references)
logging.info("BLEU: %3.2f" % bleu)
origin = sorted(zip(hypothesis, dataset.seed), key=lambda t: t[1])
hypothesis = [h for h, _ in origin]
with open(opt.output, "w", encoding="UTF-8") as out_file:
out_file.write("\n".join(hypothesis))
out_file.write("\n")
logging.info("Translation finished. ")
def translate(dataset, fields, model):
already, hypothesis_1, hypothesis_2 = 0, [], []
for batch in dataset:
predictions_1, predictions_2 = beam_search(opt, model, batch.source,
fields)
hypothesis_1 += [fields["summary_cn"].decode(p) for p in predictions_1]
hypothesis_2 += [fields["summary_en"].decode(p) for p in predictions_2]
already += len(predictions_1)
logging.info("Finished: %7d/%7d" % (already, dataset.num_examples))
origin = sorted(zip(hypothesis_1, hypothesis_2, dataset.seed),
key=lambda t: t[2])
hypothesis_1 = [h for h, _, _ in origin]
hypothesis_2 = [h for _, h, _ in origin]
with open(opt.output[0], "w", encoding="UTF-8") as out_file:
out_file.write("\n".join(hypothesis_1))
out_file.write("\n")
with open(opt.output[1], "w", encoding="UTF-8") as out_file:
out_file.write("\n".join(hypothesis_2))
out_file.write("\n")
logging.info("All finished. ")