def evaluate(
model,
iterator,
criterion,
TRG,
SRC,
print_results=False,
beam=False):
model.eval()
epoch_loss = 0
with torch.no_grad():
for i, batch in enumerate(iterator):
src = batch.src
trg = batch.trg
output = model(src, trg, 0) # turn off teacher forcing
if beam:
beam_search(output, TRG)
# trg = [trg sent len, batch size]
# output = [trg sent len, batch size, output dim]
# grail = ['<unk>', '*Ident-IO(voi)', 'Agree', '*D', '*D_sigma', '<eos>']
# for i in range(output.shape[1]):
# readable_output = list(map(lambda x: TRG.vocab.itos[x], torch.argmax(
# output[:, i, :], 1).tolist()))
# if readable_output == grail:
# print('found grail!')
if print_results and i % random.randint(
1, 20) or i % random.randint(1, 15) == 0:
source = list(
map(lambda x: SRC.vocab.itos[x], src[:, 0].tolist()))[1:-1]
print(''.join(source).replace('<sep>', ', '), end=' & ')
output_list = list(map(lambda x: TRG.vocab.itos[x], torch.argmax(
output[:, 0, :], 1).tolist()))[1:-1]
print(' >> '.join(output_list))
output = output[1:].view(-1, output.shape[-1])
trg = trg[1:].view(-1)
# trg = [(trg sent len - 1) * batch size]
# output = [(trg sent len - 1) * batch size, output dim]
loss = criterion(output, trg)
epoch_loss += loss.item()
return epoch_loss / len(iterator)
def translate_sentence(sentence, net, args, src_vocab, tgt_vocab):
net.eval()
indexed = []
for tok in sentence:
if tok not in ["<pad>", "<s>", "</s>"]:
try:
tok_i = src_vocab.stoi[tok]
except KeyError:
tok_i = src_vocab.stoi["<unk>"]
indexed.append(tok_i)
sentence = Variable(torch.LongTensor([indexed]))
if args.use_cuda:
sentence = sentence.cuda()
sentence = beam_search(sentence, net, src_vocab, tgt_vocab, args)
return multiple_replace(
{
' ?': '?',
' !': '!',
' .': '.',
'\' ': '\'',
' ,': ','
}, sentence)
def solve(self, wp):
def score_func(d):
return self.probability_of_derivation(d)
def final_eval_func(derivations):
total_probs = defaultdict(int)
for d in derivations:
sol = d.solve()
total_probs[tuple(sorted(sol))] += score_func(d)
best_prob = max(total_probs.values())
for sol, prob in total_probs.iteritems():
if prob == best_prob:
return list(sol)
return None
solution = beam_search(wp, self.unique_templates, score_func,
lambda d: True, final_eval_func)
correct_sol = wp.labeled_example.solutions[:]
correct = True
for c in correct_sol:
if c in solution:
solution.remove(c)
else:
correct = False
break
print('guessed: {}, correct: {}, got it?: {}'.format(
solution, correct_sol, correct))
return int(correct)
def log_likelihood_gradient(self, word_problems, wp_template_indices,
unique_templates):
def score_func(derivation):
return self.probability_of_derivation(derivation)
def final_evaluation_func(derivations):
if len(derivations) == 0:
print('no derivations in beam for log likelihood gradient')
gradient = numpy.zeros(len(self.parameters))
probs = list()
for d in derivations:
prob = score_func(d)
probs.append(prob)
instance = self.feature_extractor.extract(d).instance
gradient += prob * numpy.array(instance)
total_prob = sum(probs)
if total_prob == 0:
print('no probablity to normalize in gradient')
return gradient
return gradient / total_prob
total_gradient = numpy.zeros(len(self.parameters))
for i, wp in enumerate(word_problems):
correct_index = wp_template_indices[i]
solutions = wp.labeled_example.solutions
print('ll gradient for wp: {} with template: {}'.format(
i, correct_index))
def validator_func(d):
return self.can_derive_correct_equations(
d, correct_index, solutions)
total_gradient += beam_search(wp, unique_templates, score_func,
validator_func,
final_evaluation_func)
total_gradient -= beam_search(wp, unique_templates, score_func,
lambda d: True,
final_evaluation_func)
return total_gradient
def translate_sentence(sentence, model, opt):
model.eval()
indexed = []
toks = sentence.split(' ')
for tok in toks:
indexed.append(get_synonym(tok))
indexed.append(get_synonym(' . '))
sentence = torch.IntTensor(indexed)
if opt.device == 0:
sentence = sentence.cuda()
sentence = beam_search(sentence, model, opt)
return multiple_replace({' ?': '?', ' !': '!', ' .': '.', '\' ': '\'', ' ,': ','}, sentence)
def translate(text):
"""
Translates the given text with beam search.
"""
ids = text2ids(text, SRC)
_, preds = beam_search(
model=model,
inputs=ids,
indices=indices,
beam_size=args.beam_size,
device=device)
output = ids2text([preds.squeeze()], TRG)[0]
return ' '.join(w for w in output
if w not in (PAD, END))
def log_likelihood(self, word_problems, wp_template_indices,
unique_templates):
def score_func(derivation):
return self.probability_of_derivation(derivation)
def final_evaluation_func(derivations):
if len(derivations) == 0:
print('no derivations in beam for log likelihood')
probs = list()
for d in derivations:
probs.append(score_func(d))
total = sum(probs)
result = 0
for p in probs:
result += math.log(p / total)
return result
total = 0
for i, wp in enumerate(word_problems):
correct_index = wp_template_indices[i]
solutions = wp.labeled_example.solutions
print('ll for wp: {} with template: {}, with solutions: {}'.format(
i, correct_index, solutions))
def validator_func(d):
return self.can_derive_correct_equations(
d, correct_index, solutions)
total += beam_search(wp, unique_templates, score_func,
validator_func, final_evaluation_func)
print('log likelihood total after word problem: {} is {}'.format(
i, total))
return total
import torch
from beam import beam_search
data = [[0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1],
[0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1],
[0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1],
[0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1],
[0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1]]
data = torch.tensor(data)
result = beam_search(data, -1, 2, len(data))
print(result)