def validate(self, valid_iter, step=0):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
with torch.no_grad():
for batch in valid_iter:
src = batch.src
labels = batch.src_sent_labels
#segs = batch.segs
clss = batch.clss
#mask = batch.mask_src
mask_cls = batch.mask_cls
#sent_scores, mask = self.model(src, segs, clss, mask, mask_cls)
sent_scores, mask = self.model(src, clss, mask_cls)
loss = self.loss(sent_scores, labels.float())
loss = (loss * mask.float()).sum()
batch_stats = Statistics(float(loss.cpu().data.numpy()),
len(labels))
stats.update(batch_stats)
self._report_step(0, step, valid_stats=stats)
return stats
def validate(self, valid_iter):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
with torch.no_grad():
for batch in valid_iter:
src = batch.src
tgt = batch.tgt
outputs, _ = self.model(src, tgt)
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs)
stats.update(batch_stats)
return stats
def sharded_compute_loss(self, batch, output, shard_size, normalization):
"""Compute the forward loss and backpropagate. Computation is done
with shards and optionally truncation for memory efficiency.
Also supports truncated BPTT for long sequences by taking a
range in the decoder output sequence to back propagate in.
Range is from `(cur_trunc, cur_trunc + trunc_size)`.
Note sharding is an exact efficiency trick to relieve memory
required for the generation buffers. Truncation is an
approximate efficiency trick to relieve the memory required
in the RNN buffers.
Args:
batch (batch) : batch of labeled examples
output (:obj:`FloatTensor`) :
output of decoder model `[tgt_len x batch x hidden]`
attns (dict) : dictionary of attention distributions
`[tgt_len x batch x src_len]`
cur_trunc (int) : starting position of truncation window
trunc_size (int) : length of truncation window
shard_size (int) : maximum number of examples in a shard
normalization (int) : Loss is divided by this number
Returns:
:obj:`onmt.utils.Statistics`: validation loss statistics
"""
batch_stats = Statistics()
shard_state = self._make_shard_state(batch, output)
for shard in shards(shard_state, shard_size):
loss, stats = self._compute_loss(batch, **shard)
loss.div(float(normalization)).backward()
batch_stats.update(stats)
return batch_stats
def test(self, test_iter, step, cal_lead=False, cal_oracle=False):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
def _get_ngrams(n, text):
ngram_set = set()
text_length = len(text)
max_index_ngram_start = text_length - n
for i in range(max_index_ngram_start + 1):
ngram_set.add(tuple(text[i:i + n]))
return ngram_set
def _block_tri(c, p):
tri_c = _get_ngrams(3, c.split())
for s in p:
tri_s = _get_ngrams(3, s.split())
if len(tri_c.intersection(tri_s)) > 0:
return True
return False
if (not cal_lead and not cal_oracle):
self.model.eval()
stats = Statistics()
can_path = '%s_step%d.candidate' % (self.args.result_path, step)
gold_path = '%s_step%d.gold' % (self.args.result_path, step)
with open(can_path, 'w') as save_pred:
with open(gold_path, 'w') as save_gold:
with torch.no_grad():
for batch in test_iter:
src = batch.src
labels = batch.src_sent_labels
#segs = batch.segs
clss = batch.clss
#mask = batch.mask_src
mask_cls = batch.mask_cls
gold = []
pred = []
if (cal_lead):
print('not implemented!')
exit(1)
#selected_ids = [list(range(batch.clss.size(1)))] * batch.batch_size
elif (cal_oracle):
print('not implemented!')
exit(1)
#selected_ids = [[j for j in range(batch.clss.size(1)) if labels[i][j] == 1] for i in
#range(batch.batch_size)]
else:
sent_scores, mask = self.model(src, clss, mask_cls)
loss = self.loss(sent_scores, labels.float())
loss = (loss * mask.float()).sum()
batch_stats = Statistics(
float(loss.cpu().data.numpy()), len(labels))
stats.update(batch_stats)
sent_scores = sent_scores + mask.float()
sent_scores = sent_scores.cpu().data.numpy()
selected_ids = np.argsort(-sent_scores, 1)
# selected_ids = np.sort(selected_ids,1)
for i, idx in enumerate(selected_ids):
_pred = []
if (len(batch.src_str[i]) == 0):
continue
for j in selected_ids[i][:len(batch.src_str[i])]:
if (j >= len(batch.src_str[i])):
continue
candidate = batch.src_str[i][j].strip()
if (self.args.block_trigram):
if (not _block_tri(candidate, _pred)):
_pred.append(candidate)
else:
_pred.append(candidate)
if ((not cal_oracle)
and (not self.args.recall_eval)
and len(_pred) == 3):
break
_pred = '<q>'.join(_pred)
if (self.args.recall_eval):
_pred = ' '.join(
_pred.split()
[:len(batch.tgt_str[i].split())])
pred.append(_pred)
gold.append(batch.tgt_str[i])
for i in range(len(gold)):
save_gold.write(gold[i].strip() + '\n')
for i in range(len(pred)):
save_pred.write(pred[i].strip() + '\n')
if (step != -1 and self.args.report_rouge):
#raise NotImplementedError
self.logger.info("Calculating Rouge")
candidates = codecs.open(can_path, encoding="utf-8")
references = codecs.open(gold_path, encoding="utf-8")
rouges = test_rouge(candidates, references, 1)
self.logger.info('Rouges at step %d \n%s' %
(step, rouge_results_to_str(rouges)))
if self.tensorboard_writer is not None:
self.tensorboard_writer.add_scalar('test/rouge1-F',
rouges['rouge_1_f_score'],
step)
self.tensorboard_writer.add_scalar('test/rouge2-F',
rouges['rouge_2_f_score'],
step)
self.tensorboard_writer.add_scalar('test/rougeL-F',
rouges['rouge_l_f_score'],
step)
self._report_step(0, step, valid_stats=stats)
return stats