def test(self, vs, reporting_fns, phase, **kwargs): if phase == 'Test': return self._evaluate(vs, reporting_fns, **kwargs) self.model.eval() total_loss = total_toks = 0 steps = len(vs) self.valid_epochs += 1 preds = [] golds = [] start = time.time() pg = create_progress_bar(steps) for batch_dict in pg(vs): input_ = self._input(batch_dict) tgt = input_['tgt'] tgt_lens = batch_dict['tgt_lengths'] pred = self.model(input_) loss = self.crit(pred, tgt) toks = self._num_toks(tgt_lens) total_loss += loss.item() * toks total_toks += toks greedy_preds = [ p[0] for p in self._predict(input_, beam=1, make_input=False) ] preds.extend(convert_seq2seq_preds(greedy_preds, self.tgt_rlut)) golds.extend( convert_seq2seq_golds(tgt.cpu().numpy(), tgt_lens, self.tgt_rlut)) metrics = self.calc_metrics(total_loss, total_toks) metrics['bleu'] = bleu(preds, golds)[0] self.report(self.valid_epochs, metrics, start, phase, 'EPOCH', reporting_fns) return metrics
def test(self, vs, reporting_fns, phase='Valid'): if phase == 'Test': return self._evaluate(vs, reporting_fns) self.valid_epochs += 1 total_loss = 0 total_toks = 0 metrics = {} preds = [] golds = [] start = time.time() pg = create_progress_bar(len(vs)) for batch_dict in pg(vs): feed_dict = self.model.make_input(batch_dict) lossv, top_preds = self.model.sess.run( [self.test_loss, self.model.decoder.best], feed_dict=feed_dict) toks = self._num_toks(batch_dict['tgt_lengths']) total_loss += lossv * toks total_toks += toks preds.extend(convert_seq2seq_preds(top_preds.T, self.tgt_rlut)) golds.extend( convert_seq2seq_golds(batch_dict['tgt'], batch_dict['tgt_lengths'], self.tgt_rlut)) metrics = self.calc_metrics(total_loss, total_toks) metrics['bleu'] = bleu(preds, golds)[0] self.report(self.valid_epochs, metrics, start, phase, 'EPOCH', reporting_fns) return metrics
def test(self, vs, reporting_fns, phase): if phase == 'Test': return self._evaluate(vs, reporting_fns) self.model.eval() total_loss = total_toks = 0 steps = len(vs) self.valid_epochs += 1 preds = [] golds = [] start = time.time() pg = create_progress_bar(steps) for batch_dict in pg(vs): input_ = self._input(batch_dict) tgt = input_['tgt'] tgt_lens = batch_dict['tgt_lengths'] pred = self.model(input_) loss = self.crit(pred, tgt) toks = self._num_toks(tgt_lens) total_loss += loss.item() * toks total_toks += toks greedy_preds = [p[0] for p in self._predict(input_, beam=1, make_input=False)] preds.extend(convert_seq2seq_preds(greedy_preds, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt.cpu().numpy(), tgt_lens, self.tgt_rlut)) metrics = self.calc_metrics(total_loss, total_toks) metrics['bleu'] = bleu(preds, golds)[0] self.report( self.valid_epochs, metrics, start, phase, 'EPOCH', reporting_fns ) return metrics
def test(self, vs, reporting_fns, phase='Valid'): if phase == 'Test': return self._evaluate(vs, reporting_fns) self.valid_epochs += 1 total_loss = 0 total_toks = 0 metrics = {} preds = [] golds = [] start = time.time() pg = create_progress_bar(len(vs)) for batch_dict in pg(vs): feed_dict = self.model.make_input(batch_dict) lossv, top_preds = self.model.sess.run([self.test_loss, self.model.decoder.best], feed_dict=feed_dict) toks = self._num_toks(batch_dict['tgt_lengths']) total_loss += lossv * toks total_toks += toks preds.extend(convert_seq2seq_preds(top_preds.T, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(batch_dict['tgt'], batch_dict['tgt_lengths'], self.tgt_rlut)) metrics = self.calc_metrics(total_loss, total_toks) metrics['bleu'] = bleu(preds, golds)[0] self.report( self.valid_epochs, metrics, start, phase, 'EPOCH', reporting_fns ) return metrics
def _evaluate(self, es, reporting_fns): self.model.train = False pg = create_progress_bar(len(es)) preds = [] golds = [] start = time.time() for batch_dict in pg(es): tgt = batch_dict['tgt'] tgt_lens = batch_dict['tgt_lengths'] pred = [p[0] for p in self.model.predict(batch_dict)] preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut)) metrics = {'bleu': bleu(preds, golds)[0]} self.report(0, metrics, start, 'Test', 'EPOCH', reporting_fns)
def _evaluate(self, es, reporting_fns): """Run the model with beam search and report Bleu.""" pg = create_progress_bar(len(es)) preds = [] golds = [] start = time.time() for batch_dict in pg(es): tgt = batch_dict.pop('tgt') tgt_lens = batch_dict.pop('tgt_lengths') pred = [p[0] for p in self.model.predict(batch_dict)] preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut)) metrics = {'bleu': bleu(preds, golds)[0]} self.report(0, metrics, start, 'Test', 'EPOCH', reporting_fns) return metrics
def _evaluate(self, es, reporting_fns): self.model.train = False pg = create_progress_bar(len(es)) preds = [] golds = [] start = time.time() for batch_dict in pg(es): tgt = batch_dict['tgt'] tgt_lens = batch_dict['tgt_lengths'] pred = [p[0] for p in self.model.predict(batch_dict)] preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut)) metrics = {'bleu': bleu(preds, golds)[0]} self.report( 0, metrics, start, 'Test', 'EPOCH', reporting_fns ) return metrics
def _evaluate(self, es, reporting_fns): """Run the model with beam search and report Bleu.""" pg = create_progress_bar(len(es)) preds = [] golds = [] start = time.time() for batch_dict in pg(es): tgt = batch_dict.pop('tgt') tgt_lens = batch_dict.pop('tgt_lengths') pred = [p[0] for p in self.model.predict(batch_dict)] preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut)) metrics = {'bleu': bleu(preds, golds)[0]} self.report( 0, metrics, start, 'Test', 'EPOCH', reporting_fns ) return metrics
def test(self, vs, reporting_fns, phase): if phase == 'Test': return self._evaluate(vs, reporting_fns) self.model.train = False total_loss = total_toks = 0 steps = len(vs) self.valid_epochs += 1 preds = [] golds = [] start = time.time() pg = create_progress_bar(steps) for batch_dict in pg(vs): dy.renew_cg() inputs = self.model.make_input(batch_dict) tgt = inputs.pop('tgt') tgt_lens = batch_dict['tgt_lengths'] output = self.model.forward(inputs) loss = self._loss(output, tgt, tgt_lens) toks = self._num_toks(tgt_lens) loss_val = loss.npvalue().item() total_loss += loss_val * toks total_toks += toks top_preds = np.transpose( np.stack([np.argmax(o.npvalue(), axis=0) for o in output])) if len(top_preds.shape) == 1: top_preds = np.expand_dims(top_preds, 0) preds.extend(convert_seq2seq_preds(top_preds, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt.T, tgt_lens, self.tgt_rlut)) metrics = self.calc_metrics(total_loss, total_toks) metrics['bleu'] = bleu(preds, golds)[0] self.report(self.valid_epochs, metrics, start, phase, 'EPOCH', reporting_fns) return metrics
def test(self, vs, reporting_fns, phase): if phase == 'Test': return self._evaluate(vs, reporting_fns) self.model.train = False total_loss = total_toks = 0 steps = len(vs) self.valid_epochs += 1 preds = [] golds = [] start = time.time() pg = create_progress_bar(steps) for batch_dict in pg(vs): dy.renew_cg() inputs = self.model.make_input(batch_dict) tgt = inputs.pop('tgt') tgt_lens = batch_dict['tgt_lengths'] output = self.model.forward(inputs) loss = self._loss(output, tgt, tgt_lens) toks = self._num_toks(tgt_lens) loss_val = loss.npvalue().item() total_loss += loss_val * toks total_toks += toks pred = [p[0] for p in self.model.predict(batch_dict, beam=1)] preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut)) golds.extend(convert_seq2seq_golds(tgt.T, tgt_lens, self.tgt_rlut)) metrics = self.calc_metrics(total_loss, total_toks) metrics['bleu'] = bleu(preds, golds)[0] self.report( self.valid_epochs, metrics, start, phase, 'EPOCH', reporting_fns ) return metrics