def _step(self, loader, update, verbose=None): steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) total_loss = 0 i = 1 preds, losses, ys = [], [], [] dy.renew_cg() for batch_dict in pg(loader): x, y, l = self.model.make_input(batch_dict) pred = self.model.forward(x, l) preds.append(pred) loss = self.model.loss(pred, y) losses.append(loss) ys.append(y) if i % self.autobatchsz == 0: loss = dy.esum(losses) preds = dy.concatenate_cols(preds) total_loss += loss.npvalue().item() _add_to_cm(cm, np.array(ys), preds.npvalue()) update(loss) preds, losses, ys = [], [], [] dy.renew_cg() i += 1 loss = dy.esum(losses) preds = dy.concatenate_cols(preds) total_loss += loss.npvalue().item() _add_to_cm(cm, np.array(ys), preds.npvalue()) update(loss) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(steps) verbose_output(verbose, cm) return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None): steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) epoch_loss = 0 epoch_div = 0 for batch_dict in pg(loader): dy.renew_cg() inputs = self.model.make_input(batch_dict) ys = inputs.pop('y') preds = self.model.forward(inputs) losses = self.model.loss(preds, ys) loss = dy.mean_batches(losses) batchsz = self._get_batchsz(batch_dict) lossv = loss.npvalue().item() * batchsz epoch_loss += lossv epoch_div += batchsz _add_to_cm(cm, ys, preds.npvalue()) update(loss) log(self.optimizer.global_step, lossv, batchsz, reporting_fns) metrics = cm.get_all_metrics() metrics['avg_loss'] = epoch_loss / float(epoch_div) verbose_output(verbose, cm) return metrics
def _test(self, loader, **kwargs): if self.ema: self.sess.run(self.ema_load) cm = ConfusionMatrix(self.model.labels) steps = len(loader) total_loss = 0 total_norm = 0 verbose = kwargs.get("verbose", None) pg = create_progress_bar(steps) for batch_dict in pg(loader): y = batch_dict['y'] feed_dict = self.model.make_input(batch_dict) guess, lossv = self.sess.run([self.model.best, self.test_loss], feed_dict=feed_dict) batchsz = self._get_batchsz(batch_dict) total_loss += lossv * batchsz total_norm += batchsz cm.add_batch(y, guess) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(total_norm) verbose_output(verbose, cm) return metrics
def _test(self, loader, **kwargs): """Test an epoch of data using either the input loader or using `tf.dataset` In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps to train. :param loader: A data feed :param kwargs: See below :Keyword Arguments: * *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True` * *reporting_fns* (`list`) A list of reporting hooks to use * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on :return: Metrics """ if self.ema: self.sess.run(self.ema_load) use_dataset = kwargs.get('dataset', True) cm = ConfusionMatrix(self.model.labels) steps = len(loader) total_loss = 0 total_norm = 0 verbose = kwargs.get("verbose", None) pg = create_progress_bar(steps) for i, batch_dict in enumerate(pg(loader)): y = batch_dict['y'] if use_dataset: guess, lossv = self.sess.run([self.model.best, self.test_loss]) else: feed_dict = self.model.make_input(batch_dict, False) guess, lossv = self.sess.run([self.model.best, self.test_loss], feed_dict=feed_dict) batchsz = len(guess) total_loss += lossv * batchsz total_norm += batchsz cm.add_batch(y, guess) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(total_norm) verbose_output(verbose, cm) return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None, output=None, txts=None): steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) epoch_loss = 0 epoch_div = 0 preds, losses, ys = [], [], [] dy.renew_cg() for i, batch_dict in enumerate(pg(loader), 1): inputs = self.model.make_input(batch_dict) y = inputs.pop('y') pred = self.model.forward(inputs) preds.append(pred) loss = self.model.loss(pred, y) losses.append(loss) ys.append(y) if i % self.autobatchsz == 0: loss = dy.average(losses) preds = dy.concatenate_cols(preds) batchsz = len(losses) lossv = loss.npvalue().item() * batchsz epoch_loss += lossv epoch_div += batchsz _add_to_cm(cm, np.array(ys), preds.npvalue()) update(loss) log(self.optimizer.global_step, lossv, batchsz, reporting_fns) preds, losses, ys = [], [], [] dy.renew_cg() loss = dy.average(losses) preds = dy.concatenate_cols(preds) batchsz = len(losses) epoch_loss += loss.npvalue().item() * batchsz epoch_div += batchsz _add_to_cm(cm, np.array(ys), preds.npvalue()) update(loss) metrics = cm.get_all_metrics() metrics['avg_loss'] = epoch_loss / float(epoch_div) verbose_output(verbose, cm) return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None, output=None, txts=None): steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) epoch_loss = 0 epoch_div = 0 handle = None line_number = 0 if output is not None and txts is not None: handle = open(output, "w") for batch_dict in pg(loader): dy.renew_cg() inputs = self.model.make_input(batch_dict) ys = inputs.pop('y') preds = self.model.forward(inputs) losses = self.model.loss(preds, ys) loss = dy.mean_batches(losses) batchsz = self._get_batchsz(batch_dict) lossv = loss.npvalue().item() * batchsz if handle is not None: for p, y in zip(preds, ys): handle.write('{}\t{}\t{}\n'.format( " ".join(txts[line_number]), self.model.labels[p], self.model.labels[y])) line_number += 1 epoch_loss += lossv epoch_div += batchsz _add_to_cm(cm, ys, preds.npvalue()) update(loss) log(self.optimizer.global_step, lossv, batchsz, reporting_fns) metrics = cm.get_all_metrics() metrics['avg_loss'] = epoch_loss / float(epoch_div) verbose_output(verbose, cm) if handle is not None: handle.close() return metrics
def _test(self, loader, steps=0, **kwargs): """Test an epoch of data using either the input loader or using `tf.dataset` In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps to train. :param loader: A data feed :param kwargs: See below :Keyword Arguments: * *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True` * *reporting_fns* (`list`) A list of reporting hooks to use * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on :return: Metrics """ cm = ConfusionMatrix(self.model.labels) total_loss = 0 total_norm = 0 verbose = kwargs.get("verbose", None) pg = create_progress_bar(steps) SET_TRAIN_FLAG(False) for features, y in pg(loader): logits = self.model(features) y_ = tf.argmax(logits, axis=1, output_type=tf.int32) cm.add_batch(y, y_) lossv = tf.compat.v1.losses.sparse_softmax_cross_entropy( labels=y, logits=logits).numpy() batchsz = int(y.shape[0]) assert len(y_) == batchsz total_loss += lossv * batchsz total_norm += batchsz cm.add_batch(y, y_) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(total_norm) verbose_output(verbose, cm) return metrics
def _step(self, loader, update, verbose=None): steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) total_loss = 0 for batch_dict in pg(loader): dy.renew_cg() xs, ys, ls = self.model.make_input(batch_dict) preds = self.model.forward(xs, ls) losses = self.model.loss(preds, ys) loss = dy.sum_batches(losses) total_loss += loss.npvalue().item() _add_to_cm(cm, ys, preds.npvalue()) update(loss) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(steps) verbose_output(verbose, cm) return metrics
def _test(self, loader, **kwargs): if self.ema: self.sess.run(self.ema_load) cm = ConfusionMatrix(self.model.labels) steps = len(loader) total_loss = 0 total_norm = 0 verbose = kwargs.get("verbose", None) output = kwargs.get('output') txts = kwargs.get('txts') handle = None line_number = 0 if output is not None and txts is not None: handle = open(output, "w") pg = create_progress_bar(steps) for batch_dict in pg(loader): y = batch_dict['y'] feed_dict = self.model.make_input(batch_dict) guess, lossv = self.sess.run([self.model.best, self.test_loss], feed_dict=feed_dict) batchsz = self._get_batchsz(batch_dict) if handle is not None: for predicted, gold in zip(guess, y): handle.write('{}\t{}\t{}\n'.format( " ".join(txts[line_number]), self.model.labels[predicted], self.model.labels[gold])) line_number += 1 total_loss += lossv * batchsz total_norm += batchsz cm.add_batch(y, guess) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(total_norm) verbose_output(verbose, cm) if handle is not None: handle.close() return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None): steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) epoch_loss = 0 epoch_div = 0 preds, losses, ys = [], [], [] dy.renew_cg() for i, batch_dict in enumerate(pg(loader), 1): inputs = self.model.make_input(batch_dict) y = inputs.pop('y') pred = self.model.forward(inputs) preds.append(pred) loss = self.model.loss(pred, y) losses.append(loss) ys.append(y) if i % self.autobatchsz == 0: loss = dy.average(losses) preds = dy.concatenate_cols(preds) batchsz = len(losses) lossv = loss.npvalue().item() * batchsz epoch_loss += lossv epoch_div += batchsz _add_to_cm(cm, np.array(ys), preds.npvalue()) update(loss) log(self.optimizer.global_step, lossv, batchsz, reporting_fns) preds, losses, ys = [], [], [] dy.renew_cg() loss = dy.average(losses) preds = dy.concatenate_cols(preds) batchsz = len(losses) epoch_loss += loss.npvalue().item() * batchsz epoch_div += batchsz _add_to_cm(cm, np.array(ys), preds.npvalue()) update(loss) metrics = cm.get_all_metrics() metrics['avg_loss'] = epoch_loss / float(epoch_div) verbose_output(verbose, cm) return metrics
def _test(self, loader, **kwargs): self.model.eval() total_loss = 0 total_norm = 0 steps = len(loader) pg = create_progress_bar(steps) no_cm = bool(kwargs.get('no_cm', False)) cm = None if no_cm else ConfusionMatrix(self.labels) verbose = kwargs.get("verbose", None) output = kwargs.get('output') txts = kwargs.get('txts') handle = None line_number = 0 if output is not None and txts is not None: handle = open(output, "w") with torch.no_grad(): for batch_dict in pg(loader): example = self._make_input(batch_dict) ys = example.pop('y') pred = self.model(example) loss = self.crit(pred, ys) if handle is not None: for p, y in zip(pred, ys): handle.write('{}\t{}\t{}\n'.format( " ".join(txts[line_number]), self.model.labels[p], self.model.labels[y])) line_number += 1 batchsz = self._get_batchsz(batch_dict) total_loss += loss.item() * batchsz total_norm += batchsz _add_to_cm(cm, ys, pred) metrics = cm.get_all_metrics() if cm is not None else {} metrics['avg_loss'] = total_loss / float(total_norm) verbose_output(verbose, cm) if handle is not None: handle.close() return metrics
def _test(self, loader, **kwargs): self.model.eval() total_loss = 0 steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) verbose = kwargs.get("verbose", None) for batch_dict in loader: vec = self._make_input(batch_dict) y = vec[-1] pred = self.model(vec[:-1]) loss = self.crit(pred, y) total_loss += loss.item() _add_to_cm(cm, y, pred) pg.update() pg.done() metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(steps) verbose_output(verbose, cm) return metrics
def _test(self, loader, **kwargs): self.model.eval() total_loss = 0 total_norm = 0 steps = len(loader) pg = create_progress_bar(steps) cm = ConfusionMatrix(self.labels) verbose = kwargs.get("verbose", None) for batch_dict in pg(loader): example = self._make_input(batch_dict) y = example.pop('y') pred = self.model(example) loss = self.crit(pred, y) batchsz = self._get_batchsz(batch_dict) total_loss += loss.item() * batchsz total_norm += batchsz _add_to_cm(cm, y, pred) metrics = cm.get_all_metrics() metrics['avg_loss'] = total_loss / float(total_norm) verbose_output(verbose, cm) return metrics