def _step(self, loader, update, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
i = 1
preds, losses, ys = [], [], []
dy.renew_cg()
for batch_dict in pg(loader):
x, y, l = self.model.make_input(batch_dict)
pred = self.model.forward(x, l)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.esum(losses)
preds = dy.concatenate_cols(preds)
total_loss += loss.npvalue().item()
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
preds, losses, ys = [], [], []
dy.renew_cg()
i += 1
loss = dy.esum(losses)
preds = dy.concatenate_cols(preds)
total_loss += loss.npvalue().item()
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
verbose_output(verbose, cm)
return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss], feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
"""Test an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train.
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
:return: Metrics
"""
if self.ema:
self.sess.run(self.ema_load)
use_dataset = kwargs.get('dataset', True)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for i, batch_dict in enumerate(pg(loader)):
y = batch_dict['y']
if use_dataset:
guess, lossv = self.sess.run([self.model.best, self.test_loss])
else:
feed_dict = self.model.make_input(batch_dict, False)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = len(guess)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _step(self,
loader,
update,
log,
reporting_fns,
verbose=None,
output=None,
txts=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
preds, losses, ys = [], [], []
dy.renew_cg()
for i, batch_dict in enumerate(pg(loader), 1):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.forward(inputs)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
preds, losses, ys = [], [], []
dy.renew_cg()
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
epoch_loss += loss.npvalue().item() * batchsz
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _step(self,
loader,
update,
log,
reporting_fns,
verbose=None,
output=None,
txts=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
if handle is not None:
for p, y in zip(preds, ys):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]), self.model.labels[p],
self.model.labels[y]))
line_number += 1
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def _test(self, loader, steps=0, **kwargs):
"""Test an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train.
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
:return: Metrics
"""
cm = ConfusionMatrix(self.model.labels)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
SET_TRAIN_FLAG(False)
for features, y in pg(loader):
logits = self.model(features)
y_ = tf.argmax(logits, axis=1, output_type=tf.int32)
cm.add_batch(y, y_)
lossv = tf.compat.v1.losses.sparse_softmax_cross_entropy(
labels=y, logits=logits).numpy()
batchsz = int(y.shape[0])
assert len(y_) == batchsz
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, y_)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _step(self, loader, update, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
for batch_dict in pg(loader):
dy.renew_cg()
xs, ys, ls = self.model.make_input(batch_dict)
preds = self.model.forward(xs, ls)
losses = self.model.loss(preds, ys)
loss = dy.sum_batches(losses)
total_loss += loss.npvalue().item()
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
output = kwargs.get('output')
txts = kwargs.get('txts')
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
if handle is not None:
for predicted, gold in zip(guess, y):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]),
self.model.labels[predicted], self.model.labels[gold]))
line_number += 1
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
preds, losses, ys = [], [], []
dy.renew_cg()
for i, batch_dict in enumerate(pg(loader), 1):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.forward(inputs)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
preds, losses, ys = [], [], []
dy.renew_cg()
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
epoch_loss += loss.npvalue().item() * batchsz
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
no_cm = bool(kwargs.get('no_cm', False))
cm = None if no_cm else ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
output = kwargs.get('output')
txts = kwargs.get('txts')
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
with torch.no_grad():
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
ys = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, ys)
if handle is not None:
for p, y in zip(pred, ys):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]), self.model.labels[p],
self.model.labels[y]))
line_number += 1
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, ys, pred)
metrics = cm.get_all_metrics() if cm is not None else {}
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in loader:
vec = self._make_input(batch_dict)
y = vec[-1]
pred = self.model(vec[:-1])
loss = self.crit(pred, y)
total_loss += loss.item()
_add_to_cm(cm, y, pred)
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, y, pred)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, y, pred)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
