def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _step(self, loader, update, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
i = 1
preds, losses, ys = [], [], []
dy.renew_cg()
for batch_dict in pg(loader):
x, y, l = self.model.make_input(batch_dict)
pred = self.model.forward(x, l)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.esum(losses)
preds = dy.concatenate_cols(preds)
total_loss += loss.npvalue().item()
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
preds, losses, ys = [], [], []
dy.renew_cg()
i += 1
loss = dy.esum(losses)
preds = dy.concatenate_cols(preds)
total_loss += loss.npvalue().item()
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss], feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _train(self, loader):
self.model.train()
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
for x, y in loader:
self.optimizer.zero_grad()
if type(x) == list:
x = [torch.autograd.Variable(item.cuda()) for item in x]
else:
x = torch.autograd.Variable(x.cuda())
y = torch.autograd.Variable(y.cuda())
pred = self.model(x)
loss = self.crit(pred, y)
total_loss += loss.data[0]
loss.backward()
_add_to_cm(cm, y, pred)
self.optimizer.step()
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss/float(steps)
return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _train(self, loader, **kwargs):
self.model.train()
reporting_fns = kwargs.get('reporting_fns', [])
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
self.optimizer.zero_grad()
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
report_loss = loss.item() * batchsz
epoch_loss += report_loss
epoch_div += batchsz
self.nstep_agg += report_loss
self.nstep_div += batchsz
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
_add_to_cm(cm, y, pred)
self.optimizer.step()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(self.optimizer.global_step + 1, metrics,
self.nstep_start, 'Train', 'STEP', reporting_fns,
self.nsteps)
self.reset_nstep()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
verbose = kwargs.get("verbose", False)
pg = create_progress_bar(steps)
for batch_dict in loader:
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
total_loss += lossv
cm.add_batch(y, guess)
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
if verbose:
print(cm)
return metrics
def _step(self,
loader,
update,
log,
reporting_fns,
verbose=None,
output=None,
txts=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
preds, losses, ys = [], [], []
dy.renew_cg()
for i, batch_dict in enumerate(pg(loader), 1):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.forward(inputs)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
preds, losses, ys = [], [], []
dy.renew_cg()
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
epoch_loss += loss.npvalue().item() * batchsz
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _step(self,
loader,
update,
log,
reporting_fns,
verbose=None,
output=None,
txts=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
if handle is not None:
for p, y in zip(preds, ys):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]), self.model.labels[p],
self.model.labels[y]))
line_number += 1
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def _train(self, loader):
cm = ConfusionMatrix(self.model.labels)
total_loss = 0
steps = len(loader)
pg = create_progress_bar(steps)
for batch_dict in loader:
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict, do_dropout=True)
_, step, lossv, guess = self.sess.run([self.train_op, self.global_step, self.loss, self.model.best], feed_dict=feed_dict)
cm.add_batch(y, guess)
total_loss += lossv
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss/float(steps)
return metrics
def _test(self, loader):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.model.labels)
for batch_dict in loader:
truth = batch_dict.pop('y')
inputs = self.model.make_input(batch_dict)
pred = self.model.impl.predict_on_batch(inputs)
guess = np.argmax(pred, axis=-1)
cm.add_batch(truth, guess)
pg.update()
pg.done()
test_metrics = cm.get_all_metrics()
#for k, v in test_metrics.items():
# test_metrics[k] /= steps
return test_metrics
def _test(self, loader):
total_loss = 0
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
pg = ProgressBar(steps)
for x, y in loader:
feed_dict = self.model.ex2dict(x, y)
lossv, guess = self.sess.run([self.loss, self.model.best],
feed_dict=feed_dict)
cm.add_batch(y, guess)
total_loss += lossv
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
return metrics
def _train(self, loader):
cm = ConfusionMatrix(self.model.labels)
total_loss = 0
steps = len(loader)
pg = ProgressBar(steps)
for x, y in loader:
feed_dict = self.model.ex2dict(x, y, do_dropout=True)
_, step, lossv, guess = self.sess.run(
[self.train_op, self.global_step, self.loss, self.model.best],
feed_dict=feed_dict)
cm.add_batch(y, guess)
total_loss += lossv
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
return metrics
def _step(self, loader, update):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
for batch_dict in pg(loader):
dy.renew_cg()
xs, ys, ls = self.model.make_input(batch_dict)
preds = self.model.forward(xs, ls)
losses = self.model.loss(preds, ys)
loss = dy.sum_batches(losses)
total_loss += loss.npvalue().item()
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
return metrics
def _test(self, ts):
total_loss = 0
steps = len(ts)
cm = ConfusionMatrix(self.model.labels)
pg = create_progress_bar(steps)
for batch_dict in ts:
y = fill_y(len(self.model.labels), batch_dict['y'])
feed_dict = self.model.make_input(batch_dict, do_dropout=True)
preds, lossv, = self.model.sess.run(
[self.model.best, self.model.loss], feed_dict=feed_dict)
# print(preds)
cm.add_batch(y, preds)
total_loss += lossv
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = float(total_loss) / steps
return metrics
def _test(self, loader):
self.model.eval()
total_loss = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
for batch_dict in loader:
x, y = self.model.make_input(batch_dict)
pred = self.model(x)
loss = self.crit(pred, y)
total_loss += loss.data[0]
_add_to_cm(cm, y, pred)
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss/float(steps)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
output = kwargs.get('output')
txts = kwargs.get('txts')
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
if handle is not None:
for predicted, gold in zip(guess, y):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]),
self.model.labels[predicted], self.model.labels[gold]))
line_number += 1
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def _train(self, loader):
self.model.train()
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
for batch_dict in loader:
self.optimizer.zero_grad()
x, y = self.model.make_input(batch_dict)
pred = self.model(x)
loss = self.crit(pred, y)
total_loss += loss.data[0]
loss.backward()
_add_to_cm(cm, y, pred)
self.optimizer.step()
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss/float(steps)
return metrics
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
preds, losses, ys = [], [], []
dy.renew_cg()
for i, batch_dict in enumerate(pg(loader), 1):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.forward(inputs)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
preds, losses, ys = [], [], []
dy.renew_cg()
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
epoch_loss += loss.npvalue().item() * batchsz
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
output = kwargs.get('output')
txts = kwargs.get('txts')
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
ys = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, ys)
if handle is not None:
for p, y in zip(pred, ys):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]), self.model.labels[p],
self.model.labels[y]))
line_number += 1
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, ys, pred)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in loader:
vec = self._make_input(batch_dict)
y = vec[-1]
pred = self.model(vec[:-1])
loss = self.crit(pred, y)
total_loss += loss.item()
_add_to_cm(cm, y, pred)
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
verbose_output(verbose, cm)
return metrics
def _train(self, loader):
self.model.train()
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
for batch_dict in loader:
self.optimizer.zero_grad()
vec = self._make_input(batch_dict)
y = vec[-1]
pred = self.model(vec[:-1])
loss = self.crit(pred, y)
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
_add_to_cm(cm, y, pred)
self.optimizer.step()
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, y, pred)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _train(self, loader, **kwargs):
self.model.train()
reporting_fns = kwargs.get('reporting_fns', [])
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
self.optimizer.zero_grad()
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
report_loss = loss.item() * batchsz
epoch_loss += report_loss
epoch_div += batchsz
self.nstep_agg += report_loss
self.nstep_div += batchsz
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
_add_to_cm(cm, y, pred)
self.optimizer.step()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fns, self.nsteps
)
self.reset_nstep()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, y, pred)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _test(self, loader):
self.model.eval()
total_loss = 0
steps = len(loader)
pg = ProgressBar(steps)
cm = ConfusionMatrix(self.labels)
for x, y in loader:
if type(x) == list:
x = [torch.autograd.Variable(item.cuda()) for item in x]
else:
x = torch.autograd.Variable(x.cuda())
y = torch.autograd.Variable(y.cuda())
pred = self.model(x)
loss = self.crit(pred, y)
total_loss += loss.data[0]
_add_to_cm(cm, y, pred)
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
return metrics