def _train(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_restore)
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_div = 0
steps = len(loader)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
feed_dict = self.model.make_input(batch_dict, True)
_, step, lossv = self.sess.run([self.train_op, self.global_step, self.loss], feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
report_lossv = lossv * batchsz
epoch_loss += report_lossv
epoch_div += batchsz
self.nstep_agg += report_lossv
self.nstep_div += batchsz
if (step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fns, self.nsteps
)
self.reset_nstep()
metrics = self.calc_metrics(epoch_loss, epoch_div)
return metrics
def _test(self, loader, dataset=True, **kwargs):
epoch_probs = []
epoch_y = []
epoch_loss = 0
epoch_norm = 0
pg = create_progress_bar(len(loader))
for batch_dict in pg(loader):
if dataset:
probs, lossv, y = self.sess.run(
[self.probs, self.loss, self.y],
feed_dict={TRAIN_FLAG(): 0}
)
batchsz = len(y)
else:
feed_dict = self.model.make_input(batch_dict, False)
probs, lossv, y = self.sess.run(
[self.probs, self.loss, self.y],
feed_dict=feed_dict
)
batchsz = self._get_batchsz(batch_dict)
epoch_loss += lossv * batchsz
epoch_norm += batchsz
epoch_probs.append(probs)
epoch_y.append(y)
probs = np.concatenate(epoch_probs, axis=0)
y = np.argmax(np.concatenate(epoch_y, axis=0), axis=1)
bins = multiclass_calibration_bins(y, probs, bins=int(kwargs.get('bins', 10)))
metrics = {
"ECE": expected_calibration_error(bins.accs, bins.confs, bins.counts),
"MCE": maximum_calibration_error(bins.accs, bins.confs, bins.counts),
"avg_loss": epoch_loss / float(epoch_norm)
}
return metrics
def web_downloader(url):
from baseline.progress import create_progress_bar
r = requests.get(url, stream=True)
if r.status_code != 200:
raise RuntimeError("The file can not be downloaded")
path_to_save = "/tmp/data.dload-{}".format(os.getpid())
try:
print("downloading {}".format(url))
with open(path_to_save, 'wb') as f:
total_length = r.headers.get('content-length', None)
if total_length is not None:
total_length = int(total_length)
chunk_size = 8 * 1024 # experimented with a couple, this seemed fastest.
pg = create_progress_bar(total_length / chunk_size)
for chunk in r.iter_content(chunk_size=chunk_size):
if chunk:
f.write(chunk)
pg.update()
f.flush()
pg.done()
else:
raise RuntimeWarning(
"Total length can not be calculated, aborting download")
except: # this is too broad but there are too many exceptions to handle separately
raise RuntimeError(
"failed to download data from [url]: {} [to]: {}".format(
url, path_to_save))
return path_to_save
def test(self, vs, reporting_fns, phase, **kwargs):
if phase == 'Test':
return self._evaluate(vs, reporting_fns, **kwargs)
self.model.eval()
total_loss = total_toks = 0
steps = len(vs)
self.valid_epochs += 1
preds = []
golds = []
start = time.time()
pg = create_progress_bar(steps)
for batch_dict in pg(vs):
input_ = self._input(batch_dict)
tgt = input_['tgt']
tgt_lens = batch_dict['tgt_lengths']
pred = self.model(input_)
loss = self.crit(pred, tgt)
toks = self._num_toks(tgt_lens)
total_loss += loss.item() * toks
total_toks += toks
greedy_preds = [
p[0] for p in self._predict(input_, beam=1, make_input=False)
]
preds.extend(convert_seq2seq_preds(greedy_preds, self.tgt_rlut))
golds.extend(
convert_seq2seq_golds(tgt.cpu().numpy(), tgt_lens,
self.tgt_rlut))
metrics = self.calc_metrics(total_loss, total_toks)
metrics['bleu'] = bleu(preds, golds)[0]
self.report(self.valid_epochs, metrics, start, phase, 'EPOCH',
reporting_fns)
return metrics
def _train(self, ts, debug=False):
debug = True
total_loss = 0
steps = len(ts) if not debug else 2
metrics = {}
pg = create_progress_bar(steps)
count = 0
for batch_dict in ts:
feed_dict = self.model.make_input(batch_dict, do_dropout=True)
preds, lossv, _, summs = self.model.sess.run([
self.model.probs, self.model.loss,
self.model.all_optimizer_var_updates_op,
self.model.summary_merged
],
feed_dict=feed_dict)
# print(preds)
total_loss += lossv
self.model.test_summary_writer.add_summary(summs)
pg.update()
count += 1
if debug and count == 2:
break
pg.done()
metrics['avg_loss'] = float(total_loss) / steps
return metrics
def _test(self, ts, **kwargs):
self.model.train = False
total_correct = 0
total_sum = 0
total_gold_count = 0
total_guess_count = 0
total_overlap_count = 0
metrics = {}
steps = len(ts)
conll_output = kwargs.get('conll_output', None)
txts = kwargs.get('txts', None)
handle = None
if conll_output is not None and txts is not None:
handle = open(conll_output, "w")
pg = create_progress_bar(steps)
for batch_dict in pg(ts):
lengths = batch_dict[self.model.lengths_key]
ids = batch_dict['ids']
y = batch_dict['y']
pred = self.model.predict(batch_dict)
correct, count, overlaps, golds, guesses = self.process_output(pred, y, lengths, ids, handle, txts)
total_correct += correct
total_sum += count
total_gold_count += golds
total_guess_count += guesses
total_overlap_count += overlaps
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
metrics['f1'] = f_score(total_overlap_count, total_gold_count, total_guess_count)
metrics['acc'] = total_acc
return metrics
def _train(self, loader, **kwargs):
self.model.train()
reporting_fns = kwargs.get('reporting_fns', [])
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
self.optimizer.zero_grad()
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
report_loss = loss.item() * batchsz
epoch_loss += report_loss
epoch_div += batchsz
self.nstep_agg += report_loss
self.nstep_div += batchsz
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
_add_to_cm(cm, y, pred)
self.optimizer.step()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(self.optimizer.global_step + 1, metrics,
self.nstep_start, 'Train', 'STEP', reporting_fns,
self.nsteps)
self.reset_nstep()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
return metrics
def test(self, vs, reporting_fns, phase):
self.model.eval()
metrics = {}
total_loss = total = 0
steps = len(vs)
epochs = 0
if phase == 'Valid':
self.valid_epochs += 1
epochs = self.valid_epochs
pg = create_progress_bar(steps)
for src, tgt, src_len, tgt_len in vs:
src, dst, tgt = self._wrap(src, tgt)
pred = self.model((src, dst))
loss = self.crit(pred, tgt)
total_loss += loss.data[0]
total += self._total(tgt)
pg.update()
pg.done()
avg_loss = float(total_loss) / total
metrics['avg_loss'] = avg_loss
metrics['perplexity'] = np.exp(avg_loss)
for reporting in reporting_fns:
reporting(metrics, epochs, phase)
return metrics
def test(self, vs, reporting_fns, phase):
if phase == 'Test':
return self._evaluate(vs, reporting_fns)
self.model.eval()
total_loss = total_toks = 0
steps = len(vs)
self.valid_epochs += 1
preds = []
golds = []
start = time.time()
pg = create_progress_bar(steps)
for batch_dict in pg(vs):
input_ = self._input(batch_dict)
tgt = input_['tgt']
tgt_lens = batch_dict['tgt_lengths']
pred = self.model(input_)
loss = self.crit(pred, tgt)
toks = self._num_toks(tgt_lens)
total_loss += loss.item() * toks
total_toks += toks
greedy_preds = [p[0] for p in self._predict(input_, beam=1, make_input=False)]
preds.extend(convert_seq2seq_preds(greedy_preds, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(tgt.cpu().numpy(), tgt_lens, self.tgt_rlut))
metrics = self.calc_metrics(total_loss, total_toks)
metrics['bleu'] = bleu(preds, golds)[0]
self.report(
self.valid_epochs, metrics, start,
phase, 'EPOCH', reporting_fns
)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def test(self, vs, reporting_fns, phase='Valid'):
if phase == 'Test':
return self._evaluate(vs, reporting_fns)
self.valid_epochs += 1
total_loss = 0
total_toks = 0
metrics = {}
preds = []
golds = []
start = time.time()
pg = create_progress_bar(len(vs))
for batch_dict in pg(vs):
feed_dict = self.model.make_input(batch_dict)
lossv, top_preds = self.model.sess.run([self.test_loss, self.model.decoder.best], feed_dict=feed_dict)
toks = self._num_toks(batch_dict['tgt_lengths'])
total_loss += lossv * toks
total_toks += toks
preds.extend(convert_seq2seq_preds(top_preds.T, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(batch_dict['tgt'], batch_dict['tgt_lengths'], self.tgt_rlut))
metrics = self.calc_metrics(total_loss, total_toks)
metrics['bleu'] = bleu(preds, golds)[0]
self.report(
self.valid_epochs, metrics, start,
phase, 'EPOCH', reporting_fns
)
return metrics
def _train(self, ts, **kwargs):
self.model.train()
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_norm = 0
steps = len(ts)
pg = create_progress_bar(steps)
for batch_dict in pg(ts):
inputs = self.model.make_input(batch_dict)
self.optimizer.zero_grad()
loss = self.model.compute_loss(inputs)
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
self.optimizer.step()
bsz = self._get_batchsz(batch_dict)
report_loss = loss.item() * bsz
epoch_loss += report_loss
epoch_norm += bsz
self.nstep_agg += report_loss
self.nstep_div += bsz
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(self.optimizer.global_step + 1, metrics,
self.nstep_start, 'Train', 'STEP', reporting_fns,
self.nsteps)
self.reset_nstep()
metrics = self.calc_metrics(epoch_loss, epoch_norm)
return metrics
def test(self, vs, reporting_fns, phase):
self.model.train = False
metrics = {}
total_loss = total = 0
steps = len(vs)
epochs = 0
if phase == 'Valid':
self.valid_epochs += 1
epochs = self.valid_epochs
start = time.time()
pg = create_progress_bar(steps)
for batch_dict in vs:
dy.renew_cg()
src, dst, src_len, tgt = self.model.make_input(batch_dict)
output = self.model.forward((src, dst, src_len))
loss = self._loss(output, tgt)
total += self._total(tgt)
loss_val = loss.npvalue().item()
total_loss += loss_val
pg.update()
pg.done()
self.log.debug({'phase': phase, 'time': time.time() - start})
avg_loss = float(total_loss) / total
metrics['avg_loss'] = avg_loss
metrics['perplexity'] = np.exp(avg_loss)
for reporting in reporting_fns:
reporting(metrics, epochs, phase)
return metrics
def test(self, vs, reporting_fns, phase='Valid'):
if phase == 'Test':
return self._evaluate(vs, reporting_fns)
self.valid_epochs += 1
total_loss = 0
total_toks = 0
metrics = {}
preds = []
golds = []
start = time.time()
pg = create_progress_bar(len(vs))
for batch_dict in pg(vs):
feed_dict = self.model.make_input(batch_dict)
lossv, top_preds = self.model.sess.run(
[self.test_loss, self.model.decoder.best], feed_dict=feed_dict)
toks = self._num_toks(batch_dict['tgt_lengths'])
total_loss += lossv * toks
total_toks += toks
preds.extend(convert_seq2seq_preds(top_preds.T, self.tgt_rlut))
golds.extend(
convert_seq2seq_golds(batch_dict['tgt'],
batch_dict['tgt_lengths'],
self.tgt_rlut))
metrics = self.calc_metrics(total_loss, total_toks)
metrics['bleu'] = bleu(preds, golds)[0]
self.report(self.valid_epochs, metrics, start, phase, 'EPOCH',
reporting_fns)
return metrics
def __init__(self, directory, pattern, vocabs, vectorizers, nctx):
super().__init__()
self.src_vectorizer = vectorizers['src']
self.tgt_vectorizer = vectorizers['tgt']
self.pattern = pattern
self.nctx = nctx
self.directory = directory
self.vocab = vocabs
self.samples = 0
self.rank = 0
self.world_size = 1
if torch.distributed.is_initialized():
self.rank = torch.distributed.get_rank()
self.world_size = torch.distributed.get_world_size()
if os.path.exists(f"{directory}/md.yml"):
f = read_yaml(f"{directory}/md.yml")
self.samples = f['num_samples']
else:
files = list(glob.glob(f"{directory}/{self.pattern}"))
pg = create_progress_bar(len(files))
for file in pg(files):
with open(file) as rf:
for _ in rf:
self.samples += 1
write_yaml({'num_samples': self.samples}, f"{directory}/md.yml")
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
verbose = kwargs.get("verbose", False)
pg = create_progress_bar(steps)
for batch_dict in loader:
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
total_loss += lossv
cm.add_batch(y, guess)
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
if verbose:
print(cm)
return metrics
def _test(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_load)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
y = batch_dict['y']
feed_dict = self.model.make_input(batch_dict)
guess, lossv = self.sess.run([self.model.best, self.test_loss], feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _step(self, loader, update, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
i = 1
preds, losses, ys = [], [], []
dy.renew_cg()
for batch_dict in pg(loader):
x, y, l = self.model.make_input(batch_dict)
pred = self.model.forward(x, l)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.esum(losses)
preds = dy.concatenate_cols(preds)
total_loss += loss.npvalue().item()
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
preds, losses, ys = [], [], []
dy.renew_cg()
i += 1
loss = dy.esum(losses)
preds = dy.concatenate_cols(preds)
total_loss += loss.npvalue().item()
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(steps)
verbose_output(verbose, cm)
return metrics
def _train(self, loader, **kwargs):
if self.ema:
self.sess.run(self.ema_restore)
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_div = 0
steps = len(loader)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
feed_dict = self.model.make_input(batch_dict, True)
_, step, lossv = self.sess.run(
[self.train_op, self.global_step, self.loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
report_lossv = lossv * batchsz
epoch_loss += report_lossv
epoch_div += batchsz
self.nstep_agg += report_lossv
self.nstep_div += batchsz
if (step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(step + 1, metrics, self.nstep_start, 'Train',
'STEP', reporting_fns, self.nsteps)
self.reset_nstep()
metrics = self.calc_metrics(epoch_loss, epoch_div)
return metrics
def _train(self, loader):
self.model.train()
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
total_loss = 0
for x, y in loader:
self.optimizer.zero_grad()
if type(x) == list:
x = [torch.autograd.Variable(item.cuda()) for item in x]
else:
x = torch.autograd.Variable(x.cuda())
y = torch.autograd.Variable(y.cuda())
pred = self.model(x)
loss = self.crit(pred, y)
total_loss += loss.data[0]
loss.backward()
_add_to_cm(cm, y, pred)
self.optimizer.step()
pg.update()
pg.done()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss/float(steps)
return metrics
def test(self, ts, conll_output=None, txts=None):
total_correct = total_sum = 0
total_gold_count = total_guess_count = total_overlap_count = 0
steps = len(ts)
pg = create_progress_bar(steps)
metrics = {}
# Only if they provide a file and the raw txts, we can write CONLL file
handle = None
if conll_output is not None and txts is not None:
handle = open(conll_output, "w")
try:
for batch_dict in pg(ts):
correct, count, overlaps, golds, guesses = self.process_batch(
batch_dict, handle, txts)
total_correct += correct
total_sum += count
total_gold_count += golds
total_guess_count += guesses
total_overlap_count += overlaps
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
metrics['f1'] = f_score(total_overlap_count, total_gold_count,
total_guess_count)
metrics['acc'] = total_acc
finally:
if handle is not None:
handle.close()
return metrics
def _train(self, ts, **kwargs):
self.model.train()
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_norm = 0
steps = len(ts)
pg = create_progress_bar(steps)
for batch_dict in pg(ts):
inputs = self.model.make_input(batch_dict)
self.optimizer.zero_grad()
loss = self.model.compute_loss(inputs)
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
self.optimizer.step()
bsz = self._get_batchsz(batch_dict)
report_loss = loss.item() * bsz
epoch_loss += report_loss
epoch_norm += bsz
self.nstep_agg += report_loss
self.nstep_div += bsz
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fns, self.nsteps
)
self.reset_nstep()
metrics = self.calc_metrics(epoch_loss, epoch_norm)
return metrics
def _test(self, ts):
self.model.eval()
total_correct = 0
total_sum = 0
total_gold_count = 0
total_guess_count = 0
total_overlap_count = 0
metrics = {}
steps = len(ts)
pg = create_progress_bar(steps)
for batch_dict in ts:
x, xch, lengths, y, ids = self.model.make_input(batch_dict)
inputs = (x, xch, lengths)
pred = self.model(inputs)
correct, count, overlaps, golds, guesses = self.process_output(
pred, y.data, lengths, ids, None, None)
total_correct += correct
total_sum += count
total_gold_count += golds
total_guess_count += guesses
total_overlap_count += overlaps
pg.update()
pg.done()
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
metrics['f1'] = f_score(total_overlap_count, total_gold_count,
total_guess_count)
metrics['acc'] = total_acc
return metrics
def test(self, vs, reporting_fns, phase):
self.model.eval()
metrics = {}
total_loss = total = 0
steps = len(vs)
epochs = 0
if phase == 'Valid':
self.valid_epochs += 1
epochs = self.valid_epochs
pg = create_progress_bar(steps)
for batch_dict in vs:
fx = self._input(batch_dict)
tgt = fx[-1]
fx = fx[:-1]
pred = self.model(fx)
loss = self.crit(pred, tgt)
total_loss += loss.data[0]
total += self._total(tgt)
pg.update()
pg.done()
avg_loss = float(total_loss)/total
metrics['avg_loss'] = avg_loss
metrics['perplexity'] = np.exp(avg_loss)
for reporting in reporting_fns:
reporting(metrics, epochs, phase)
return metrics
def test(self, ts, conll_output=None, txts=None):
total_correct = total_sum = 0
total_gold_count = total_guess_count = total_overlap_count = 0
steps = len(ts)
pg = create_progress_bar(steps)
metrics = {}
# Only if they provide a file and the raw txts, we can write CONLL file
handle = None
if conll_output is not None and txts is not None:
handle = open(conll_output, "w")
try:
for batch_dict in pg(ts):
correct, count, overlaps, golds, guesses = self.process_batch(batch_dict, handle, txts)
total_correct += correct
total_sum += count
total_gold_count += golds
total_guess_count += guesses
total_overlap_count += overlaps
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
metrics['f1'] = f_score(total_overlap_count, total_gold_count, total_guess_count)
metrics['acc'] = total_acc
finally:
if handle is not None:
handle.close()
return metrics
def run(args):
# Limit it to a single GPU.
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
conn = create_db(args.db)
m = Manager()
logs = args.logging
datasets = args.datasets
embeddings = args.embeddings
settings = args.settings
# So we don't litter the fs
dir_ = tempfile.mkdtemp(prefix='baseline-speed-test-')
try:
configs = get_configs(args.config)
if not args.single:
full_configs = []
for config in configs:
full_configs.extend(edit_config(config, args.frameworks, args.no_crf, args.no_attn))
configs = full_configs
if args.verbose:
for config in configs:
pprint(config)
print()
print()
steps = len(configs)
pg = create_progress_bar(steps)
for config in configs:
write_config = deepcopy(config)
config['train']['epochs'] = args.trials
task_name = config['task']
system_info = m.dict()
p = Process(
target=run_model,
args=(
system_info,
config,
logs,
settings,
datasets,
embeddings,
task_name,
dir_,
int(args.gpu)
)
)
p.start()
pid = p.pid
p.join()
log_file = os.path.join(dir_, 'timing-{}.log'.format(pid))
speeds = parse_logs(log_file)
save_data(conn, speeds, write_config, system_info)
pg.update()
pg.done()
finally:
shutil.rmtree(dir_)
def _train(self, loader, steps=0, **kwargs):
"""Train an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train. We do use a `feed_dict` for passing the `TRAIN_FLAG` in either case
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
:return: Metrics
"""
SET_TRAIN_FLAG(True)
reporting_fns = kwargs.get('reporting_fns', [])
pg = create_progress_bar(steps)
epoch_loss = tf.Variable(0.0)
epoch_div = tf.Variable(0, dtype=tf.int32)
nstep_loss = tf.Variable(0.0)
nstep_div = tf.Variable(0, dtype=tf.int32)
self.nstep_start = time.time()
@tf.function
def _train_step(inputs):
features, y = inputs
loss = self.optimizer.update(self.model, features, y)
batchsz = get_shape_as_list(y)[0]
report_loss = loss * batchsz
return report_loss, batchsz
with autograph_options({
"function_optimization": False,
"layout_optimizer": False
}):
for inputs in pg(loader):
step_report_loss, step_batchsz = _train_step(inputs)
epoch_loss.assign_add(step_report_loss)
nstep_loss.assign_add(step_report_loss)
epoch_div.assign_add(step_batchsz)
nstep_div.assign_add(step_batchsz)
step = self.optimizer.global_step.numpy() + 1
if step % self.nsteps == 0:
metrics = self.calc_metrics(nstep_loss.numpy(),
nstep_div.numpy())
self.report(step, metrics, self.nstep_start, 'Train',
'STEP', reporting_fns, self.nsteps)
nstep_loss.assign(0.0)
nstep_div.assign(0)
self.nstep_start = time.time()
epoch_loss = epoch_loss.numpy()
epoch_div = epoch_div.numpy()
metrics = self.calc_metrics(epoch_loss, epoch_div)
return metrics
def test(self, ts, steps=0, **kwargs):
"""Method that evaluates on some data. There are 2 modes this can run in, `feed_dict` and `dataset`
In `feed_dict` mode, the model cycles the test data batch-wise and feeds each batch in with a `feed_dict`.
In `dataset` mode, the data is still passed in to this method, but it is not passed in a `feed_dict` and is
mostly superfluous since the features are grafted right onto the graph. However, we do use it for supplying
the ground truth, ids and text, so it is essential that the caller does not shuffle the data
:param ts: The test set
:param conll_output: (`str`) An optional file output
:param txts: A list of text data associated with the encoded batch
:param dataset: (`bool`) Is this using `tf.dataset`s
:return: The metrics
"""
total_correct = total_sum = 0
gold_spans = []
pred_spans = []
handle = None
if kwargs.get("conll_output") is not None and kwargs.get(
'txts') is not None:
handle = open(kwargs.get("conll_output"), "w")
try:
pg = create_progress_bar(steps)
metrics = {}
for (features, y), batch in pg(
zip_longest(ts, kwargs.get('batches', []), fillvalue={})):
correct, count, golds, guesses = self.process_batch(
features,
y,
handle=handle,
txts=kwargs.get("txts"),
ids=batch.get("ids"))
total_correct += correct
total_sum += count
gold_spans.extend(golds)
pred_spans.extend(guesses)
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
metrics['f1'] = span_f1(gold_spans, pred_spans)
metrics['acc'] = total_acc
if self.verbose:
conll_metrics = per_entity_f1(gold_spans, pred_spans)
conll_metrics['acc'] = total_acc * 100
conll_metrics['tokens'] = total_sum
logger.info(conlleval_output(conll_metrics))
finally:
if handle is not None:
handle.close()
return metrics
def _train(self, ts):
total_loss = 0
steps = len(ts)
metrics = {}
pg = create_progress_bar(steps)
for batch_dict in ts:
feed_dict = self.model.make_input(batch_dict, do_dropout=True)
_, step, lossv = self.model.sess.run([self.train_op, self.global_step, self.loss], feed_dict=feed_dict)
total_loss += lossv
pg.update()
pg.done()
metrics['avg_loss'] = float(total_loss)/steps
return metrics
def test(self, vs, reporting_fns, phase='Valid', dataset=True):
"""Run an epoch of testing over the dataset
If we are using a `tf.dataset`-based `fit_func`, we will just
cycle the number of steps and let the `dataset` yield new batches.
If we are using `feed_dict`s, we convert each batch from the `DataFeed`
and pass that into TF as the `feed_dict`
:param vs: A validation set
:param reporting_fns: Reporting hooks
:param phase: The phase of evaluation (`Test`, `Valid`)
:param dataset: (`bool`) Are we using `tf.dataset`s
:return: Metrics
"""
if phase == 'Test' and not dataset:
return self._evaluate(vs, reporting_fns)
self.valid_epochs += 1
total_loss = 0
total_toks = 0
preds = []
golds = []
start = time.time()
pg = create_progress_bar(len(vs))
for batch_dict in pg(vs):
if dataset:
lossv, top_preds = self.model.sess.run(
[self.test_loss, self.model.decoder.best])
else:
feed_dict = self.model.make_input(batch_dict)
lossv, top_preds = self.model.sess.run(
[self.test_loss, self.model.decoder.best],
feed_dict=feed_dict)
toks = self._num_toks(batch_dict['tgt_lengths'])
total_loss += lossv * toks
total_toks += toks
preds.extend(convert_seq2seq_preds(top_preds.T, self.tgt_rlut))
golds.extend(
convert_seq2seq_golds(batch_dict['tgt'],
batch_dict['tgt_lengths'],
self.tgt_rlut))
metrics = self.calc_metrics(total_loss, total_toks)
metrics['bleu'] = bleu(preds, golds, self.bleu_n_grams)[0]
self.report(self.valid_epochs, metrics, start, phase, 'EPOCH',
reporting_fns)
return metrics
def _train(self, loader, dataset=True, **kwargs):
"""Train an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train. We do use a `feed_dict` for passing the `TRAIN_FLAG` in either case
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
:return: Metrics
"""
if self.ema:
self.sess.run(self.ema_restore)
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_div = 0
steps = len(loader)
pg = create_progress_bar(steps)
for batch_dict in pg(loader):
if dataset:
_, step, lossv = self.sess.run(
[self.train_op, self.global_step, self.loss],
feed_dict={TRAIN_FLAG(): 1})
else:
feed_dict = self.model.make_input(batch_dict, True)
_, step, lossv = self.sess.run(
[self.train_op, self.global_step, self.loss],
feed_dict=feed_dict)
batchsz = self._get_batchsz(batch_dict)
report_lossv = lossv * batchsz
epoch_loss += report_lossv
epoch_div += batchsz
self.nstep_agg += report_lossv
self.nstep_div += batchsz
if (step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(step + 1, metrics, self.nstep_start, 'Train',
'STEP', reporting_fns, self.nsteps)
self.reset_nstep()
metrics = self.calc_metrics(epoch_loss, epoch_div)
return metrics
def _evaluate(self, es, reporting_fns):
self.model.train = False
pg = create_progress_bar(len(es))
preds = []
golds = []
start = time.time()
for batch_dict in pg(es):
tgt = batch_dict['tgt']
tgt_lens = batch_dict['tgt_lengths']
pred = [p[0] for p in self.model.predict(batch_dict)]
preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut))
metrics = {'bleu': bleu(preds, golds)[0]}
self.report(0, metrics, start, 'Test', 'EPOCH', reporting_fns)
def _evaluate(self, es, reporting_fns):
"""Run the model with beam search and report Bleu."""
pg = create_progress_bar(len(es))
preds = []
golds = []
start = time.time()
for batch_dict in pg(es):
tgt = batch_dict.pop('tgt')
tgt_lens = batch_dict.pop('tgt_lengths')
pred = [p[0] for p in self.model.predict(batch_dict)]
preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut))
metrics = {'bleu': bleu(preds, golds)[0]}
self.report(0, metrics, start, 'Test', 'EPOCH', reporting_fns)
return metrics
def _test(self, loader, **kwargs):
"""Test an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train.
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
:return: Metrics
"""
if self.ema:
self.sess.run(self.ema_load)
use_dataset = kwargs.get('dataset', True)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for i, batch_dict in enumerate(pg(loader)):
y = batch_dict['y']
if use_dataset:
guess, lossv = self.sess.run([self.model.best, self.test_loss])
else:
feed_dict = self.model.make_input(batch_dict, False)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = len(guess)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _step(self,
loader,
update,
log,
reporting_fns,
verbose=None,
output=None,
txts=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
preds, losses, ys = [], [], []
dy.renew_cg()
for i, batch_dict in enumerate(pg(loader), 1):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.forward(inputs)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
preds, losses, ys = [], [], []
dy.renew_cg()
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
epoch_loss += loss.npvalue().item() * batchsz
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def test(self, ts, conll_output=None, txts=None, dataset=True):
"""Method that evaluates on some data. There are 2 modes this can run in, `feed_dict` and `dataset`
In `feed_dict` mode, the model cycles the test data batch-wise and feeds each batch in with a `feed_dict`.
In `dataset` mode, the data is still passed in to this method, but it is not passed in a `feed_dict` and is
mostly superfluous since the features are grafted right onto the graph. However, we do use it for supplying
the ground truth, ids and text, so it is essential that the caller does not shuffle the data
:param ts: The test set
:param conll_output: (`str`) An optional file output
:param txts: A list of text data associated with the encoded batch
:param dataset: (`bool`) Is this using `tf.dataset`s
:return: The metrics
"""
total_correct = total_sum = 0
gold_spans = []
pred_spans = []
steps = len(ts)
pg = create_progress_bar(steps)
metrics = {}
# Only if they provide a file and the raw txts, we can write CONLL file
handle = None
if conll_output is not None and txts is not None:
handle = open(conll_output, "w")
try:
for batch_dict in pg(ts):
correct, count, golds, guesses = self.process_batch(batch_dict, handle, txts, dataset)
total_correct += correct
total_sum += count
gold_spans.extend(golds)
pred_spans.extend(guesses)
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
metrics['f1'] = span_f1(gold_spans, pred_spans)
metrics['acc'] = total_acc
if self.verbose:
conll_metrics = per_entity_f1(gold_spans, pred_spans)
conll_metrics['acc'] = total_acc * 100
conll_metrics['tokens'] = total_sum
logger.info(conlleval_output(conll_metrics))
finally:
if handle is not None:
handle.close()
return metrics
def _evaluate(self, es, reporting_fns):
"""Run the model with beam search and report Bleu."""
pg = create_progress_bar(len(es))
preds = []
golds = []
start = time.time()
for batch_dict in pg(es):
tgt = batch_dict.pop('tgt')
tgt_lens = batch_dict.pop('tgt_lengths')
pred = [p[0] for p in self.model.predict(batch_dict)]
preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut))
metrics = {'bleu': bleu(preds, golds)[0]}
self.report(
0, metrics, start, 'Test', 'EPOCH', reporting_fns
)
return metrics
def _evaluate(self, es, reporting_fns):
self.model.train = False
pg = create_progress_bar(len(es))
preds = []
golds = []
start = time.time()
for batch_dict in pg(es):
tgt = batch_dict['tgt']
tgt_lens = batch_dict['tgt_lengths']
pred = [p[0] for p in self.model.predict(batch_dict)]
preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(tgt, tgt_lens, self.tgt_rlut))
metrics = {'bleu': bleu(preds, golds)[0]}
self.report(
0, metrics, start, 'Test', 'EPOCH', reporting_fns
)
return metrics
def _test(self, loader):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.model.labels)
for batch_dict in loader:
truth = batch_dict.pop('y')
inputs = self.model.make_input(batch_dict)
pred = self.model.impl.predict_on_batch(inputs)
guess = np.argmax(pred, axis=-1)
cm.add_batch(truth, guess)
pg.update()
pg.done()
test_metrics = cm.get_all_metrics()
#for k, v in test_metrics.items():
# test_metrics[k] /= steps
return test_metrics
def _train(self, loader):
train_metrics = {}
steps = len(loader)
pg = create_progress_bar(steps)
for batch_dict in loader:
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
metrics = self.model.impl.train_on_batch(inputs, y)
for i in range(len(self.model.impl.metrics_names)):
name = self.model.impl.metrics_names[i]
name = ClassifyTrainerKeras.METRIC_REMAP.get(name, name)
train_metrics[name] = train_metrics.get(name, 0) + metrics[i]
pg.update()
for k, v in train_metrics.items():
train_metrics[k] /= steps
pg.done()
return train_metrics
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
preds, losses, ys = [], [], []
dy.renew_cg()
for i, batch_dict in enumerate(pg(loader), 1):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.forward(inputs)
preds.append(pred)
loss = self.model.loss(pred, y)
losses.append(loss)
ys.append(y)
if i % self.autobatchsz == 0:
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
preds, losses, ys = [], [], []
dy.renew_cg()
loss = dy.average(losses)
preds = dy.concatenate_cols(preds)
batchsz = len(losses)
epoch_loss += loss.npvalue().item() * batchsz
epoch_div += batchsz
_add_to_cm(cm, np.array(ys), preds.npvalue())
update(loss)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def test(self, vs, reporting_fns, phase):
if phase == 'Test':
return self._evaluate(vs, reporting_fns)
self.model.train = False
total_loss = total_toks = 0
steps = len(vs)
self.valid_epochs += 1
preds = []
golds = []
start = time.time()
pg = create_progress_bar(steps)
for batch_dict in pg(vs):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
tgt = inputs.pop('tgt')
tgt_lens = batch_dict['tgt_lengths']
output = self.model.forward(inputs)
loss = self._loss(output, tgt, tgt_lens)
toks = self._num_toks(tgt_lens)
loss_val = loss.npvalue().item()
total_loss += loss_val * toks
total_toks += toks
pred = [p[0] for p in self.model.predict(batch_dict, beam=1)]
preds.extend(convert_seq2seq_preds(pred, self.tgt_rlut))
golds.extend(convert_seq2seq_golds(tgt.T, tgt_lens, self.tgt_rlut))
metrics = self.calc_metrics(total_loss, total_toks)
metrics['bleu'] = bleu(preds, golds)[0]
self.report(
self.valid_epochs, metrics, start,
phase, 'EPOCH', reporting_fns
)
return metrics
def _test(self, loader, **kwargs):
self.model.eval()
total_loss = 0
total_norm = 0
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
verbose = kwargs.get("verbose", None)
for batch_dict in pg(loader):
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
total_loss += loss.item() * batchsz
total_norm += batchsz
_add_to_cm(cm, y, pred)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _train(self, loader, **kwargs):
self.model.train()
reporting_fns = kwargs.get('reporting_fns', [])
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
self.optimizer.zero_grad()
example = self._make_input(batch_dict)
y = example.pop('y')
pred = self.model(example)
loss = self.crit(pred, y)
batchsz = self._get_batchsz(batch_dict)
report_loss = loss.item() * batchsz
epoch_loss += report_loss
epoch_div += batchsz
self.nstep_agg += report_loss
self.nstep_div += batchsz
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
_add_to_cm(cm, y, pred)
self.optimizer.step()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fns, self.nsteps
)
self.reset_nstep()
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
return metrics
def _report_hook(count, block_size, total_size):
if Context.pg is None:
length = int((total_size + block_size - 1) / float(block_size)) if total_size != -1 else 1
Context.pg = create_progress_bar(length)
Context.pg.update()
def _train(self, ts, **kwargs):
self.model.train = True
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_norm = 0
auto_norm = 0
metrics = {}
steps = len(ts)
last = steps
losses = []
i = 1
pg = create_progress_bar(steps)
dy.renew_cg()
for batch_dict in pg(ts):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.compute_unaries(inputs)
bsz = self._get_batchsz(y)
if self.autobatchsz is None:
losses = self.model.loss(pred, y)
loss = dy.mean_batches(losses)
lossv = loss.npvalue().item()
report_loss = lossv * bsz
epoch_loss += report_loss
epoch_norm += bsz
self.nstep_agg += report_loss
self.nstep_div += bsz
loss.backward()
self.optimizer.update()
dy.renew_cg()
# TODO: Abstract this somewhat, or else once we have a batched tagger have 2 trainers
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fns, self.nsteps
)
self.reset_nstep()
else:
loss = self.model.loss(pred, y)
losses.append(loss)
self.nstep_div += bsz
epoch_norm += bsz
auto_norm += bsz
if i % self.autobatchsz == 0 or i == last:
loss = dy.average(losses)
lossv = loss.npvalue().item()
loss.backward()
self.optimizer.update()
report_loss = lossv * auto_norm
epoch_loss += report_loss
self.nstep_agg += report_loss
losses = []
dy.renew_cg()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fnsa, self.nsteps
)
self.reset_nstep()
auto_norm = 0
i += 1
metrics = self.calc_metrics(epoch_loss, epoch_norm)
return metrics