result = lm.process_batch(batch, training=True)
nancheck = result["loss"].value()
while (not isinstance(numpy.isnan(nancheck), numpy.bool_) and True in numpy.isnan(nancheck)) or \
(isinstance(numpy.isnan(nancheck), numpy.bool_) and numpy.isnan(nancheck) == True):
print "nan encountered...redoing"
sys.stdout.flush()
result = lm.process_batch(batch, training=True)
training_loss = dynet.sum_batches(result["loss"]) * dynet.scalarInput(
1. / len(batch), args.param_device)
training_loss.backward()
trainer.update()
train_accumulator.update(result)
if args.update_num % args.log_train_every_n == 0:
print "(%d) Update %6d |" % (
args.completed_epochs, args.update_num), train_accumulator.pp(
), "| Time: %4f" % (time.time() - _start), " ",
trainer.status()
print
train_accumulator = Accumulator(accs, disps)
args.temperature *= .99
_start = time.time()
if args.update_num % args.log_valid_every_n == 0:
valid_accumulator = Accumulator(accs, disps)
_start = time.time()
for v_i, v_batch in enumerate(valid_batches[:4]):
dynet.renew_cg()
result = lm.process_batch(v_batch, training=False)
valid_accumulator.update(result)
print "[[ Validation %6d | " % args.update_num, valid_accumulator.pp(
), "| Time: %4f ]]" % (time.time() - _start),
