def report_func(epoch, batch, num_batches, progress_step, start_time, lr,
report_stats):
"""
This is batch-level traing progress report function.
Args:
epoch(int): current epoch count.
batch(int): current batch count.
num_batches(int): total number of batches.
progress_step(int): the progress step.
start_time(float): last report time.
lr(float): current learning rate.
report_stats(Statistics): old Statistics instance.
Returns:
report_stats(Statistics): updated Statistics instance.
"""
if batch % opt.report_every == -1 % opt.report_every:
report_stats.output(epoch, batch + 1, num_batches, start_time)
if opt.exp_host:
report_stats.log("progress", experiment, lr)
if opt.tensorboard:
# Log the progress using the number of batches on the x-axis.
report_stats.log_tensorboard("progress", writer, lr, progress_step)
report_stats = tools.Statistics()
return report_stats
def test_pickle_logbook(self):
stats = tools.Statistics()
logbook = tools.Logbook()
stats.register("mean", numpy.mean)
record = stats.compile([1,2,3,4,5,6,8,9,10])
logbook.record(**record)
stats_s = pickle.dumps(logbook)
logbook_r = pickle.loads(stats_s)
self.assertEqual(logbook, logbook_r, "Unpickled logbook != pickled logbook")
def _stats(self, xent, kl, scores, target):
"""
Args:
loss (:obj:`FloatTensor`): the loss computed by the loss criterion.
scores (:obj:`FloatTensor`): a score for each possible output
target (:obj:`FloatTensor`): true targets
Returns:
:obj:`Statistics` : statistics for this batch.
"""
pred = scores.max(1)[1]
non_padding = target.ne(self.padding_idx)
num_correct = pred.eq(target) \
.masked_select(non_padding) \
.sum()
return tools.Statistics(xent.item(), kl.item(), non_padding.sum().item(), num_correct.item())
def sharded_compute_loss(self, batch, output, attns,
cur_trunc, trunc_size, shard_size,
normalization, dist_info=None,
output_baseline=None):
"""Compute the forward loss and backpropagate. Computation is done
with shards and optionally truncation for memory efficiency.
Also supports truncated BPTT for long sequences by taking a
range in the decoder output sequence to back propagate in.
Range is from `(cur_trunc, cur_trunc + trunc_size)`.
Note sharding is an exact efficiency trick to relieve memory
required for the generation buffers. Truncation is an
approximate efficiency trick to relieve the memory required
in the RNN buffers.
Args:
batch (batch) : batch of labeled examples
output (:obj:`FloatTensor`) :
output of decoder model `[tgt_len x batch x hidden]`
attns (dict) : dictionary of attention distributions
`[tgt_len x batch x src_len]`
cur_trunc (int) : starting position of truncation window
trunc_size (int) : length of truncation window
shard_size (int) : maximum number of examples in a shard
normalization (int) : Loss is divided by this number
Returns:
:obj:`tools.Statistics`: validation loss statistics
"""
batch_stats = tools.Statistics()
range_ = (cur_trunc, cur_trunc + trunc_size)
shard_state = self._make_shard_state(batch, output, range_, attns, dist_info=dist_info, output_baseline=output_baseline)
if dist_info is not None:
self.dist_type = dist_info.p.dist_type
for shard in shards(shard_state, shard_size):
loss, stats = self._compute_loss(batch, **shard)
loss.div(normalization).backward()
batch_stats.update(stats)
return batch_stats