def report(self) -> None:
this_report_time = time.time()
self.last_report_sents_since_start = self.training_task.training_state.sents_since_start
self.fractional_epoch = (self.training_task.training_state.epoch_num - 1) \
+ self.training_task.training_state.sents_into_epoch / self.training_task.cur_num_sentences()
dev_time = self.time_tracker.get_and_reset()
utils.log_readable_and_tensorboard(
template=DevLossTracker.REPORT_TEMPLATE_DEV,
args={self.dev_score.metric_name(): self.dev_score.value()},
n_iter=self.fractional_epoch,
data_name="dev",
task_name=self.name,
score=self.dev_score,
time=utils.format_time(this_report_time - self.start_time))
for score in self.aux_scores:
utils.log_readable_and_tensorboard(
template=DevLossTracker.REPORT_TEMPLATE_DEV_AUX,
args={score.metric_name(): score.value()},
n_iter=self.fractional_epoch,
data_name="dev",
task_name=self.name,
score=score)
logger.info(
DevLossTracker.REPORT_TEMPLATE_TIME_NEEDED.format(
time_needed=utils.format_time(dev_time),
extra={"task_name": self.name}))
self.aux_scores = []
def report(self, loss_data, trg) -> None:
"""
Accumulate training loss and report every REPORT_EVERY sentences.
"""
for loss_name, (loss_value, units) in loss_data.items():
self.epoch_words[loss_name] += units
self.epoch_loss[loss_name] += loss_value
self.epoch_words["__TRG__"] += self.count_trg_words(trg)
sent_num_not_report = self.training_task.training_state.sents_since_start - self.last_report_sents_since_start
should_report = sent_num_not_report >= TrainLossTracker.REPORT_EVERY \
or self.training_task.training_state.sents_into_epoch == self.training_task.cur_num_sentences()
if should_report:
fractional_epoch = (self.training_task.training_state.epoch_num - 1) \
+ self.training_task.training_state.sents_into_epoch / self.training_task.cur_num_sentences()
accum_time = self.time_tracker.get_and_reset()
rep_train_loss = sum([
self.epoch_loss[k] / self.epoch_words[k]
for k in self.epoch_loss.keys()
])
utils.log_readable_and_tensorboard(
template=TrainLossTracker.REPORT_TEMPLATE_SPEED
if accum_time else TrainLossTracker.REPORT_TEMPLATE,
args={"loss": rep_train_loss},
n_iter=fractional_epoch,
time=utils.format_time(time.time() - self.start_time),
words=self.epoch_words["__TRG__"],
data_name="train",
task_name=self.name,
words_per_sec=(self.epoch_words["__TRG__"] -
self.last_report_words) /
accum_time if accum_time else None)
if len(self.epoch_loss) > 1:
for loss_name, loss_values in self.epoch_loss.items():
utils.log_readable_and_tensorboard(
template=TrainLossTracker.REPORT_TEMPLATE_ADDITIONAL,
args={
loss_name:
loss_values / self.epoch_words[loss_name]
},
n_iter=fractional_epoch,
data_name="train",
task_name=self.name,
loss_name=loss_name,
loss=loss_values / self.epoch_words[loss_name],
)
self.last_report_words = self.epoch_words["__TRG__"]
self.last_report_sents_since_start = self.training_task.training_state.sents_since_start
def report(self, trg: Union[sent.Sequence, batchers.Batch],
loss: losses.FactoredLossVal) -> None:
"""
Accumulate training loss and report every REPORT_EVERY sentences.
"""
self.epoch_words += self.count_trg_words(trg)
self.epoch_loss += loss
sent_num_not_report = self.training_task.training_state.sents_since_start - self.last_report_sents_since_start
should_report = sent_num_not_report >= TrainLossTracker.REPORT_EVERY \
or self.training_task.training_state.sents_into_epoch == self.training_task.cur_num_sentences()
if should_report:
fractional_epoch = (self.training_task.training_state.epoch_num - 1) \
+ self.training_task.training_state.sents_into_epoch / self.training_task.cur_num_sentences()
accum_time = self.time_tracker.get_and_reset()
rep_train_loss = self.epoch_loss.sum_factors() / self.epoch_words
utils.log_readable_and_tensorboard(
template=TrainLossTracker.REPORT_TEMPLATE_SPEED
if accum_time else TrainLossTracker.REPORT_TEMPLATE,
args={"loss": rep_train_loss},
n_iter=fractional_epoch,
time=utils.format_time(time.time() - self.start_time),
words=self.epoch_words,
data_name="train",
task_name=self.name,
words_per_sec=(self.epoch_words - self.last_report_words) /
accum_time if accum_time else None)
if len(self.epoch_loss) > 1:
for loss_name, loss_values in self.epoch_loss.items():
utils.log_readable_and_tensorboard(
template=TrainLossTracker.REPORT_TEMPLATE_ADDITIONAL,
args={loss_name: loss_values / self.epoch_words},
n_iter=fractional_epoch,
data_name="train",
task_name=self.name,
loss_name=loss_name,
loss=loss_values / self.epoch_words,
)
self.last_report_words = self.epoch_words
self.last_report_sents_since_start = self.training_task.training_state.sents_since_start