def report_timing(level=Logging.VERBOSE):
Logging(level).log("Time consumed:")
for k in sorted(_TimingHelperClass.time_records.keys()):
v = _TimingHelperClass.time_records[k]
Logging(level).log(f"> {k}: {v:f}")
Logging(level).log("------")
_TimingHelperClass.time_records = {}
def train(self) -> None:
# TODO: Incorporate `torchutils.prevent_oom`
Logging(1).log("Training start.", timestamp=self.timestamp)
while self.max_epochs == -1 or self.epoch < self.max_epochs:
iterator = self.create_data_iterator()
for batch in iterator:
self.iterations += 1
for hook in self.before_iteration_hooks:
hook(self)
record_values = self.train_step(batch)
for name, value in record_values.items():
if isinstance(value, tuple):
self.records[name].record.add(*value)
else:
self.records[name].record.add(value)
for hook in self.after_iteration_hooks:
hook(self)
if self.iterations % self.log_iters == 0:
self._print_summary(period='log')
if self.iterations % self.valid_iters == 0:
self._print_summary(period='validate')
metric = self.validate()
if len(self.validation_history) == 0 or \
(metric > max(self.validation_history) and self.metric_higher_better) or \
(metric < min(self.validation_history) and not self.metric_higher_better):
self.bad_counter = 0
self.save_model()
else:
self.bad_counter += 1
Logging(1).log(
f"{utils.ordinal(self.bad_counter)} time degradation "
f"(threshold={self.decay_threshold}).")
if self.bad_counter >= self.decay_threshold:
self.decay_times += 1
if self.decay_times > self.patience:
Logging(1).log("Early stop!", color='red')
return
self.bad_counter = 0
self.decay()
self.validation_history.append(metric)
self.epoch += 1
self._print_summary(period='epoch')
def _print_summary(self, period: str = 'log') -> None:
summary = []
for record in self.records.values():
if record.period == period:
value = record.record.value()
if record.post_compute is not None:
value = record.post_compute(value)
if isinstance(value, float):
value = f'{value:.{record.precision}f}'
summary.append((record.display, value))
record.record.clear()
if len(summary) == 0:
return
records = ', '.join(f'{name}={value}' for name, value in summary)
log_message = self.LOG_MESSAGE.format(epoch=self.epoch,
iter=self.iterations,
records=records)
Logging(1).log(log_message, timestamp=self.timestamp)