def after_epoch(self, engine, data_loader):
if distributed.is_local_master():
self._bar.close()
self._bar = None
self._context.__exit__(None, None, None)
self._context = None
def after_epoch(self, engine: Trainer, data_loader):
if distributed.is_local_master():
current = self._summary[self._target_field].get_value(
global_step=engine.global_step)
if self._best_value is None or self._best_value < current:
self._best_value = current
self._manager.save(engine.state_dict(),
global_step=engine.global_step,
logger=self._logger)
def after_batch(self, engine: Engine, inputs: Dict[str, Any], outputs: Dict[str, Any]):
if distributed.is_local_master():
texts = []
for name in sorted(self._summary.names()):
if self._matcher.match(name) and self._summary[name].indices[-1] == engine.global_step:
texts.append(f'[{name}] = {self._summary[name].get_value():.{self._num_digits}g}')
if texts:
self._bar.set_description(', '.join(texts))
self._bar.update()
def after_epoch(self, engine: Engine, data_loader):
if distributed.is_local_master():
for name in sorted(self._summary.names()):
storage = self._summary[name]
if not storage or storage.indices[-1] != engine.global_step:
continue
if storage.item_type == SummaryItemType.ITEM_SCALAR:
self._writer.add_scalar(
name,
storage.get_value(global_step=engine.global_step),
global_step=engine.global_step)
def prior_all(self, engine: Engine):
if distributed.is_local_master():
from torch.utils.tensorboard import SummaryWriter
self._writer = SummaryWriter(log_dir=self._path,
purge_step=engine.global_step)
def after_all(self, engine: Engine):
if distributed.is_local_master():
self._writer.close()
self._writer = None
def after_epoch(self, engine: Trainer, data_loader):
if distributed.is_local_master(
) and engine.global_step % self._num_save_epochs == 0:
self._manager.save(engine.state_dict(),
global_step=engine.global_step,
logger=self._logger)
def make_session_path(self):
if distributed.is_local_master():
ensure_directory(self.current_session_path)
distributed.barrier()
def prior_epoch(self, engine, data_loader):
if distributed.is_local_master():
self._bar = tqdm.tqdm(total=len(data_loader), ncols=0, leave=False)
self._context = self._logger_group.switch_handler_context(
self._console_handler_key, CallbackHandler(self._bar.write))
self._context.__enter__()