def __call__(self, manager=None):
"""Executes the evaluator extension.
Unlike usual extensions, this extension can be executed without passing
a manager object. This extension reports the performance on validation
dataset using the :func:`~reporting.report` function.
Thus, users can use this extension independently from any manager
by manually configuring a :class:`~Reporter` object.
Args:
manager (~pytorch_pfn_extras.training.ExtensionsManager): Manager
object that invokes this extension. It can be omitted
in case of calling this extension manually.
Returns:
dict: Result dictionary that contains mean statistics of values
reported by the evaluation function.
"""
# set up a reporter
reporter = reporting.Reporter()
if self.name is not None:
prefix = self.name + '/'
else:
prefix = ''
for name, target in self._targets.items():
reporter.add_observer(prefix + name, target)
reporter.add_observers(prefix + name, target.named_modules())
with reporter:
with torch.no_grad():
result = self.evaluate()
reporting.report(result)
return result
def set_evaluation_completed(engine: Engine) -> None:
ignite_metrics: Dict[str, Any] = {}
with reporting.report_scope(ignite_metrics):
metrics = self.evaluator.state.metrics
for metric in metrics:
reporting.report(
{'val/{}'.format(metric): metrics[metric]})
self.summary.add(ignite_metrics)
def dump_elapsed_time_log(self) -> None:
report({
f"time.{tag}": time / n
for tag, (n, time) in self.elapsed_time_log.items()
})
report({
f"gpu.time.{tag}": time / n
for tag, (n, time) in self.gpu_elapsed_time_log.items()
})
self.elapsed_time_log = {}
self.gpu_elapsed_time_log = {}
def __call__(self, manager):
observation = manager.observation
if not (self._numerator_key in observation
and self._denominator_key in observation):
return
self._numerator += observation[self._numerator_key]
self._denominator += observation[self._denominator_key]
if self._trigger(manager):
result = float(self._numerator) / self._denominator
self._numerator = 0
self._denominator = 0
reporting.report({self._result_key: result})
def __call__(self, manager):
"""Execute the statistics extension.
Collect statistics for the current state of parameters.
Note that this method will merely update its statistic summary, unless
the internal trigger is fired. If the trigger is fired, the summary
will also be reported and then reset for the next accumulation.
Args:
manager (~pytorch_pfn_extras.training.ExtensionsManager):
Associated manager that invoked this extension.
"""
statistics = {}
for link in self._links:
for param_name, param in link.named_parameters():
for attr_name in self._attrs:
for function_name, function in self._statistics.items():
# Get parameters as a flattened one-dimensional array
# since the statistics function should make no
# assumption about the axes
params = getattr(param, attr_name).flatten()
if (self._skip_nan_params
and (
torch.isnan(params).any())):
value = float('nan')
else:
value = function(params)
key = self.report_key_template.format(
prefix=self._prefix + '/' if self._prefix else '',
param_name=param_name,
attr_name=attr_name,
function_name=function_name
)
if (isinstance(value, torch.Tensor)
and value.numel() > 1):
# Append integer indices to the keys if the
# statistic function return multiple values
statistics.update({'{}/{}'.format(key, i): v for
i, v in enumerate(value)})
else:
statistics[key] = value
self._summary.add(statistics)
if self._trigger(manager):
reporting.report(self._summary.compute_mean())
self._summary = reporting.DictSummary() # Clear summary
def run(self,
loader: Iterable[Any],
*,
eval_len: Optional[int] = None) -> None:
"""Executes the evaluation loop.
Args:
loader (torch.utils.data.DataLoader):
A data loader for evaluation.
eval_len (int, optional):
The number of iterations per one evaluation epoch.
"""
# Note: setup_manager is done by the Trainer.
self._idxs: 'queue.Queue[int]' = queue.Queue()
self._inputs: 'queue.Queue[DictBatch]' = queue.Queue()
self._observed: 'queue.Queue[Observation]' = queue.Queue()
if eval_len is None:
eval_len = len(loader) # type: ignore[arg-type]
self._eval_len = eval_len
self._summary = reporting.DictSummary()
observation: Observation = {}
self.handler.eval_loop_begin(self)
self._pbar = _progress_bar('validation', self._progress_bar, eval_len)
self._update = self._pbar.__enter__()
loader_iter = iter(loader)
with self._profile or _nullcontext() as prof:
with torch.no_grad(): # type: ignore[no-untyped-call]
for idx in range(eval_len):
try:
x = next(loader_iter)
except StopIteration:
break
self._idxs.put(idx)
self._inputs.put(x)
self._observed.put(observation)
with self._reporter.scope(observation):
self.handler.eval_step(self, idx, x,
self._complete_step)
# Some of the DataLoaders might need an explicit break
# since they could start cycling on their data
if (idx + 1) == eval_len:
break
if prof is not None:
prof.step() # type: ignore[no-untyped-call]
# This will report to the trainer main reporter
self.handler.eval_loop_end(self)
reporting.report(self._summary.compute_mean())
def train_post_step(self, trainer, batch_idx, batch, outputs):
"""A method called after each training step.
Args:
trainer (Trainer): The trainer that calls this method.
batch_idx (int): Number of iterations
batch (dict of torch.Tensor): Input tensors of this batch.
outputs (dict of torch.Tensor): Output tensors of this batch.
"""
# Context: Trainer
# Called after train_step.
for _, sm, rt in self._runtime_iterator(trainer.models):
rt.train_post_step(trainer, sm, batch_idx, batch, outputs)
for out in self._train_report_keys:
reporting.report({"train/{}".format(out): outputs[out]})
def eval_post_step(self, evaluator, batch_idx, batch, outputs):
"""A method called after each evaluation step.
Args:
evaluator (Evaluator): The evaluator.
batch_idx (int): Number of iterations already finished.
batch (dict of torch.Tensor): Input tensors of this batch.
complete_fn (callable): A callback function called after
training step.
"""
# Context: Evaluator
# Called after eval_step.
for _, sm, rt in self._runtime_iterator(evaluator.models):
rt.eval_post_step(evaluator, sm, batch_idx, batch, outputs)
for out in self._eval_report_keys:
reporting.report({"val/{}".format(out): outputs[out]})
def __call__(self, manager: ExtensionsManagerProtocol) -> None:
if manager.is_before_training or self._trigger(manager):
with self.time_summary.summary(clear=True) as s:
st, additional = s
stats = st.make_statistics()
stats.update(additional)
writer = manager.writer if self._writer is None else self._writer
# report
if self._report_keys is not None:
reports = {
f"time.{k}": v
for k, v in stats.items() if k in self._report_keys
}
reporting.report(reports)
# output the result
if self._store_keys is not None:
stats = {
k: v
for k, v in stats.items() if k in self._store_keys
}
stats_cpu = {k: float(v) for k, v in stats.items()}
stats_cpu["epoch"] = manager.epoch
stats_cpu["iteration"] = manager.iteration
stats_cpu["elapsed_time"] = manager.elapsed_time
# Recreate dict to fix order of logs
out = OrderedDict([(k, stats_cpu[k])
for k in sorted(stats_cpu.keys())])
self._log.append(out)
# write to the log file
if self._log_name is not None:
log_name = self._log_name.format(**out)
assert self._format is not None
savefun = log_report.LogWriterSaveFunc(self._format,
self._append)
writer(
log_name,
out,
self._log, # type: ignore
savefun=savefun,
append=self._append)
if self._append:
self._log = []
def _complete_step(
self,
idx: int,
outs: Any,
*,
is_deferred: bool = False,
) -> None:
self._deferred = False # notify that the function was called
c_idx = self._idxs.get()
# Asure that iterations complete in order
if c_idx != idx:
raise RuntimeError(
'Completed a not expected iteration. '
'{} was expected but completion of {} happened'.format(
c_idx, idx))
x = self._inputs.get()
begin = self._times.get()
observed = self._observed.get()
(
record_iteration,
record_run_iteration,
record_train_step,
) = self._profile_records.get()
# If the iteration was not deferred this is still under the
# `manager.run_iteration` scope
# Change the current reporter observation
# To be the one to be completed
if is_deferred:
# Complete profiler record of `train_step`
record_train_step.complete()
# We want to report the previously obtained values in `train_step`
cm_iter = self.manager.complete_iteration(observation=observed)
cm_iter.__enter__()
else:
reporting.get_current_reporter().observation = observed
self.manager.observation = observed
self.handler.train_post_step(self, idx, x, outs)
reporting.report({"elapsed_time": time.time() - begin})
if is_deferred:
cm_iter.__exit__(None, None, None)
# Complete profiler record of `run_iteration` and iteration
record_run_iteration.complete()
record_iteration.complete()
def _complete_step(
self,
idx: int,
outs: Any,
) -> None:
c_idx = self._idxs.get()
# Asure that iterations complete in order
if c_idx != idx:
raise RuntimeError(
'Completed a not expected iteration. '
'{} was expected but completion of {} happened'.format(
c_idx, idx)
)
x = self._inputs.get()
begin = self._times.get()
(
record_iteration,
record_run_iteration,
record_train_step,
) = self._profile_records.get()
self.handler.train_post_step(self, idx, x, outs)
reporting.report({"elapsed_time": time.time() - begin})
def __call__(
self,
group: Optional[torch.distributed.group] = None,
) -> EvaluationResult[Code, GroundTruth]:
total = {}
generated = []
times = []
for n in self.top_n:
t = {}
for name in self.metrics.keys():
t[name] = 0.0
total[n] = t
evaluate_sample: EvaluateSample[Code] = \
EvaluateSample(self.synthesizer, self.metrics, self.top_n)
results: List[Result[Code, GroundTruth]] = []
rank = distributed.rank(group=group)
size = distributed.size(group=group)
n_sample = (len(self.dataset) + size - 1) // size
logger.info(
f"Evalute with {len(self.dataset)} samples w/ {size} processes " +
f"({n_sample} per process)")
samples = self.dataset[rank * n_sample:(rank + 1) * n_sample]
results = [
evaluate_sample(elem)
for elem in tqdm(total=len(samples),
iterable=logger.iterable_block(
"evaluate_sample", enumerate(samples)))
]
gathered_results = distributed.all_gather(results)
results = []
for r in gathered_results:
results.extend(r)
logger.info("Summarize results")
for result in results:
generated.append(1.0 if result.generated else 0.0)
if result.generated:
times.append(result.time)
for n in self.top_n:
m = result.metrics[n]
for name in self.metrics.keys():
total[n][name] += \
m[name] if m[name] is not None else 0
total = {
n: {
name: value / len(self.dataset)
for name, value in metric.items()
}
for n, metric in total.items()
}
r = EvaluationResult(results, total, np.mean(generated),
np.mean(times))
# report
for n, metric in total.items():
for name, value in metric.items():
report({f"{name}@{n}": value})
report({"generation_rate": r.generation_rate})
report({"generation_time": r.generation_time})
# logging
logger.info(f"{r.metrics}")
logger.info(f"generation rate: {r.generation_rate}")
logger.info(f"generation time: {r.generation_time}")
return r
def forward(self, **kwargs: torch.Tensor) -> torch.Tensor:
losses = {key: loss.sum().reshape(1) for key, loss in kwargs.items()}
report({key: loss.item() for key, loss in losses.items()})
return torch.sum(torch.cat(list(losses.values())))
def __call__(self):
report({self.key: 0.0})
def __call__(self):
self.model.eval()
report({self.key: 0.0})
def set_evaluation_completed(engine):
metrics = self.evaluator.state.metrics
for metric in metrics:
reporting.report({'val/{}'.format(metric): metrics[metric]})
self.cm.__exit__(None, None, None)
self.summary.add(self.observation)
