def __call__(self, manager=None): """Executes the evaluator extension. Unlike usual extensions, this extension can be executed without passing a manager object. This extension reports the performance on validation dataset using the :func:`~reporting.report` function. Thus, users can use this extension independently from any manager by manually configuring a :class:`~Reporter` object. Args: manager (~pytorch_pfn_extras.training.ExtensionsManager): Manager object that invokes this extension. It can be omitted in case of calling this extension manually. Returns: dict: Result dictionary that contains mean statistics of values reported by the evaluation function. """ # set up a reporter reporter = reporting.Reporter() if self.name is not None: prefix = self.name + '/' else: prefix = '' for name, target in self._targets.items(): reporter.add_observer(prefix + name, target) reporter.add_observers(prefix + name, target.named_modules()) with reporter: with torch.no_grad(): result = self.evaluate() reporting.report(result) return result
def set_evaluation_completed(engine: Engine) -> None: ignite_metrics: Dict[str, Any] = {} with reporting.report_scope(ignite_metrics): metrics = self.evaluator.state.metrics for metric in metrics: reporting.report( {'val/{}'.format(metric): metrics[metric]}) self.summary.add(ignite_metrics)
def dump_elapsed_time_log(self) -> None: report({ f"time.{tag}": time / n for tag, (n, time) in self.elapsed_time_log.items() }) report({ f"gpu.time.{tag}": time / n for tag, (n, time) in self.gpu_elapsed_time_log.items() }) self.elapsed_time_log = {} self.gpu_elapsed_time_log = {}
def __call__(self, manager): observation = manager.observation if not (self._numerator_key in observation and self._denominator_key in observation): return self._numerator += observation[self._numerator_key] self._denominator += observation[self._denominator_key] if self._trigger(manager): result = float(self._numerator) / self._denominator self._numerator = 0 self._denominator = 0 reporting.report({self._result_key: result})
def __call__(self, manager): """Execute the statistics extension. Collect statistics for the current state of parameters. Note that this method will merely update its statistic summary, unless the internal trigger is fired. If the trigger is fired, the summary will also be reported and then reset for the next accumulation. Args: manager (~pytorch_pfn_extras.training.ExtensionsManager): Associated manager that invoked this extension. """ statistics = {} for link in self._links: for param_name, param in link.named_parameters(): for attr_name in self._attrs: for function_name, function in self._statistics.items(): # Get parameters as a flattened one-dimensional array # since the statistics function should make no # assumption about the axes params = getattr(param, attr_name).flatten() if (self._skip_nan_params and ( torch.isnan(params).any())): value = float('nan') else: value = function(params) key = self.report_key_template.format( prefix=self._prefix + '/' if self._prefix else '', param_name=param_name, attr_name=attr_name, function_name=function_name ) if (isinstance(value, torch.Tensor) and value.numel() > 1): # Append integer indices to the keys if the # statistic function return multiple values statistics.update({'{}/{}'.format(key, i): v for i, v in enumerate(value)}) else: statistics[key] = value self._summary.add(statistics) if self._trigger(manager): reporting.report(self._summary.compute_mean()) self._summary = reporting.DictSummary() # Clear summary
def run(self, loader: Iterable[Any], *, eval_len: Optional[int] = None) -> None: """Executes the evaluation loop. Args: loader (torch.utils.data.DataLoader): A data loader for evaluation. eval_len (int, optional): The number of iterations per one evaluation epoch. """ # Note: setup_manager is done by the Trainer. self._idxs: 'queue.Queue[int]' = queue.Queue() self._inputs: 'queue.Queue[DictBatch]' = queue.Queue() self._observed: 'queue.Queue[Observation]' = queue.Queue() if eval_len is None: eval_len = len(loader) # type: ignore[arg-type] self._eval_len = eval_len self._summary = reporting.DictSummary() observation: Observation = {} self.handler.eval_loop_begin(self) self._pbar = _progress_bar('validation', self._progress_bar, eval_len) self._update = self._pbar.__enter__() loader_iter = iter(loader) with self._profile or _nullcontext() as prof: with torch.no_grad(): # type: ignore[no-untyped-call] for idx in range(eval_len): try: x = next(loader_iter) except StopIteration: break self._idxs.put(idx) self._inputs.put(x) self._observed.put(observation) with self._reporter.scope(observation): self.handler.eval_step(self, idx, x, self._complete_step) # Some of the DataLoaders might need an explicit break # since they could start cycling on their data if (idx + 1) == eval_len: break if prof is not None: prof.step() # type: ignore[no-untyped-call] # This will report to the trainer main reporter self.handler.eval_loop_end(self) reporting.report(self._summary.compute_mean())
def train_post_step(self, trainer, batch_idx, batch, outputs): """A method called after each training step. Args: trainer (Trainer): The trainer that calls this method. batch_idx (int): Number of iterations batch (dict of torch.Tensor): Input tensors of this batch. outputs (dict of torch.Tensor): Output tensors of this batch. """ # Context: Trainer # Called after train_step. for _, sm, rt in self._runtime_iterator(trainer.models): rt.train_post_step(trainer, sm, batch_idx, batch, outputs) for out in self._train_report_keys: reporting.report({"train/{}".format(out): outputs[out]})
def eval_post_step(self, evaluator, batch_idx, batch, outputs): """A method called after each evaluation step. Args: evaluator (Evaluator): The evaluator. batch_idx (int): Number of iterations already finished. batch (dict of torch.Tensor): Input tensors of this batch. complete_fn (callable): A callback function called after training step. """ # Context: Evaluator # Called after eval_step. for _, sm, rt in self._runtime_iterator(evaluator.models): rt.eval_post_step(evaluator, sm, batch_idx, batch, outputs) for out in self._eval_report_keys: reporting.report({"val/{}".format(out): outputs[out]})
def __call__(self, manager: ExtensionsManagerProtocol) -> None: if manager.is_before_training or self._trigger(manager): with self.time_summary.summary(clear=True) as s: st, additional = s stats = st.make_statistics() stats.update(additional) writer = manager.writer if self._writer is None else self._writer # report if self._report_keys is not None: reports = { f"time.{k}": v for k, v in stats.items() if k in self._report_keys } reporting.report(reports) # output the result if self._store_keys is not None: stats = { k: v for k, v in stats.items() if k in self._store_keys } stats_cpu = {k: float(v) for k, v in stats.items()} stats_cpu["epoch"] = manager.epoch stats_cpu["iteration"] = manager.iteration stats_cpu["elapsed_time"] = manager.elapsed_time # Recreate dict to fix order of logs out = OrderedDict([(k, stats_cpu[k]) for k in sorted(stats_cpu.keys())]) self._log.append(out) # write to the log file if self._log_name is not None: log_name = self._log_name.format(**out) assert self._format is not None savefun = log_report.LogWriterSaveFunc(self._format, self._append) writer( log_name, out, self._log, # type: ignore savefun=savefun, append=self._append) if self._append: self._log = []
def _complete_step( self, idx: int, outs: Any, *, is_deferred: bool = False, ) -> None: self._deferred = False # notify that the function was called c_idx = self._idxs.get() # Asure that iterations complete in order if c_idx != idx: raise RuntimeError( 'Completed a not expected iteration. ' '{} was expected but completion of {} happened'.format( c_idx, idx)) x = self._inputs.get() begin = self._times.get() observed = self._observed.get() ( record_iteration, record_run_iteration, record_train_step, ) = self._profile_records.get() # If the iteration was not deferred this is still under the # `manager.run_iteration` scope # Change the current reporter observation # To be the one to be completed if is_deferred: # Complete profiler record of `train_step` record_train_step.complete() # We want to report the previously obtained values in `train_step` cm_iter = self.manager.complete_iteration(observation=observed) cm_iter.__enter__() else: reporting.get_current_reporter().observation = observed self.manager.observation = observed self.handler.train_post_step(self, idx, x, outs) reporting.report({"elapsed_time": time.time() - begin}) if is_deferred: cm_iter.__exit__(None, None, None) # Complete profiler record of `run_iteration` and iteration record_run_iteration.complete() record_iteration.complete()
def _complete_step( self, idx: int, outs: Any, ) -> None: c_idx = self._idxs.get() # Asure that iterations complete in order if c_idx != idx: raise RuntimeError( 'Completed a not expected iteration. ' '{} was expected but completion of {} happened'.format( c_idx, idx) ) x = self._inputs.get() begin = self._times.get() ( record_iteration, record_run_iteration, record_train_step, ) = self._profile_records.get() self.handler.train_post_step(self, idx, x, outs) reporting.report({"elapsed_time": time.time() - begin})
def __call__( self, group: Optional[torch.distributed.group] = None, ) -> EvaluationResult[Code, GroundTruth]: total = {} generated = [] times = [] for n in self.top_n: t = {} for name in self.metrics.keys(): t[name] = 0.0 total[n] = t evaluate_sample: EvaluateSample[Code] = \ EvaluateSample(self.synthesizer, self.metrics, self.top_n) results: List[Result[Code, GroundTruth]] = [] rank = distributed.rank(group=group) size = distributed.size(group=group) n_sample = (len(self.dataset) + size - 1) // size logger.info( f"Evalute with {len(self.dataset)} samples w/ {size} processes " + f"({n_sample} per process)") samples = self.dataset[rank * n_sample:(rank + 1) * n_sample] results = [ evaluate_sample(elem) for elem in tqdm(total=len(samples), iterable=logger.iterable_block( "evaluate_sample", enumerate(samples))) ] gathered_results = distributed.all_gather(results) results = [] for r in gathered_results: results.extend(r) logger.info("Summarize results") for result in results: generated.append(1.0 if result.generated else 0.0) if result.generated: times.append(result.time) for n in self.top_n: m = result.metrics[n] for name in self.metrics.keys(): total[n][name] += \ m[name] if m[name] is not None else 0 total = { n: { name: value / len(self.dataset) for name, value in metric.items() } for n, metric in total.items() } r = EvaluationResult(results, total, np.mean(generated), np.mean(times)) # report for n, metric in total.items(): for name, value in metric.items(): report({f"{name}@{n}": value}) report({"generation_rate": r.generation_rate}) report({"generation_time": r.generation_time}) # logging logger.info(f"{r.metrics}") logger.info(f"generation rate: {r.generation_rate}") logger.info(f"generation time: {r.generation_time}") return r
def forward(self, **kwargs: torch.Tensor) -> torch.Tensor: losses = {key: loss.sum().reshape(1) for key, loss in kwargs.items()} report({key: loss.item() for key, loss in losses.items()}) return torch.sum(torch.cat(list(losses.values())))
def __call__(self): report({self.key: 0.0})
def __call__(self): self.model.eval() report({self.key: 0.0})
def set_evaluation_completed(engine): metrics = self.evaluator.state.metrics for metric in metrics: reporting.report({'val/{}'.format(metric): metrics[metric]}) self.cm.__exit__(None, None, None) self.summary.add(self.observation)