def _infer(self, modeldir, last_step, eval_section): self.logger.log("Inferring...") orig_data = registry.construct("dataset", self.config["data"][eval_section]) preproc_data: List[RATPreprocItem] = self.model_preproc.dataset( eval_section) self.model.eval() with torch.no_grad(): if isinstance(self.model.preproc.transition_system, SpiderTransitionSystem): # Orig data only used with SpiderTransitionSystem assert len(orig_data) == len(preproc_data) inferred_lines = list( self._inner_infer( model=self.model, orig_data=orig_data, preproc_data=preproc_data, nproc=self.config["train"]["eval_nproc"], beam_size=self.config["train"]["eval_beam_size"], decode_max_time_step=self.config["train"] ["eval_decode_max_time_step"], )) self.model.train() with open(f"{modeldir}/output-{last_step}", "w") as output_dst: for line in inferred_lines: output_dst.write(line) if isinstance(self.model.preproc.transition_system, SpiderTransitionSystem): return evaluate_default(orig_data, load_from_lines(inferred_lines)) else: raise NotImplementedError
def main(args=None, logdir_suffix: List[str] = None): parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) parser.add_argument("--config-args") parser.add_argument("--section", required=True) parser.add_argument("--inferred", required=True) parser.add_argument("--output-eval") parser.add_argument("--logdir") parser.add_argument("--evaluate-beams-individually", action="store_true") args, _ = parser.parse_known_args(args) if logdir_suffix: args.logdir = os.path.join(args.logdir, *logdir_suffix) real_logdir, metrics = evaluation.compute_metrics( args.config, args.config_args, args.section, list(evaluation.load_from_lines(open(args.inferred))), args.logdir, evaluate_beams_individually=args.evaluate_beams_individually, ) if args.output_eval: if real_logdir: output_path = args.output_eval.replace("__LOGDIR__", real_logdir) else: output_path = args.output_eval with open(output_path, "w") as f: json.dump(metrics, f) print("Wrote eval results to {}".format(output_path)) else: print(metrics)
def load_outputs( experiment_path: str, output_file: str, trial_ids: Iterable[int] = None ) -> Dict[int, Any]: if trial_ids is None: trial_ids = [int(trial_id) for trial_id in os.listdir(experiment_path)] all_outputs = {} for trial_id in trial_ids: path = f"{experiment_path}/{trial_id}/{output_file}" with open(path) as src: all_outputs[trial_id] = list(load_from_lines(list(src))) return all_outputs