def _infer(self, modeldir, last_step, eval_section):
self.logger.log("Inferring...")
orig_data = registry.construct("dataset",
self.config["data"][eval_section])
preproc_data: List[RATPreprocItem] = self.model_preproc.dataset(
eval_section)
self.model.eval()
with torch.no_grad():
if isinstance(self.model.preproc.transition_system,
SpiderTransitionSystem):
# Orig data only used with SpiderTransitionSystem
assert len(orig_data) == len(preproc_data)
inferred_lines = list(
self._inner_infer(
model=self.model,
orig_data=orig_data,
preproc_data=preproc_data,
nproc=self.config["train"]["eval_nproc"],
beam_size=self.config["train"]["eval_beam_size"],
decode_max_time_step=self.config["train"]
["eval_decode_max_time_step"],
))
self.model.train()
with open(f"{modeldir}/output-{last_step}", "w") as output_dst:
for line in inferred_lines:
output_dst.write(line)
if isinstance(self.model.preproc.transition_system,
SpiderTransitionSystem):
return evaluate_default(orig_data, load_from_lines(inferred_lines))
else:
raise NotImplementedError
def main(args=None, logdir_suffix: List[str] = None):
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--config-args")
parser.add_argument("--section", required=True)
parser.add_argument("--inferred", required=True)
parser.add_argument("--output-eval")
parser.add_argument("--logdir")
parser.add_argument("--evaluate-beams-individually", action="store_true")
args, _ = parser.parse_known_args(args)
if logdir_suffix:
args.logdir = os.path.join(args.logdir, *logdir_suffix)
real_logdir, metrics = evaluation.compute_metrics(
args.config,
args.config_args,
args.section,
list(evaluation.load_from_lines(open(args.inferred))),
args.logdir,
evaluate_beams_individually=args.evaluate_beams_individually,
)
if args.output_eval:
if real_logdir:
output_path = args.output_eval.replace("__LOGDIR__", real_logdir)
else:
output_path = args.output_eval
with open(output_path, "w") as f:
json.dump(metrics, f)
print("Wrote eval results to {}".format(output_path))
else:
print(metrics)
def load_outputs(
experiment_path: str, output_file: str, trial_ids: Iterable[int] = None
) -> Dict[int, Any]:
if trial_ids is None:
trial_ids = [int(trial_id) for trial_id in os.listdir(experiment_path)]
all_outputs = {}
for trial_id in trial_ids:
path = f"{experiment_path}/{trial_id}/{output_file}"
with open(path) as src:
all_outputs[trial_id] = list(load_from_lines(list(src)))
return all_outputs