def evaluate(self, dataset, encoder, dir):
interpreter = self.interpreter()
model = self.prepare_model(encoder)
eval(
os.path.join(dir, "RL"), dir,
dataset,
model,
self.prepare_synthesizer(model, encoder, interpreter,
rollout=False),
{}, top_n=[],
)
return torch.load(os.path.join(dir, "result.pt"))
def evaluate(self, dataset, encoder, dir):
with tempfile.TemporaryDirectory() as tmpdir:
interpreter = self.interpreter()
model = self.prepare_model(encoder)
eval(
dir, tmpdir, dir,
dataset,
model,
self.prepare_synthesizer(model, encoder, interpreter,
rollout=False),
{}, top_n=[],
)
return torch.load(os.path.join(dir, "result.pt"))
def evaluate(self, qencoder, aencoder, dir):
model = self.prepare_model(qencoder, aencoder)
eval(
dir,
dir,
test_dataset,
model,
self.prepare_synthesizer(model, qencoder, aencoder),
{
"accuracy":
use_environment(
metric=Accuracy(),
in_keys=["actual", ["ground_truth", "expected"]],
value_key="actual")
},
top_n=[5],
)
return torch.load(os.path.join(dir, "result.pt"))