def test_step_ARSC(self,
data_path: str,
n_unlabeled=0,
n_episodes=1000,
set_type="test"):
assert set_type in ("dev", "test")
accuracies = list()
losses = list()
self.eval()
for i in range(n_episodes):
episode = create_ARSC_test_episode(prefix=data_path,
n_query=5,
n_unlabeled=n_unlabeled,
set_type=set_type)
with torch.no_grad():
if n_unlabeled:
loss, loss_dict = self.loss_softkmeans(episode)
else:
loss, loss_dict = self.loss(episode)
accuracies.append(loss_dict["acc"])
losses.append(loss_dict["loss"])
return {"loss": np.mean(losses), "acc": np.mean(accuracies)}
def test_step_ARSC(self, data_path: str, n_episodes=1000, set_type="test"):
assert set_type in ("dev", "test")
metrics = collections.defaultdict(list)
self.eval()
for i in range(n_episodes):
episode = create_ARSC_test_episode(prefix=data_path,
n_query=5,
set_type=set_type)
with torch.no_grad():
loss, loss_dict = self.loss(episode)
for key, value in loss_dict["metrics"].items():
metrics[key].append(value)
return {key: np.mean(value) for key, value in metrics.items()}