def test_assert_shapes() -> None:
num_modes, future_len, num_coords = 4, 12, 2
gt = np.random.randn(future_len, num_coords)
pred = np.random.randn(num_modes, future_len, num_coords)
avail = np.ones(future_len)
conf = np.random.rand(num_modes)
conf /= np.sum(conf, axis=-1, keepdims=True)
# test un-normalised conf
with pytest.raises(AssertionError):
conf_un = np.random.rand(4)
_assert_shapes(gt, pred, conf_un, avail)
# test single pred with no axis
with pytest.raises(AssertionError):
_assert_shapes(gt, pred[0], conf, avail)
# NLL shape must be ()
assert neg_multi_log_likelihood(gt, pred, conf, avail).shape == ()
# RMSE shape must be ()
assert rmse(gt, pred, conf, avail).shape == ()
# prob_true_mode shape must be (M)
assert prob_true_mode(gt, pred, conf, avail).shape == (num_modes,)
# displace_t shape must be (T)
assert time_displace(gt, pred, conf, avail).shape == (future_len,)
def compute_metric(self, batch, outputs):
target_availabilities = batch["target_availabilities"].unsqueeze(-1)
targets = batch["target_positions"]
outputs = outputs.reshape(targets.shape)
eval_metric = 0
for target, output, avail in zip(targets, outputs,
target_availabilities):
eval_metric += neg_multi_log_likelihood(
target.cpu().numpy(),
output.unsqueeze(0).detach().cpu().numpy(), np.ones(1),
avail.squeeze(1).cpu().numpy())
return torch.tensor(eval_metric)
def test_neg_multi_log_likelihood_known_results() -> None:
# below M=2, T=3, C=1 and CONF=[1,1,1]
num_modes, future_len, num_coords = 2, 3, 1
avail = np.ones(future_len)
gt = np.zeros((future_len, num_coords))
pred = np.zeros((num_modes, future_len, num_coords))
pred[0] = [[0], [0], [0]]
pred[1] = [[10], [10], [10]]
# single mode, one 100% right
confs = np.asarray((1, 0))
assert np.allclose(neg_multi_log_likelihood(gt, pred, confs, avail), 0)
assert np.allclose(rmse(gt, pred, confs, avail), 0)
# two equal modes, one 100% right
confs = np.asarray((0.5, 0.5))
assert np.allclose(neg_multi_log_likelihood(gt, pred, confs, avail), 0.69314, atol=1e-4)
assert np.allclose(rmse(gt, pred, confs, avail), np.sqrt(2 * 0.69314 / future_len), atol=1e-4)
# two equal modes, answer in between
gt = np.full((future_len, num_coords), 5)
confs = np.asarray((0.5, 0.5))
assert np.allclose(neg_multi_log_likelihood(gt, pred, confs, avail), 37.5, atol=1e-4)
assert np.allclose(rmse(gt, pred, confs, avail), np.sqrt(2 * 37.5 / future_len), atol=1e-4)
# two modes, one 50% right = answer in between
gt = np.full((future_len, num_coords), 5)
confs = np.asarray((1, 0))
assert np.allclose(neg_multi_log_likelihood(gt, pred, confs, avail), 37.5, atol=1e-4)
assert np.allclose(rmse(gt, pred, confs, avail), np.sqrt(2 * 37.5 / future_len), atol=1e-4)
# Example 5
gt = np.zeros((future_len, num_coords))
gt[1, 0] = 10
confs = np.asarray((1, 0))
assert np.allclose(neg_multi_log_likelihood(gt, pred, confs, avail), 50, atol=1e-4)
assert np.allclose(rmse(gt, pred, confs, avail), np.sqrt(2 * 50 / future_len), atol=1e-4)
# Example 6
gt = np.zeros((future_len, num_coords))
gt[1, 0] = 10
confs = np.asarray((0.5, 0.5))
assert np.allclose(neg_multi_log_likelihood(gt, pred, confs, avail), 50.6931, atol=1e-4)
assert np.allclose(rmse(gt, pred, confs, avail), np.sqrt(2 * 50.6931 / future_len), atol=1e-4)
# Test overflow resistance in two situations
confs = np.asarray((0.5, 0.5))
pred[0] = [[1000], [1000], [1000]]
pred[1] = [[1000], [1000], [1000]]
gt = np.zeros((future_len, num_coords))
assert not np.isinf(neg_multi_log_likelihood(gt, pred, confs, avail))
assert not np.isinf(rmse(gt, pred, confs, avail))
# this breaks also max-version if confidence is not included in exp
confs = np.asarray((1.0, 0.0))
pred[0] = [[100000], [1000], [1000]]
pred[1] = [[1000], [1000], [1000]]
gt = np.zeros((future_len, num_coords))
assert not np.isinf(neg_multi_log_likelihood(gt, pred, confs, avail))
assert not np.isinf(rmse(gt, pred, confs, avail))
n_coords = 2
n_modes = 3
gt = np.random.uniform(-1.0, 1.0, (batchsize, future_len, n_coords))
pred = np.broadcast_to(
gt[:, None, :, :],
(batchsize, n_modes, future_len, n_coords)) + np.random.uniform(
-0.2, 0.2, (n_modes, future_len, n_coords))
confidences = np.random.uniform(0.0, 1.0, (batchsize, n_modes))
confidences /= np.sum(confidences, axis=1, keepdims=True)
avails = (np.random.uniform(0.0, 1.0,
(batchsize, future_len)) > 0.3).astype(
np.float64)
value_numpy = [
neg_multi_log_likelihood(gt[i], pred[i], confidences[i], avails[i])
for i in range(batchsize)
]
value_numpy_mean = np.mean(value_numpy)
value_torch = torch.tensor([
pytorch_neg_multi_log_likelihood(torch.tensor(gt[i]),
torch.tensor(pred[i]),
torch.tensor(confidences[i]),
torch.tensor(avails[i]))
for i in range(batchsize)
])
value_torch_mean = torch.mean(value_torch)
print("value_numpy: ", value_numpy, value_numpy_mean)
print("value_torch: ", value_torch, value_torch_mean)
error = pytorch_neg_multi_log_likelihood_batch(torch.tensor(gt),