def test_reshape_means():
static_dim = 2
T = 10
for windows in _get_windows_set():
means = np.random.rand(T, static_dim * len(windows))
reshaped_means = G.reshape_means(means, static_dim)
assert reshaped_means.shape == (T * len(windows), static_dim)
reshaped_means2 = G.reshape_means(reshaped_means, static_dim)
# Test if call on reshaped means doesn't change anything
assert np.allclose(reshaped_means, reshaped_means2)
def test_unit_variance_mlpg():
static_dim = 2
T = 10
for windows in _get_windows_set():
means = np.random.rand(T, static_dim * len(windows))
variances = np.ones(static_dim * len(windows))
y = G.mlpg(means, variances, windows)
R = G.unit_variance_mlpg_matrix(windows, T)
y_hat = R.dot(G.reshape_means(means, static_dim))
assert np.allclose(y_hat, y)
def test_unit_variance_mlpg_gradcheck():
static_dim = 2
T = 10
for windows in _get_windows_set():
torch.manual_seed(1234)
# Meens, input for MLPG
means = Variable(torch.rand(T, static_dim * len(windows)),
requires_grad=True)
# Input for UnitVarianceMLPG
reshaped_means = G.reshape_means(
means.data.clone().numpy(), static_dim)
reshaped_means = Variable(torch.from_numpy(reshaped_means),
requires_grad=True)
# Compute MLPG matrix
R = G.unit_variance_mlpg_matrix(windows, T).astype(np.float32)
R = torch.from_numpy(R)
# UnitVarianceMLPG can take input with both means and reshaped_means
y1 = UnitVarianceMLPG(R)(means)
y2 = UnitVarianceMLPG(R)(reshaped_means)
# Unit variances
variances = torch.ones(static_dim * len(windows)
).expand(T, static_dim * len(windows))
y_hat = MLPG(variances, windows)(means)
# Make sure UnitVarianceMLPG and MLPG can get same result
# if we use unit variances
for y in [y1, y2]:
assert np.allclose(y.data.numpy(), y_hat.data.numpy())
# Grad check
inputs = (reshaped_means,)
assert gradcheck(UnitVarianceMLPG(R),
inputs, eps=1e-3, atol=1e-3)
inputs = (means,)
assert gradcheck(UnitVarianceMLPG(R),
inputs, eps=1e-3, atol=1e-3)
def test_minibatch_unit_variance_mlpg_gradcheck():
static_dim = 2
T = 5
for windows in _get_windows_set():
batch_size = 5
torch.manual_seed(1234)
# Prepare inputs
means = torch.rand(T, static_dim * len(windows))
means_expanded = means.expand(
batch_size, means.shape[0], means.shape[1])
reshaped_means = torch.from_numpy(
G.reshape_means(means.numpy(), static_dim))
reshaped_means_expanded = reshaped_means.expand(
batch_size, reshaped_means.shape[0], reshaped_means.shape[1])
# Target
y = G.mlpg(means.numpy(), np.ones(static_dim * len(windows)), windows)
y = Variable(torch.from_numpy(y), requires_grad=False)
y_expanded = y.expand(batch_size, y.size(0), y.size(1))
# Pack into variables
means = Variable(means, requires_grad=True)
means_expanded = Variable(means_expanded, requires_grad=True)
reshaped_means = Variable(reshaped_means, requires_grad=True)
reshaped_means_expanded = Variable(
reshaped_means_expanded, requires_grad=True)
# Case 1: 2d with reshaped means
R = torch.from_numpy(G.unit_variance_mlpg_matrix(windows, T))
y_hat1 = AF.unit_variance_mlpg(R, reshaped_means)
# Case 2: 3d with reshaped means
y_hat2 = AF.unit_variance_mlpg(R, reshaped_means_expanded)
for i in range(batch_size):
assert np.allclose(y_hat1.data.numpy(), y_hat2[i].data.numpy())
nn.MSELoss()(y_hat1, y).backward()
nn.MSELoss()(y_hat2, y_expanded).backward()
# Check grad consistency
for i in range(batch_size):
grad1 = reshaped_means.grad.data.numpy()
grad2 = reshaped_means_expanded.grad[i].data.numpy()
assert np.allclose(grad1, grad2)
# Case 3: 2d with non-reshaped input
y_hat3 = AF.unit_variance_mlpg(R, means)
# Case 4: 3d with non-reshaped input
y_hat4 = AF.unit_variance_mlpg(R, means_expanded)
for i in range(batch_size):
assert np.allclose(y_hat1.data.numpy(), y_hat3.data.numpy())
assert np.allclose(y_hat3.data.numpy(), y_hat4[i].data.numpy())
nn.MSELoss()(y_hat3, y).backward()
nn.MSELoss()(y_hat4, y_expanded).backward()
# Check grad consistency
for i in range(batch_size):
grad1 = means.grad.data.numpy()
grad2 = means_expanded.grad[i].data.numpy()
assert np.allclose(grad1, grad2)
def benchmark_mlpg(static_dim=59, T=100, batch_size=10, use_cuda=True):
if use_cuda and not torch.cuda.is_available():
return
windows = _get_windows_set()[-1]
np.random.seed(1234)
torch.manual_seed(1234)
means = np.random.rand(T, static_dim * len(windows)).astype(np.float32)
variances = np.ones(static_dim * len(windows))
reshaped_means = G.reshape_means(means, static_dim)
# Ppseud target
y = G.mlpg(means, variances, windows).astype(np.float32)
# Pack into variables
means = Variable(torch.from_numpy(means), requires_grad=True)
reshaped_means = Variable(torch.from_numpy(reshaped_means),
requires_grad=True)
y = Variable(torch.from_numpy(y), requires_grad=False)
criterion = nn.MSELoss()
# Case 1: MLPG
since = time.time()
for _ in range(batch_size):
y_hat = AF.mlpg(means, torch.from_numpy(variances), windows)
L = criterion(y_hat, y)
assert np.allclose(y_hat.data.numpy(), y.data.numpy())
L.backward() # slow!
elapsed_mlpg = time.time() - since
# Case 2: UnitVarianceMLPG
since = time.time()
if use_cuda:
y = y.cuda()
R = G.unit_variance_mlpg_matrix(windows, T)
R = torch.from_numpy(R)
# Assuming minibatch are zero-ppaded, we only need to create MLPG matrix
# per-minibatch, not per-utterance.
if use_cuda:
R = R.cuda()
for _ in range(batch_size):
if use_cuda:
means = means.cpu()
means = means.cuda()
y_hat = AF.unit_variance_mlpg(R, means)
L = criterion(y_hat, y)
assert np.allclose(y_hat.cpu().data.numpy(),
y.cpu().data.numpy(),
atol=1e-5)
L.backward()
elapsed_unit_variance_mlpg = time.time() - since
ratio = elapsed_mlpg / elapsed_unit_variance_mlpg
print(
"MLPG vs UnitVarianceMLPG (static_dim, T, batch_size, use_cuda) = ({}):"
.format((static_dim, T, batch_size, use_cuda)))
if ratio > 1:
s = "faster"
sys.stdout.write(OKGREEN)
else:
s = "slower"
sys.stdout.write(FAIL)
print(
"UnitVarianceMLPG, {:4f} times {}. Elapsed times {:4f} / {:4f}".format(
ratio, s, elapsed_mlpg, elapsed_unit_variance_mlpg))
print(ENDC)
