def test_cp_vonneumann_entropy_mixed_state():
"""Test for cp_vonneumann_entropy on CP tensors.
This test checks that the VNE of mixed states is calculated correctly.
"""
state1 = tl.tensor([[
0.03004805, 0.42426117, 0.5483771, 0.4784077, 0.25792725, 0.34388784,
0.99927586, 0.96605812
]])
state1 = state1 / tl.norm(state1)
state2 = tl.tensor([[
0.84250089, 0.43429687, 0.26551928, 0.18262211, 0.55584835, 0.2565509,
0.33197401, 0.97741178
]])
state2 = state2 / tl.norm(state2)
mat_mixed = tl.tensor((tl.dot(tl.transpose(state1), state1) +
tl.dot(tl.transpose(state2), state2)) / 2.)
actual_vne = 0.5546
mat = parafac(tl.tensor(mat_mixed), rank=2, normalize_factors=True)
mat_unnorm = parafac(tl.tensor(mat_mixed), rank=2, normalize_factors=False)
tl_vne = cp_vonneumann_entropy(mat)
tl_vne_unnorm = cp_vonneumann_entropy(mat_unnorm)
tl.testing.assert_array_almost_equal(tl_vne, actual_vne, decimal=3)
tl.testing.assert_array_almost_equal(tl_vne_unnorm, actual_vne, decimal=3)
assert_array_almost_equal(tl_vne, actual_vne, decimal=3)
assert_array_almost_equal(tl_vne_unnorm, actual_vne, decimal=3)
def test_tt_factorized_linear():
x = tlr.random_tt((2, 7), rank=[1, 7, 1])
weights = tlr.random_tt_matrix((2, 7, 2, 7), rank=[1, 10, 1])
out = tt_factorized_linear(x, weights).to_tensor()
out = tl.reshape(out, (-1, 1))
manual_out = tl.dot(weights.to_matrix(),
tl.reshape(x.to_tensor(), (-1, 1)))
assert_array_almost_equal(out, manual_out, decimal=4)
def test_cp_copy():
shape = (3, 4, 5)
rank = 4
cp_tensor = random_cp(shape, rank)
weights, factors = cp_tensor
weights_normalized, factors_normalized = cp_normalize(cp_tensor.cp_copy())
# Check that modifying copy tensor doesn't change the original tensor
assert_array_almost_equal(cp_to_tensor((weights, factors)), cp_to_tensor(cp_tensor))
def test_cp_to_tensor():
"""Test for cp_to_tensor."""
U1 = np.reshape(np.arange(1, 10), (3, 3))
U2 = np.reshape(np.arange(10, 22), (4, 3))
U3 = np.reshape(np.arange(22, 28), (2, 3))
U4 = np.reshape(np.arange(28, 34), (2, 3))
U = [tl.tensor(t) for t in [U1, U2, U3, U4]]
true_res = tl.tensor([[[[ 46754., 51524.],
[ 52748., 58130.]],
[[ 59084., 65114.],
[ 66662., 73466.]],
[[ 71414., 78704.],
[ 80576., 88802.]],
[[ 83744., 92294.],
[ 94490., 104138.]]],
[[[ 113165., 124784.],
[ 127790., 140912.]],
[[ 143522., 158264.],
[ 162080., 178730.]],
[[ 173879., 191744.],
[ 196370., 216548.]],
[[ 204236., 225224.],
[ 230660., 254366.]]],
[[[ 179576., 198044.],
[ 202832., 223694.]],
[[ 227960., 251414.],
[ 257498., 283994.]],
[[ 276344., 304784.],
[ 312164., 344294.]],
[[ 324728., 358154.],
[ 366830., 404594.]]]])
res = cp_to_tensor((tl.ones(3), U))
assert_array_equal(res, true_res, err_msg='Khatri-rao incorrectly transformed into full tensor.')
columns = 4
rows = [3, 4, 2]
matrices = [tl.tensor(np.arange(k * columns).reshape((k, columns))) for k in rows]
tensor = cp_to_tensor((tl.ones(columns), matrices))
for i in range(len(rows)):
unfolded = unfold(tensor, mode=i)
U_i = matrices.pop(i)
reconstructed = tl.dot(U_i, tl.transpose(khatri_rao(matrices)))
assert_array_almost_equal(reconstructed, unfolded)
matrices.insert(i, U_i)
def test_vonneumann_entropy_pure_state():
"""Test for vonneumann_entropy on 2-dimensional tensors.
This test checks that pure states have a VNE of zero.
"""
state = tl.randn((8, 1))
state = state / tl.norm(state)
mat_pure = tl.dot(state, tl.transpose(state))
tl_vne = vonneumann_entropy(mat_pure)
assert_array_almost_equal(tl_vne, 0, decimal=3)
def test_cp_normalize():
shape = (3, 4, 5)
rank = 4
cp_tensor = random_cp(shape, rank)
weights, factors = cp_normalize(cp_tensor)
expected_norm = tl.ones(rank)
for f in factors:
assert_array_almost_equal(tl.norm(f, axis=0), expected_norm)
assert_array_almost_equal(cp_to_tensor((weights, factors)), cp_to_tensor(cp_tensor))
def test_tt_vonneumann_entropy_pure_state():
"""Test for tt_vonneumann_entropy TT tensors.
This test checks that pure states have a VNE of zero.
"""
state = tl.randn((8, 1))
state = state/tl.norm(state)
mat_pure = tl.reshape(tl.dot(state, tl.transpose(state)), (2, 2, 2, 2, 2, 2))
mat_pure = matrix_product_state(mat_pure, rank=(1, 3, 2, 1, 2, 3, 1))
tl_vne = tt_vonneumann_entropy(mat_pure)
assert_array_almost_equal(tl_vne, 0, decimal=3)
def test_cp_vonneumann_entropy_pure_state():
"""Test for cp_vonneumann_entropy on 2-dimensional CP tensors.
This test checks that pure states have a VNE of zero.
"""
state = tl.randn((8, 1))
state = state / tl.norm(state)
mat_pure = tl.dot(state, tl.transpose(state))
mat = parafac(mat_pure, rank=1, normalize_factors=True)
tl_vne = cp_vonneumann_entropy(mat)
assert_array_almost_equal(tl_vne, 0, decimal=3)
def test_tensor_product():
"""Test tensor_dot"""
rng = random.check_random_state(1234)
X = tl.tensor(rng.random_sample((4, 5, 6)))
Y = tl.tensor(rng.random_sample((3, 4, 7)))
tdot = tl.tensor_to_vec(tensor_dot(X, Y))
true_dot = tl.tensor_to_vec(
tenalg.outer([tl.tensor_to_vec(X),
tl.tensor_to_vec(Y)]))
testing.assert_array_almost_equal(tdot, true_dot)
def test_cp_flip_sign():
shape = (3, 4, 5)
rank = 4
cp_tensor = random_cp(shape, rank)
weights, factors = cp_flip_sign(cp_tensor)
assert_(tl.all(tl.mean(factors[1], axis=0) > 0))
assert_(tl.all(tl.mean(factors[2], axis=0) > 0))
assert_equal(cp_tensor.rank, cp_tensor.rank)
assert_array_equal(cp_tensor.weights, weights)
assert_array_almost_equal(cp_to_tensor((weights, factors)), cp_to_tensor(cp_tensor))
def test_outer_product():
"""Test outer_dot"""
rng = tl.check_random_state(1234)
X = tl.tensor(rng.random_sample((4, 5, 6)))
Y = tl.tensor(rng.random_sample((3, 4)))
Z = tl.tensor(rng.random_sample((2)))
tdot = outer([X, Y, Z])
true_dot = tenalg.tensordot(X, Y, ())
true_dot = tenalg.tensordot(true_dot, Z, ())
testing.assert_array_almost_equal(tdot, true_dot)
def test_vonneumann_entropy_mixed_state():
"""Test for vonneumann_entropy on 2-dimensional tensors.
This test checks that the VNE of mixed states is calculated correctly.
"""
state1 = tl.tensor([[0.03004805, 0.42426117, 0.5483771 , 0.4784077 , 0.25792725, 0.34388784, 0.99927586, 0.96605812]])
state1 = state1/tl.norm(state1)
state2 = tl.tensor([[0.84250089, 0.43429687, 0.26551928, 0.18262211, 0.55584835, 0.2565509 , 0.33197401, 0.97741178]])
state2 = state2/tl.norm(state2)
mat_mixed = tl.tensor((tl.dot(tl.transpose(state1), state1) + tl.dot(tl.transpose(state2), state2))/2.)
actual_vne = 0.5546
tl_vne = vonneumann_entropy(mat_mixed)
assert_array_almost_equal(tl_vne, actual_vne, decimal=3)
def test_cp_to_tensor_with_weights():
A = tl.reshape(tl.arange(1,5), (2,2))
B = tl.reshape(tl.arange(5,9), (2,2))
weigths = tl.tensor([2,-1], **tl.context(A))
out = cp_to_tensor((weigths, [A,B]))
expected = tl.tensor([[-2,-2], [6, 10]]) # computed by hand
assert_array_equal(out, expected)
(weigths, factors) = random_cp((5, 5, 5), rank=5, normalise_factors=True, full=False)
true_res = tl.dot(tl.dot(factors[0], tl.diag(weigths)),
tl.transpose(tl.tenalg.khatri_rao(factors[1:])))
true_res = tl.fold(true_res, 0, (5, 5, 5))
res = cp_to_tensor((weigths, factors))
assert_array_almost_equal(true_res, res,
err_msg='weights incorrectly incorporated in cp_to_tensor')
def test_linear_tensor_dot_tucker(factorization, factorized_linear):
in_shape = (4, 5)
in_dim = prod(in_shape)
out_shape = (6, 2)
rank = 3
batch_size = 2
tensor = tl.randn((batch_size, in_dim))
fact_weight = TensorizedTensor.new((out_shape, in_shape),
rank=rank,
factorization=factorization)
fact_weight.normal_()
full_weight = fact_weight.to_matrix()
true_res = torch.matmul(tensor, full_weight.T)
res = factorized_linear(tensor, fact_weight, transpose=True)
res = res.reshape(batch_size, -1)
testing.assert_array_almost_equal(true_res, res, decimal=5)
def test_mps_entanglement_entropy():
"""Test for the tt_mps_entanglement_entropy on TT tensors.
This test checks that the EE of both product and entangled states is calculated correctly.
"""
#Product state
mps = tl.tensor([1, 0, 0, 0, 0, 0, 0, 0], dtype=tl.float32)
mps = tl.reshape(mps, (2, 2, 2))
mps = matrix_product_state(mps, rank=[1, 2, 2, 1])
tl_ee = mps_entanglement_entropy(mps, 1)
assert_array_almost_equal(tl_ee, 0, decimal=3)
#Entangled state
mps = tl.tensor([1, 0, 0, 0, 0, 0, 0, 1], dtype=tl.float32)
mps = mps / tl.norm(mps)
mps = tl.reshape(mps, (2, 2, 2))
mps = matrix_product_state(mps, rank=[1, 2, 2, 1])
tl_ee = mps_entanglement_entropy(mps, 1)
assert_array_almost_equal(tl_ee, 1, decimal=3)
def test_batched_outer_product():
"""Test batched_outer_dot
Notes
-----
At the time of writing, MXNet doesn't support transpose
for tensors of order higher than 6
"""
rng = tl.check_random_state(1234)
batch_size = 3
X = tl.tensor(rng.random_sample((batch_size, 4, 5, 6)))
Y = tl.tensor(rng.random_sample((batch_size, 3)))
Z = tl.tensor(rng.random_sample((batch_size, 2)))
res = batched_outer([X, Y, Z])
true_res = tenalg.tensordot(X, Y, (), batched_modes=0)
true_res = tenalg.tensordot(true_res, Z, (), batched_modes=0)
testing.assert_array_almost_equal(res, true_res)
def test_batched_tensor_product():
"""Test batched-tensor_dot
Notes
-----
At the time of writing, MXNet doesn't support transpose
for tensors of order higher than 6
"""
rng = random.check_random_state(1234)
batch_size = 3
X = tl.tensor(rng.random_sample((batch_size, 4, 5, 6)))
Y = tl.tensor(rng.random_sample((batch_size, 3, 7)))
tdot = tl.unfold(batched_tensor_dot(X, Y), 0)
for i in range(batch_size):
true_dot = tl.tensor_to_vec(
tenalg.outer([tl.tensor_to_vec(X[i]),
tl.tensor_to_vec(Y[i])]))
testing.assert_array_almost_equal(tdot[i], true_dot)
def test_unfolding_dot_khatri_rao():
"""Test for unfolding_dot_khatri_rao
Check against other version check sparse safe
"""
shape = (10, 10, 10, 4)
rank = 5
tensor = tl.tensor(np.random.random(shape))
weights, factors = random_cp(shape=shape, rank=rank,
full=False, normalise_factors=True)
for mode in range(tl.ndim(tensor)):
# Version forming explicitely the khatri-rao product
unfolded = unfold(tensor, mode)
kr_factors = khatri_rao(factors, weights=weights, skip_matrix=mode)
true_res = tl.dot(unfolded, kr_factors)
# Efficient sparse-safe version
res = unfolding_dot_khatri_rao(tensor, (weights, factors), mode)
assert_array_almost_equal(true_res, res, decimal=3)
def test_FactorizedLinear(factorization):
random_state = 12345
rng = tl.check_random_state(random_state)
batch_size = 2
in_features = 9
in_shape = (3, 3)
out_features = 16
out_shape = (4, 4)
data = tl.tensor(rng.random_sample((batch_size, in_features)))
# Creat from a tensor factorization
tensor = TensorizedMatrix.new(out_shape,
in_shape,
rank='same',
factorization=factorization)
tensor.normal_()
fc = nn.Linear(in_features, out_features, bias=True)
fc.weight.data = tensor.to_matrix()
tfc = FactorizedLinear(in_shape,
out_shape,
rank='same',
factorization=tensor,
bias=True)
tfc.bias.data = fc.bias
res_fc = fc(data)
res_tfc = tfc(data)
testing.assert_array_almost_equal(res_fc, res_tfc, decimal=2)
# Decompose an existing layer
fc = nn.Linear(in_features, out_features, bias=True)
tfc = FactorizedLinear.from_linear(fc, (3, 3), (4, 4), rank=34, bias=True)
res_fc = fc(data)
res_tfc = tfc(data)
testing.assert_array_almost_equal(res_fc, res_tfc, decimal=2)
# Multi-layer factorization
fc1 = nn.Linear(in_features, out_features, bias=True)
fc2 = nn.Linear(in_features, out_features, bias=True)
tfc = FactorizedLinear.from_linear_list([fc1, fc2],
in_shape,
out_shape,
rank=38,
bias=True)
## Test first parametrized conv
res_fc = fc1(data)
res_tfc = tfc[0](data)
testing.assert_array_almost_equal(res_fc, res_tfc, decimal=2)
## Test second parametrized conv
res_fc = fc2(data)
res_tfc = tfc[1](data)
testing.assert_array_almost_equal(res_fc, res_tfc, decimal=2)
def test_FactorizedEmbedding(factorization,dims):
NUM_EMBEDDINGS,EMBEDDING_DIM=dims
BATCH_SIZE = 3
#create factorized embedding
factorized_embedding = FactorizedEmbedding(NUM_EMBEDDINGS,EMBEDDING_DIM,factorization=factorization)
#make test embedding of same shape and same weight
test_embedding = torch.nn.Embedding(factorized_embedding.weight.shape[0],factorized_embedding.weight.shape[1])
test_embedding.weight.data.copy_(factorized_embedding.weight.to_matrix().detach())
#create batch and test using all entries (shuffled since entries may not be sorted)
batch = torch.randperm(NUM_EMBEDDINGS)#.view(-1,1)
normal_embed = test_embedding(batch)
factorized_embed = factorized_embedding(batch)
testing.assert_array_almost_equal(normal_embed,factorized_embed,decimal=2)
#split batch into tensor with first dimension 3
batch = torch.randperm(NUM_EMBEDDINGS)
split_size = NUM_EMBEDDINGS//5
split_batch = [batch[:1*split_size],batch[1*split_size:2*split_size],batch[3*split_size:4*split_size]]
split_batch = torch.stack(split_batch,0)
normal_embed = test_embedding(split_batch)
factorized_embed = factorized_embedding(split_batch)
testing.assert_array_almost_equal(normal_embed,factorized_embed,decimal=2)
#BlockTT has no init_from_matrix, so skip that test
if factorization=='BlockTT':
return
del factorized_embedding
#init from test layer which is low rank
factorized_embedding = FactorizedEmbedding.from_embedding(test_embedding,factorization=factorization,rank=8)
#test using same batch as before, only test that shapes match
normal_embed = test_embedding(batch)
factorized_embed = factorized_embedding(batch)
testing.assert_array_almost_equal(normal_embed.shape,factorized_embed.shape,decimal=2)
def test_cp_mode_dot():
"""Test for cp_mode_dot
We will compare cp_mode_dot
(which operates directly on decomposed tensors)
with mode_dot (which operates on full tensors)
and check that the results are the same.
"""
rng = tl.check_random_state(12345)
shape = (5, 4, 6)
rank = 3
cp_ten = random_cp(shape, rank=rank, orthogonal=True, full=False)
full_tensor = tl.cp_to_tensor(cp_ten)
# matrix for mode 1
matrix = tl.tensor(rng.random_sample((7, shape[1])))
# vec for mode 2
vec = tl.tensor(rng.random_sample(shape[2]))
# Test cp_mode_dot with matrix
res = cp_mode_dot(cp_ten, matrix, mode=1, copy=True)
# Note that if copy=True is not respected, factors will be changes
# And the next test will fail
res = tl.cp_to_tensor(res)
true_res = mode_dot(full_tensor, matrix, mode=1)
assert_array_almost_equal(true_res, res)
# Check that the data was indeed copied
rec = tl.cp_to_tensor(cp_ten)
assert_array_almost_equal(full_tensor, rec)
# Test cp_mode_dot with vec
res = cp_mode_dot(cp_ten, vec, mode=2, copy=True)
res = tl.cp_to_tensor(res)
true_res = mode_dot(full_tensor, vec, mode=2)
assert_equal(res.shape, true_res.shape)
assert_array_almost_equal(true_res, res)
def single_conv_test(factorization,
implementation,
order=2,
rank=0.5,
rng=None,
input_channels=4,
output_channels=5,
kernel_size=3,
batch_size=1,
activation_size=(8, 7),
device='cpu'):
rng = tl.check_random_state(rng)
input_shape = (batch_size, input_channels) + activation_size
kernel_shape = (output_channels, input_channels) + (kernel_size, ) * order
if rank is None:
rank = max(kernel_shape)
if order == 1:
FullConv = nn.Conv1d
elif order == 2:
FullConv = nn.Conv2d
elif order == 3:
FullConv = nn.Conv3d
# Factorized input tensor
factorization_shape = kernel_shape_to_factorization_shape(
factorization, kernel_shape)
decomposed_weights = FactorizedTensor.new(
shape=factorization_shape, rank=rank,
factorization=factorization).normal_(0, 1)
full_weights = tensor_to_kernel(factorization,
decomposed_weights.to_tensor().to(device))
data = torch.tensor(rng.random_sample(input_shape),
dtype=torch.float32).to(device)
# PyTorch regular Conv
conv = FullConv(input_channels,
output_channels,
kernel_size,
bias=True,
padding=1)
true_bias = conv.bias.data
conv.weight.data = full_weights
# Factorized conv
fact_conv = FactorizedConv.from_factorization(
decomposed_weights,
implementation=implementation,
bias=true_bias,
padding=1)
# First check it has the correct implementation
msg = f'Created implementation={implementation} but {fact_conv.implementation} was created.'
assert fact_conv.implementation == implementation, msg
# Check that it gives the same result as the full conv
true_res = conv(data)
res = fact_conv(data)
msg = f'{fact_conv.__class__.__name__} does not give same result as {FullConv.__class__.__name__}.'
assert_array_almost_equal(true_res, res, decimal=4, err_msg=msg)
# Check that the parameters of the decomposition are transposed back correctly
decomposed_weights, bias = fact_conv.weight, fact_conv.bias
rec = tensor_to_kernel(factorization, decomposed_weights.to_tensor())
msg = msg = f'{fact_conv.__class__.__name__} does not return the decomposition it was constructed with.'
assert_array_almost_equal(rec, full_weights, err_msg=msg)
msg = msg = f'{fact_conv.__class__.__name__} does not return the bias it was constructed with.'
assert_array_almost_equal(bias, true_bias, err_msg=msg)
conv = FullConv(input_channels,
output_channels,
kernel_size,
bias=True,
padding=1)
conv.weight.data.uniform_(-1, 1)
fact_conv = FactorizedConv.from_conv(
conv, rank=30, factorization=factorization
) #, decomposition_kwargs=dict(init='svd', l2_reg=1e-5))
true_res = conv(data)
res = fact_conv(data)
msg = f'{fact_conv.__class__.__name__} does not give same result as {FullConv.__class__.__name__}.'
assert_array_almost_equal(true_res, res, decimal=2, err_msg=msg)
