def build_tensor(self, tensor=None):
'''
Build the combined tensor. Done explicitly because it's slow.
If `tensor` is not None, it is used as the underlying numeric
storage tensor. It should have the same number of dimensions
as the blend. It defaults to a new DictTensor.
'''
self.logger.info('building combined tensor.')
labels = self._labels
if tensor is None: tensor = DictTensor(ndim=self.ndim)
assert tensor.ndim == self.ndim
if self._keys_never_overlap:
self.logger.info('fast-merging.')
tensor.update((tuple(
label_list.index(label)
for label_list, label in izip(labels, key)), val)
for key, val in self._fast_iteritems())
else:
for factor, cur_tensor, name in zip(self._weights, self._tensors,
self.names):
self.logger.info('slow-merging %s' % name)
for key, val in cur_tensor.iteritems():
tensor.inc(
tuple(
label_list.index(label)
for label_list, label in izip(labels, key)),
factor * val)
self._tensor = tensor
self.logger.info('done building tensor.')
class SVD2DTest(unittest.TestCase):
def setUp(self):
self.tensor = DictTensor(2)
# Note: this command actually puts 20 values in tensor!
self.tensor.update(nested_list_to_dict(svd_2d_test_matrix))
self.svd = self.tensor.svd(k=3, offset_for_row=offset_for_row, offset_for_col=offset_for_col)
self.u, self.svals, self.v = self.svd.u, self.svd.svals, self.svd.v
def test_decomposition(self):
self.assertEqual(self.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.svals), self.u.shape[1])
self.assertEqual(len(self.svals), self.v.shape[1])
self.assertEqual(self.v.shape[0], self.tensor.shape[1])
assertTensorEqual(self.u,
[[0, 0, 1],
[0, -1, 0],
[0, 0, 0],
[-1, 0, 0]], abs=True)
assertTensorEqual(self.v,
[[0, 0, sqrt(.2)],
[-1, 0, 0],
[0, -1, 0],
[0, 0, 0],
[0, 0, sqrt(.8)]], abs=True)
assertTensorEqual(self.svals,
[4, 3, sqrt(5)])
def test_reconstructed(self):
assertTensorEqual(self.svd.reconstructed,
[[1, 0, 0, 0, 2],
[0, 0, 3, 0, 0],
[0, 0, 0, 0, 0],
[0, 4, 0, 0, 0]])
assertTensorEqual(self.svd.reconstructed[1,:],
[0, 0, 3, 0, 0])
assertTensorEqual(self.svd.reconstructed[:,2],
[0, 3, 0, 0])
def test_orthonormality(self):
identity = [[1, 0, 0],
[0, 1, 0],
[0, 0, 1]]
assertTensorEqual(self.u.T * self.u,
identity)
assertTensorEqual(self.v.T * self.v,
identity)
def test_DictMatrixMatrixDot():
# Numbers computed using numpy separately (and checked by hand)...
A = DictTensor(2)
B = DictTensor(2)
A.update({
(0, 0): 0.97878770132160475,
(0, 1): 0.38968165255179188,
(0, 2): 0.62726841877492023,
(1, 0): 0.077757604769237876,
(1, 1): 0.081345677776447523,
(1, 2): 0.64136810022648949
})
B.update({
(0, 0): 0.062059208836173663,
(0, 1): 0.67286767409459525,
(0, 2): 0.55410453533854442,
(0, 3): 0.74671274663041698,
(1, 0): 0.11565332983247767,
(1, 1): 0.48262692547766795,
(1, 2): 0.76280138705455269,
(1, 3): 0.50230554417370143,
(2, 0): 0.67149114912362429,
(2, 1): 0.7656884479264322,
(2, 2): 0.69286881606948747,
(2, 3): 0.82598232206483091
})
test_result = {
(0, 0): 0.52701596238696313,
(0, 1): 1.3269576439118278,
(0, 2): 1.2742151361864653,
(0, 3): 1.4447251324591062,
(1, 0): 0.444906476567622,
(1, 1): 0.58266833824233299,
(1, 2): 0.54952039356712779,
(1, 3): 0.62868169229370208
}
result = A * B
for key, value in result.iteritems():
assert_almost_equal(value, test_result[key])
def testDictDotProduct():
tensor = DictTensor(1)
tensor.update({
1: 0.06198828,
3: 0.24177249,
6: 0.5256805,
7: 0.46505895,
8: 0.27791615,
9: 0.02906779})
tensor2 = DictTensor(1)
tensor2.update({
0: 0.2502674,
2: 0.34907184,
3: 0.2209139,
5: 0.45788618,
6: 0.37133328,
7: 0.48278861})
result = tensor * tensor2
assert_almost_equal(result, 0.473138731464)
def test_DictMatrixVectorDot():
# Numbers computed using numpy separately (and checked by hand)...
A = DictTensor(2)
b = DictTensor(1)
A.update({(0, 0): 0.18850744743616121,
(0, 1): 0.64380371397047509,
(1, 0): 0.40673500155569442,
(1, 1): 0.77961381386745443,
(2, 0): 0.38745898104117782,
(2, 1): 0.39479530812173591})
b.update({0: 0.95308634444417639, 1: 0.41483520394218798})
test_result = {(0,): 0.44673631896111365,
(1,): 0.71106483126206554,
(2,): 0.53305685602270081}
result = A * b
for k, value in result.iteritems():
assert_almost_equal(value, test_result[k])
class NormalizedSVD2DTest(unittest.TestCase):
def setUp(self):
self.tensor = DictTensor(2)
self.tensor.update(nested_list_to_dict(
numpy.random.random_sample((10, 12))))
self.normalized_tensor = self.tensor.normalized()
self.svd = self.normalized_tensor.svd(k=3)
self.u, self.svals, self.v = self.svd.u, self.svd.svals, self.svd.v
def test_decomposition(self):
self.assertEqual(self.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.svals), self.u.shape[1])
self.assertEqual(len(self.svals), self.v.shape[1])
self.assertEqual(self.v.shape[0], self.tensor.shape[1])
# Assert that the singular values are decreasing
for i in range(1,len(self.svals)):
self.assert_(self.svals[i] < self.svals[i-1])
def test_reconstructed(self):
pass # TODO
def test_orthonormality(self):
assertTensorEqual(self.u.T * self.u, numpy.eye(self.u.shape[1]))
assertTensorEqual(self.v.T * self.v, numpy.eye(self.u.shape[1]))
def test_variance(self):
return # TODO
# Assert that the SVD explained some of the variance.
diff_k3 = self.tensor - self.svd.reconstructed
tensor_mag = self.tensor.magnitude()
diff_k3_mag = diff_k3.magnitude()
self.assert_(tensor_mag > diff_k3_mag)
# Check that a smaller SVD explains less of the variance, but still some.
svd_k1 = self.tensor.svd(k=1)
diff_k1 = self.tensor - svd_k1.reconstructed
diff_k1_mag = diff_k1.magnitude()
self.assert_(tensor_mag > diff_k1_mag > diff_k3_mag)
def testDictDotProduct():
tensor = DictTensor(1)
tensor.update({
1: 0.06198828,
3: 0.24177249,
6: 0.5256805,
7: 0.46505895,
8: 0.27791615,
9: 0.02906779
})
tensor2 = DictTensor(1)
tensor2.update({
0: 0.2502674,
2: 0.34907184,
3: 0.2209139,
5: 0.45788618,
6: 0.37133328,
7: 0.48278861
})
result = tensor * tensor2
assert_almost_equal(result, 0.473138731464)
class NormalizedSVD2DTest(unittest.TestCase):
def setUp(self):
self.tensor = DictTensor(2)
self.tensor.update(
nested_list_to_dict(numpy.random.random_sample((10, 12))))
self.normalized_tensor = self.tensor.normalized()
self.svd = self.normalized_tensor.svd(k=3)
self.u, self.svals, self.v = self.svd.u, self.svd.svals, self.svd.v
def test_decomposition(self):
self.assertEqual(self.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.svals), self.u.shape[1])
self.assertEqual(len(self.svals), self.v.shape[1])
self.assertEqual(self.v.shape[0], self.tensor.shape[1])
# Assert that the singular values are decreasing
for i in range(1, len(self.svals)):
self.assert_(self.svals[i] < self.svals[i - 1])
def test_reconstructed(self):
pass # TODO
def test_orthonormality(self):
assertTensorEqual(self.u.T * self.u, numpy.eye(self.u.shape[1]))
assertTensorEqual(self.v.T * self.v, numpy.eye(self.u.shape[1]))
def test_variance(self):
return # TODO
# Assert that the SVD explained some of the variance.
diff_k3 = self.tensor - self.svd.reconstructed
tensor_mag = self.tensor.magnitude()
diff_k3_mag = diff_k3.magnitude()
self.assert_(tensor_mag > diff_k3_mag)
# Check that a smaller SVD explains less of the variance, but still some.
svd_k1 = self.tensor.svd(k=1)
diff_k1 = self.tensor - svd_k1.reconstructed
diff_k1_mag = diff_k1.magnitude()
self.assert_(tensor_mag > diff_k1_mag > diff_k3_mag)
def test_DictMatrixMatrixDot():
# Numbers computed using numpy separately (and checked by hand)...
A = DictTensor(2)
B = DictTensor(2)
A.update({(0, 0): 0.97878770132160475,
(0, 1): 0.38968165255179188,
(0, 2): 0.62726841877492023,
(1, 0): 0.077757604769237876,
(1, 1): 0.081345677776447523,
(1, 2): 0.64136810022648949})
B.update({(0, 0): 0.062059208836173663,
(0, 1): 0.67286767409459525,
(0, 2): 0.55410453533854442,
(0, 3): 0.74671274663041698,
(1, 0): 0.11565332983247767,
(1, 1): 0.48262692547766795,
(1, 2): 0.76280138705455269,
(1, 3): 0.50230554417370143,
(2, 0): 0.67149114912362429,
(2, 1): 0.7656884479264322,
(2, 2): 0.69286881606948747,
(2, 3): 0.82598232206483091})
test_result = {(0, 0): 0.52701596238696313,
(0, 1): 1.3269576439118278,
(0, 2): 1.2742151361864653,
(0, 3): 1.4447251324591062,
(1, 0): 0.444906476567622,
(1, 1): 0.58266833824233299,
(1, 2): 0.54952039356712779,
(1, 3): 0.62868169229370208}
result = A * B
for key, value in result.iteritems():
assert_almost_equal(value, test_result[key])
class SVD2DTest(unittest.TestCase):
def setUp(self):
self.tensor = DictTensor(2)
# Note: this command actually puts 20 values in tensor!
self.tensor.update(nested_list_to_dict(svd_2d_test_matrix))
self.svd = self.tensor.svd(k=3,
offset_for_row=offset_for_row,
offset_for_col=offset_for_col)
self.u, self.svals, self.v = self.svd.u, self.svd.svals, self.svd.v
def test_decomposition(self):
self.assertEqual(self.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.svals), self.u.shape[1])
self.assertEqual(len(self.svals), self.v.shape[1])
self.assertEqual(self.v.shape[0], self.tensor.shape[1])
assertTensorEqual(self.u,
[[0, 0, 1], [0, -1, 0], [0, 0, 0], [-1, 0, 0]],
abs=True)
assertTensorEqual(self.v, [[0, 0, sqrt(.2)], [-1, 0, 0], [0, -1, 0],
[0, 0, 0], [0, 0, sqrt(.8)]],
abs=True)
assertTensorEqual(self.svals, [4, 3, sqrt(5)])
def test_reconstructed(self):
assertTensorEqual(self.svd.reconstructed,
[[1, 0, 0, 0, 2], [0, 0, 3, 0, 0], [0, 0, 0, 0, 0],
[0, 4, 0, 0, 0]])
assertTensorEqual(self.svd.reconstructed[1, :], [0, 0, 3, 0, 0])
assertTensorEqual(self.svd.reconstructed[:, 2], [0, 3, 0, 0])
def test_orthonormality(self):
identity = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]
assertTensorEqual(self.u.T * self.u, identity)
assertTensorEqual(self.v.T * self.v, identity)
def test_DictMatrixVectorDot():
# Numbers computed using numpy separately (and checked by hand)...
A = DictTensor(2)
b = DictTensor(1)
A.update({
(0, 0): 0.18850744743616121,
(0, 1): 0.64380371397047509,
(1, 0): 0.40673500155569442,
(1, 1): 0.77961381386745443,
(2, 0): 0.38745898104117782,
(2, 1): 0.39479530812173591
})
b.update({0: 0.95308634444417639, 1: 0.41483520394218798})
test_result = {
(0, ): 0.44673631896111365,
(1, ): 0.71106483126206554,
(2, ): 0.53305685602270081
}
result = A * b
for k, value in result.iteritems():
assert_almost_equal(value, test_result[k])
def build_tensor(self, tensor=None):
'''
Build the combined tensor. Done explicitly because it's slow.
If `tensor` is not None, it is used as the underlying numeric
storage tensor. It should have the same number of dimensions
as the blend. It defaults to a new DictTensor.
'''
self.logger.info('building combined tensor.')
labels = self._labels
if tensor is None: tensor = DictTensor(ndim=self.ndim)
assert tensor.ndim == self.ndim
if self._keys_never_overlap:
self.logger.info('fast-merging.')
tensor.update((tuple(label_list.index(label) for label_list, label in izip(labels, key)), val)
for key, val in self._fast_iteritems())
else:
for factor, cur_tensor, name in zip(self._weights, self._tensors, self.names):
self.logger.info('slow-merging %s' % name)
for key, val in cur_tensor.iteritems():
tensor.inc(tuple(label_list.index(label) for label_list, label in izip(labels, key)), factor*val)
self._tensor = tensor
self.logger.info('done building tensor.')
class DictTensorTest(unittest.TestCase):
slice_testcase = [[1, None, None],
[None, 2, 3 ],
[4, None, None],
[None, 5, None]]
def test_initial(self):
self.assertEqual(len(self.tensor), 0)
self.assertEqual(len(self.tensor.keys()), 0)
assert_dims_consistent(self.tensor)
self.assertEqual(self.tensor.shape, (0, 0))
assert isinstance(self.tensor[4, 5], (float, int, long))
self.assertEqual(self.tensor[5, 5], 0)
self.assertEqual(self.tensor[2, 7], 0)
def test_storage(self):
self.tensor[5, 5] = 1
self.tensor[2, 7] = 2
assertTensorEqual(self.tensor,
[[None]*8,
[None]*8,
[None]*7 + [2],
[None]*8,
[None]*8,
[None]*5 + [1, None, None]])
def test_slice(self):
self.tensor.update(nones_removed(nested_list_to_dict(self.slice_testcase)))
# Test end conditions: start index
# is included in slice, end index is not
slice = self.tensor[1:3, 0:2]
assertTensorEqual(slice,
[[None, 2],
[4, None]])
# Test that slicing on some dims correctly
# reduces the dimensionality of the tensor
slice = self.tensor[3, :]
assertTensorEqual(slice, [None, 5, None])
# Test the step parameter
slice = self.tensor[1:4:2, :]
assertTensorEqual(slice,
[[None, 2, 3],
[None, 5, None]])
def test_transpose(self):
self.tensor[0, 0] = 1
self.tensor[1, 2] = 3
self.tensor[2, 0] = 4
self.tensor[3, 1] = 5
t = self.tensor.transpose()
assertTensorEqual(t,
[[1, None, 4, None],
[None, None, None, 5],
[None, 3, None, None]])
def test_delete(self):
self.tensor.update(nones_removed(nested_list_to_dict(self.slice_testcase)))
assertTensorEqual(self.tensor, self.slice_testcase)
del self.tensor[0,0]
assertTensorEqual(self.tensor,
[[None, None, None],
[None, 2, 3 ],
[4, None, None],
[None, 5, None]])
def test_contains(self):
self.tensor[1,2] = 1
self.tensor[4,5] = 2
self.assertTrue((1,2) in self.tensor)
self.assertTrue(self.tensor.has_key((1,2)))
self.assertFalse((4,2) in self.tensor)
self.assertFalse((1,5) in self.tensor)
def setUp(self):
self.tensor = DictTensor(2)
def test_1D(self):
tensor_1D = DictTensor(1)
tensor_1D[2] = 1
assertTensorEqual(tensor_1D,
[None, None, 1])
def test_combine_by_element(self):
t1 = DictTensor(2)
t2 = DictTensor(2)
t1[1, 1] = 1
t1[1, 0] = 2
t2[1, 1] = 4
t2[0, 1] = 5
t3 = t1.combine_by_element(t2, lambda x, y: x + (2*y))
assertTensorEqual(t3,
[[None, 10],
[2, 9]])
# Make sure errors are raised when the tensors don't have the
# same shape or number of dimensions
t4 = DictTensor(2)
t4[0, 2] = 3
t4[1, 0] = 5
self.assertRaises(IndexError, lambda: t1.combine_by_element(t4, lambda x, y: x + y))
t4 = DictTensor(3)
self.assertRaises(IndexError, lambda: t1.combine_by_element(t4, lambda x, y: x + y))
def testAdd(self):
t1 = DictTensor(2)
t2 = DictTensor(2)
t1[0, 0] = 1
t1[1, 1] = 1
t1[1, 0] = 2
t2[2, 1] = 4
t2[1, 0] = 5
t3 = t1 + t2
assertTensorEqual(t3,
[[1, None],
[7, 1],
[None, 4]])
def testICmul(self):
t1 = tensor_from_nested_list([[1, 2], [3, 4]])
assertTensorEqual(t1, [[1, 2], [3, 4]])
t1 *= 2
assertTensorEqual(t1, [[2, 4], [6, 8]])
def testICdiv(self):
t1 = tensor_from_nested_list([[2, 4], [6, 8]])
t1 /= 2
assertTensorEqual(t1, [[1, 2], [3, 4]])
def testReprOfEmpty(self):
repr(self.tensor)
self.tensor.example_key()
def testNorm(self):
norm_test = [[0,0,0],
[0,1,0],
[0,5.0,0]]
self.tensor.update(nested_list_to_dict(norm_test))
self.assertEqual(self.tensor.norm(), sqrt(26.0))
self.assertEqual(self.tensor.magnitude(), sqrt(26.0))
from csc.divisi.tensor import DictTensor
from csc.divisi.normalized_view import NormalizedView
from nose.tools import raises, assert_almost_equal
from tensor_util import assertTensorEqual, nones_removed, nested_list_to_dict
normalize_testcase = [[1, None],
[3, 4]]
normalize_expected_result = [[1, None],
[3/5., 4/5.]]
raw = DictTensor(2)
raw.update(nones_removed(nested_list_to_dict(normalize_testcase)))
tensor = NormalizedView(raw, 0)
def test_result():
assertTensorEqual(tensor, normalize_expected_result)
def test_contains():
assert (0,0) in tensor
assert tensor.has_key((0,0))
assert (0,1) not in tensor
assert not tensor.has_key((0,1))
def test_unnormalize():
assert_almost_equal(tensor[1,0], 3/5.)
assert_almost_equal(tensor.unnormalized()[1,0], 3)
def test_labeled_unnormalize():
labeled = tensor.labeled([['a','b'],['A','B']])
assert_almost_equal(labeled['b','A'], 3/5.)
import numpy as np
import unittest
from nose.tools import eq_, raises
from math import sqrt
from csc.divisi.tensor import DictTensor
from csc.divisi.util import nested_list_to_dict
from tensor_util import assertTensorEqual, zeros_removed
data = np.array([[1, 2, 3, 4], [-1, 2, 3, 4], [0, 1, -1, 0]])
tensor = DictTensor(2)
tensor.update(zeros_removed(nested_list_to_dict(data)))
eq_(len(tensor), 10)
# For NumPy, "along an axis" means something different.
ms_data = data - data.mean(1)[:, np.newaxis]
ms_tensor = DictTensor(2)
ms_tensor.update(nested_list_to_dict(ms_data))
def test_means():
means = tensor.means()
eq_(len(means), 2)
assert np.allclose(means[0], [(1 + 2 + 3 + 4) / 4., (-1 + 2 + 3 + 4) / 4.,
(0 + 1 + -1 + 0) / 4.])
assert np.allclose(
means[1], [0, (2 + 2 + 1) / 3., (3 + 3 - 1) / 3., (4 + 4 + 0) / 3.])
def test_mean_subtracted():
mean_subtracted = tensor.mean_subtracted()
class SVD2DTest(unittest.TestCase):
def setUp(self):
self.tensor = DictTensor(2)
# Note: this command actually puts 20 values in tensor!
self.tensor.update(nested_list_to_dict(svd_2d_test_matrix))
self.svd = self.tensor.svd(k=3)
self.incremental = self.tensor.incremental_svd(k=3, niter=200)
self.u, self.svals, self.v = self.svd.u, self.svd.svals, self.svd.v
def test_incremental(self):
self.assertEqual(self.incremental.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.incremental.svals), self.incremental.u.shape[1])
self.assertEqual(len(self.incremental.svals), self.incremental.v.shape[1])
self.assertEqual(self.incremental.v.shape[0], self.tensor.shape[1])
assertTensorEqual(self.incremental.u,
[[0, 0, 1],
[0, 1, 0],
[0, 0, 0],
[1, 0, 0]])
assertTensorEqual(self.incremental.v,
[[0, 0, sqrt(.2)],
[1, 0, 0],
[0, 1, 0],
[0, 0, 0],
[0, 0, sqrt(.8)]])
assertTensorEqual(self.incremental.svals,
[4, 3, sqrt(5)])
def test_decomposition(self):
self.assertEqual(self.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.svals), self.u.shape[1])
self.assertEqual(len(self.svals), self.v.shape[1])
self.assertEqual(self.v.shape[0], self.tensor.shape[1])
assertTensorEqual(self.u,
[[0, 0, 1],
[0, -1, 0],
[0, 0, 0],
[-1, 0, 0]], abs=True)
assertTensorEqual(self.v,
[[0, 0, sqrt(.2)],
[-1, 0, 0],
[0, -1, 0],
[0, 0, 0],
[0, 0, sqrt(.8)]], abs=True)
assertTensorEqual(self.svals,
[4, 3, sqrt(5)])
def test_reconstructed(self):
assertTensorEqual(self.svd.reconstructed,
[[1, 0, 0, 0, 2],
[0, 0, 3, 0, 0],
[0, 0, 0, 0, 0],
[0, 4, 0, 0, 0]])
assertTensorEqual(self.svd.reconstructed[1,:],
[0, 0, 3, 0, 0])
assertTensorEqual(self.svd.reconstructed[:,2],
[0, 3, 0, 0])
def test_orthonormality(self):
identity = [[1, 0, 0],
[0, 1, 0],
[0, 0, 1]]
assertTensorEqual(self.u.T * self.u,
identity)
assertTensorEqual(self.v.T * self.v,
identity)
def test_variance(self):
# Assert that the SVD explained some of the variance.
diff_k3 = self.tensor - self.svd.reconstructed
tensor_mag = self.tensor.magnitude()
diff_k3_mag = diff_k3.magnitude()
self.assert_(tensor_mag > diff_k3_mag)
# Check that a smaller SVD explains less of the variance, but still some.
svd_k1 = self.tensor.svd(k=1)
diff_k1 = self.tensor - svd_k1.reconstructed
diff_k1_mag = diff_k1.magnitude()
self.assert_(tensor_mag > diff_k1_mag > diff_k3_mag)
import numpy as np
import unittest
from nose.tools import eq_, raises
from math import sqrt
from csc.divisi.tensor import DictTensor
from csc.divisi.util import nested_list_to_dict
from tensor_util import assertTensorEqual, zeros_removed
data = np.array([[1, 2, 3, 4],
[-1,2, 3, 4],
[0, 1, -1,0]])
tensor = DictTensor(2)
tensor.update(zeros_removed(nested_list_to_dict(data)))
eq_(len(tensor), 10)
# For NumPy, "along an axis" means something different.
ms_data = data - data.mean(1)[:,np.newaxis]
ms_tensor = DictTensor(2)
ms_tensor.update(nested_list_to_dict(ms_data))
def test_means():
means = tensor.means()
eq_(len(means), 2)
assert np.allclose(means[0], [(1+2+3+4)/4., (-1+2+3+4)/4., (0+1+-1+0)/4.])
assert np.allclose(means[1], [0, (2+2+1)/3., (3+3-1)/3., (4+4+0)/3.])
def test_mean_subtracted():
mean_subtracted = tensor.mean_subtracted()
m = np.zeros(data.shape)
for (r, c), v in mean_subtracted.iteritems():
class DictTensorTest(unittest.TestCase):
slice_testcase = [[1, None, None], [None, 2, 3], [4, None, None],
[None, 5, None]]
def test_initial(self):
self.assertEqual(len(self.tensor), 0)
self.assertEqual(len(self.tensor.keys()), 0)
assert_dims_consistent(self.tensor)
self.assertEqual(self.tensor.shape, (0, 0))
assert isinstance(self.tensor[4, 5], (float, int, long))
self.assertEqual(self.tensor[5, 5], 0)
self.assertEqual(self.tensor[2, 7], 0)
def test_storage(self):
self.tensor[5, 5] = 1
self.tensor[2, 7] = 2
assertTensorEqual(
self.tensor, [[None] * 8, [None] * 8, [None] * 7 + [2], [None] * 8,
[None] * 8, [None] * 5 + [1, None, None]])
def test_slice(self):
self.tensor.update(
nones_removed(nested_list_to_dict(self.slice_testcase)))
# Test end conditions: start index
# is included in slice, end index is not
slice = self.tensor[1:3, 0:2]
assertTensorEqual(slice, [[None, 2], [4, None]])
# Test that slicing on some dims correctly
# reduces the dimensionality of the tensor
slice = self.tensor[3, :]
assertTensorEqual(slice, [None, 5, None])
# Test the step parameter
slice = self.tensor[1:4:2, :]
assertTensorEqual(slice, [[None, 2, 3], [None, 5, None]])
def test_transpose(self):
self.tensor[0, 0] = 1
self.tensor[1, 2] = 3
self.tensor[2, 0] = 4
self.tensor[3, 1] = 5
t = self.tensor.transpose()
assertTensorEqual(
t,
[[1, None, 4, None], [None, None, None, 5], [None, 3, None, None]])
def test_delete(self):
self.tensor.update(
nones_removed(nested_list_to_dict(self.slice_testcase)))
assertTensorEqual(self.tensor, self.slice_testcase)
del self.tensor[0, 0]
assertTensorEqual(self.tensor, [[None, None, None], [None, 2, 3],
[4, None, None], [None, 5, None]])
def test_contains(self):
self.tensor[1, 2] = 1
self.tensor[4, 5] = 2
self.assertTrue((1, 2) in self.tensor)
self.assertTrue(self.tensor.has_key((1, 2)))
self.assertFalse((4, 2) in self.tensor)
self.assertFalse((1, 5) in self.tensor)
def setUp(self):
self.tensor = DictTensor(2)
def test_1D(self):
tensor_1D = DictTensor(1)
tensor_1D[2] = 1
assertTensorEqual(tensor_1D, [None, None, 1])
def test_combine_by_element(self):
t1 = DictTensor(2)
t2 = DictTensor(2)
t1[1, 1] = 1
t1[1, 0] = 2
t2[1, 1] = 4
t2[0, 1] = 5
t3 = t1.combine_by_element(t2, lambda x, y: x + (2 * y))
assertTensorEqual(t3, [[None, 10], [2, 9]])
# Make sure errors are raised when the tensors don't have the
# same shape or number of dimensions
t4 = DictTensor(2)
t4[0, 2] = 3
t4[1, 0] = 5
self.assertRaises(
IndexError, lambda: t1.combine_by_element(t4, lambda x, y: x + y))
t4 = DictTensor(3)
self.assertRaises(
IndexError, lambda: t1.combine_by_element(t4, lambda x, y: x + y))
def testAdd(self):
t1 = DictTensor(2)
t2 = DictTensor(2)
t1[0, 0] = 1
t1[1, 1] = 1
t1[1, 0] = 2
t2[2, 1] = 4
t2[1, 0] = 5
t3 = t1 + t2
assertTensorEqual(t3, [[1, None], [7, 1], [None, 4]])
def testICmul(self):
t1 = tensor_from_nested_list([[1, 2], [3, 4]])
assertTensorEqual(t1, [[1, 2], [3, 4]])
t1 *= 2
assertTensorEqual(t1, [[2, 4], [6, 8]])
def testICdiv(self):
t1 = tensor_from_nested_list([[2, 4], [6, 8]])
t1 /= 2
assertTensorEqual(t1, [[1, 2], [3, 4]])
def testReprOfEmpty(self):
repr(self.tensor)
self.tensor.example_key()
def testNorm(self):
norm_test = [[0, 0, 0], [0, 1, 0], [0, 5.0, 0]]
self.tensor.update(nested_list_to_dict(norm_test))
self.assertEqual(self.tensor.norm(), sqrt(26.0))
self.assertEqual(self.tensor.magnitude(), sqrt(26.0))
from csc.divisi.tensor import DictTensor
from csc.divisi.normalized_view import NormalizedView
from nose.tools import raises, assert_almost_equal
from tensor_util import assertTensorEqual, nones_removed, nested_list_to_dict
normalize_testcase = [[1, None], [3, 4]]
normalize_expected_result = [[1, None], [3 / 5., 4 / 5.]]
raw = DictTensor(2)
raw.update(nones_removed(nested_list_to_dict(normalize_testcase)))
tensor = NormalizedView(raw, 0)
def test_result():
assertTensorEqual(tensor, normalize_expected_result)
def test_contains():
assert (0, 0) in tensor
assert tensor.has_key((0, 0))
assert (0, 1) not in tensor
assert not tensor.has_key((0, 1))
def test_unnormalize():
assert_almost_equal(tensor[1, 0], 3 / 5.)
assert_almost_equal(tensor.unnormalized()[1, 0], 3)
def test_labeled_unnormalize():
class SVD2DTest(unittest.TestCase):
def setUp(self):
self.tensor = DictTensor(2)
# Note: this command actually puts 20 values in tensor!
self.tensor.update(nested_list_to_dict(svd_2d_test_matrix))
self.svd = self.tensor.svd(k=3)
self.incremental = self.tensor.incremental_svd(k=3, niter=200)
self.u, self.svals, self.v = self.svd.u, self.svd.svals, self.svd.v
def test_incremental(self):
self.assertEqual(self.incremental.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.incremental.svals),
self.incremental.u.shape[1])
self.assertEqual(len(self.incremental.svals),
self.incremental.v.shape[1])
self.assertEqual(self.incremental.v.shape[0], self.tensor.shape[1])
assertTensorEqual(self.incremental.u,
[[0, 0, 1], [0, 1, 0], [0, 0, 0], [1, 0, 0]])
assertTensorEqual(self.incremental.v,
[[0, 0, sqrt(.2)], [1, 0, 0], [0, 1, 0], [0, 0, 0],
[0, 0, sqrt(.8)]])
assertTensorEqual(self.incremental.svals, [4, 3, sqrt(5)])
def test_decomposition(self):
self.assertEqual(self.u.shape[0], self.tensor.shape[0])
self.assertEqual(len(self.svals), self.u.shape[1])
self.assertEqual(len(self.svals), self.v.shape[1])
self.assertEqual(self.v.shape[0], self.tensor.shape[1])
assertTensorEqual(self.u,
[[0, 0, 1], [0, -1, 0], [0, 0, 0], [-1, 0, 0]],
abs=True)
assertTensorEqual(self.v, [[0, 0, sqrt(.2)], [-1, 0, 0], [0, -1, 0],
[0, 0, 0], [0, 0, sqrt(.8)]],
abs=True)
assertTensorEqual(self.svals, [4, 3, sqrt(5)])
def test_reconstructed(self):
assertTensorEqual(self.svd.reconstructed,
[[1, 0, 0, 0, 2], [0, 0, 3, 0, 0], [0, 0, 0, 0, 0],
[0, 4, 0, 0, 0]])
assertTensorEqual(self.svd.reconstructed[1, :], [0, 0, 3, 0, 0])
assertTensorEqual(self.svd.reconstructed[:, 2], [0, 3, 0, 0])
def test_orthonormality(self):
identity = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]
assertTensorEqual(self.u.T * self.u, identity)
assertTensorEqual(self.v.T * self.v, identity)
def test_variance(self):
# Assert that the SVD explained some of the variance.
diff_k3 = self.tensor - self.svd.reconstructed
tensor_mag = self.tensor.magnitude()
diff_k3_mag = diff_k3.magnitude()
self.assert_(tensor_mag > diff_k3_mag)
# Check that a smaller SVD explains less of the variance, but still some.
svd_k1 = self.tensor.svd(k=1)
diff_k1 = self.tensor - svd_k1.reconstructed
diff_k1_mag = diff_k1.magnitude()
self.assert_(tensor_mag > diff_k1_mag > diff_k3_mag)
