def test_storagenumpy_from_storagenumpy(self):
#'''
#Create a StorageNumpy from another StorageNumpy
#'''
n = np.arange(12).reshape(3,4)
s1 = StorageNumpy(n, "test_sn_from_sn")
s2 = StorageNumpy(s1) # Create a StorageNumpy from another StorageNumpy
self.assertTrue(s2.storage_id is None)
self.assertTrue(s2._get_name() is None)
self.assertTrue(np.array_equal(s2, n))
# StorageNumpy s1 and s2 should not share memory
s1[0][0] = 42
self.assertTrue(s2[0,0] != s1[0,0])
s2[2][2] = 666
self.assertTrue(s2[2,2] != s1[2,2])
# Create a third StorageNumpy
s3 = StorageNumpy(s2)
self.assertTrue(s3.storage_id is None)
self.assertTrue(s3._get_name() is None)
self.assertTrue(np.array_equal(s3, s2))
# Clean up
s1.delete_persistent()
def test_types_persistence(self):
base_array = np.arange(256)
tablename = self.ksp + '.' + "test_types_persistence"
for typecode in np.typecodes['Integer']:
if typecode == 'p':
# TODO For now skip arrays made of pointers
pass
typed_array = StorageNumpy(base_array.astype(typecode), tablename)
self.assertTrue(np.array_equal(typed_array, base_array.astype(typecode)))
typed_array.sync() # Flush values to cassandra
typed_array = StorageNumpy(None, tablename)
self.assertTrue(np.allclose(typed_array, base_array.astype(typecode)))
typed_array.delete_persistent()
for typecode in np.typecodes['UnsignedInteger']:
if typecode == 'P':
# TODO For now skip arrays made of pointers
pass
typed_array = StorageNumpy(base_array.astype(typecode), tablename)
self.assertTrue(np.allclose(typed_array, base_array.astype(typecode)))
typed_array.sync() # Flush values to cassandra
typed_array = StorageNumpy(None, tablename)
self.assertTrue(np.allclose(typed_array, base_array.astype(typecode)))
typed_array.delete_persistent()
def test_slice_from_numpy_array(self):
obj = np.arange(8 * 8 * 8).reshape((8, 8, 8))
hecu = StorageNumpy(input_array=obj, name='test_slice_numpy')
l = np.array((0,1))
hecu_sub = hecu[l] #Access using an array of indexes
# FIXME add more testing, currently if it does not segfault, then it works
sum = hecu_sub.sum()
self.assertEqual(sum, obj[l].sum())
hecu.delete_persistent()
def test_slice_ops(self):
obj = np.arange(8 * 8 * 8).reshape((8, 8, 8))
hecu = StorageNumpy(input_array=obj, name='test_slice_ops')
hecu_sub = hecu[:2, 3:, 4:]
sum = hecu_sub.sum()
self.assertGreater(sum, 0)
description = repr(hecu_sub)
self.assertIsInstance(description, str)
hecu.delete_persistent()
def test_slicing_3d(self):
base = np.arange(8 * 8 * 4).reshape((8, 8, 4))
hecu = StorageNumpy(input_array=base, name='test_slicing_3d')
res_hecu = hecu[6:7, 4:]
res = base[6:7, 4:]
self.assertTrue(np.array_equal(res, res_hecu))
hecu.sync() # Flush values to cassandra
hecu = StorageNumpy(name="test_slicing_3d")
res_hecu = hecu[6:7, 4:]
self.assertTrue(np.array_equal(res, res_hecu))
hecu.delete_persistent()
def test_read_all(self):
nelem = 2 ** 21
elem_dim = 2 ** 7
base_array = np.arange(nelem).reshape((elem_dim, elem_dim, elem_dim))
casted = StorageNumpy(input_array=base_array, name="test_read_all")
casted.sync() # Flush values to cassandra
test_numpy = np.arange(nelem).reshape((elem_dim, elem_dim, elem_dim))
casted = StorageNumpy(name="test_read_all")
chunk = casted[slice(None, None, None)]
self.assertTrue(np.allclose(chunk.view(np.ndarray), test_numpy))
casted.delete_persistent()
def test_reconstruct(self):
base_array = np.arange(256)
tablename = self.ksp + '.' + "test_reconstruct"
typecode = 'mytype'
niter = 2
for _ in range(niter):
# Build array and store
typed_array = StorageNumpy(base_array, tablename)
self.assertTrue(np.array_equal(typed_array, base_array))
typed_array.sync() # Flush values to cassandra
# Load array
typed_array = StorageNumpy(None, tablename)
self.assertTrue(np.allclose(typed_array, base_array))
typed_array.delete_persistent()
def test_assign_element(self):
base = np.arange(8 * 8 * 4).reshape((8, 8, 4))
hecu_p = StorageNumpy(input_array=base, name='test_assign_element')
sub_hecu = hecu_p[:2, 3:]
sub_hecu[0][1][0] = 0
hecu_p.sync()
hecu_p_load = StorageNumpy(name="test_assign_element")
rep = repr(hecu_p_load)
self.assertIsInstance(rep, str)
load_sub_arr = hecu_p_load[:]
self.assertFalse(np.array_equal(load_sub_arr, np.arange(8 * 8 * 4).reshape((8, 8, 4))))
sub_hecu_load = hecu_p_load[:2, 3:]
self.assertTrue(sub_hecu_load[0][1][0] == 0)
# Clean up
hecu_p_load.delete_persistent()
def test_assign_slice(self):
base = np.arange(8 * 8 * 4).reshape((8, 8, 4))
hecu_p = StorageNumpy(input_array=base, name='test_assign_slice')
sub_hecu = hecu_p[:2, 3:]
sub_hecu[0][2:] = 0
hecu_p.sync() # Flush values to cassandra
hecu_p_load = StorageNumpy(name="test_assign_slice")
rep = repr(hecu_p_load)
self.assertIsInstance(rep, str)
# StorageNumpy in memory and in database should share data
load_sub_arr = hecu_p_load[:]
self.assertFalse(np.array_equal(load_sub_arr, np.arange(8 * 8 * 4).reshape((8, 8, 4))))
self.assertTrue(np.array_equal(sub_hecu, hecu_p_load[:2, 3:]))
# Clean up
hecu_p_load.delete_persistent()
def test_iter_numpy(self):
obj = np.arange(8 * 8 * 8).reshape((8, 8, 8))
hecu = StorageNumpy(input_array=obj, name='test_iter_numpy')
acc = 0
for i in hecu:
acc = acc + 1
hecu_sub = hecu[:2, 3:, 4:]
acc2 = 0
for i in hecu_sub:
acc2 = acc2 + 1
self.assertGreater(acc, acc2)
hecu.delete_persistent()
def test_transpose(self):
#'''
#Test the transpose
#'''
n=np.arange(12).reshape(3,4)
s=StorageNumpy(n,"testTranspose")
t=s.transpose()
self.assertTrue(t[0,1] == s [1,0])
t[0,1]=42
self.assertTrue(t[0,1] == s[1,0])
# Clean up
s.delete_persistent()
def test_storagenumpy_copy_memory(self):
#'''
#Check that the memory from a StorageNumpy does not share original array
#'''
n = np.arange(12).reshape(3,4)
s1 = StorageNumpy(n, "test_storagenumpy_copy_memory")
# StorageNumpy s1 and n should NOT share memory
s1[0][0] = 42
self.assertTrue(not np.array_equal(s1, n))
s1[0][0] = n[0][0] # Undo
n[2][2] = 666
self.assertTrue(not np.array_equal(s1, n))
# Clean up
s1.delete_persistent()
def test_copy_storageNumpyPersist(self):
#'''
#Test that a copy of a StorageNumpy does not share memory (Persistent version)
#'''
n=np.arange(12).reshape(3,4)
s=StorageNumpy(n,"test_copy_storageNumpyPersist")
c=s.copy()
self.assertTrue(c.storage_id is None)
self.assertTrue(c._get_name() is None)
self.assertTrue(c[0,0]==s[0,0])
c[0,0]=42
self.assertTrue(c[0,0]!=s[0,0])
# Clean up
s.delete_persistent()
def test_storagenumpy_reshape(self):
#'''
#Reshape a StorageNumpy
#'''
n = np.arange(12).reshape(3,4)
s1 = StorageNumpy(n, "test_storagenumpy_reshape")
try:
r = s1.view()
r.shape = (4,3)
self.assertTrue(r.storage_id == s1.storage_id)
except: # If this exception code is executed means that a COPY is needed and therefore the resulting object is VOLATILE!
r = s1.reshape(4,3)
self.assertTrue(r.storage_id is None)
self.assertTrue(r._is_persistent == False)
self.assertTrue(r.shape != s1.shape)
self.assertTrue(r.strides != s1.strides)
# Clean up
s1.delete_persistent()
def test_slicing_ndims(self):
import random
ndims = 10
max_elements = 2048
for dims in range(1, ndims):
elem_per_dim = int(max_elements ** (1 / dims))
select = (slice(random.randint(0, elem_per_dim)),) * dims
base = np.arange(elem_per_dim ** dims).reshape((elem_per_dim,) * dims)
hecu = StorageNumpy(input_array=base, name='test_slicing_ndims')
res_hecu = hecu[select]
res = base[select]
self.assertTrue(np.array_equal(res, res_hecu))
hecu.sync() # Flush values to cassandra
hecu = StorageNumpy(name="test_slicing_ndims")
res_hecu = hecu[select]
res = base[select]
self.assertTrue(np.array_equal(res, res_hecu))
hecu.delete_persistent()
del res_hecu
del hecu
