def test_split_content(self):
n = np.arange(88*66).reshape(88,66)
s = StorageNumpy(n,"test_split_content")
s.sync() # Flush values to cassandra
del s
s = StorageNumpy(None,"test_split_content")
rows = [i for i in s.split(cols=False)]
self.assertTrue(len(rows)==4)
columns = [ i for i in s.split(cols=True)]
self.assertTrue(len(columns)==3)
blocks = [i for i in s.split()]
self.assertTrue(len(blocks)==12)
for i in rows:
self.assertTrue(i.shape == (22,66))
for i in columns:
self.assertTrue(i.shape == (88,22))
for i in blocks:
self.assertTrue(i.shape == (22,22))
self.assertTrue(np.array_equal(rows[0],n[0:22,:]))
self.assertTrue(np.array_equal(rows[1],n[22:44,:]))
self.assertTrue(np.array_equal(rows[2],n[44:66,:]))
self.assertTrue(np.array_equal(rows[3],n[66:,:]))
self.assertTrue(np.array_equal(columns[0],n[:,0:22]))
self.assertTrue(np.array_equal(columns[1],n[:,22:44]))
self.assertTrue(np.array_equal(columns[2],n[:,44:]))
def test_loaded(self):
n = np.arange(88*66).reshape(88,66)
s = StorageNumpy(n, "test_loaded")
self.assertTrue(s._numpy_full_loaded is True)
s.sync() # Flush values to cassandra
del s
s = StorageNumpy(None, "test_loaded")
self.assertTrue(s._numpy_full_loaded is False)
# The accessed element must be FULL loaded
row = s[0,:]
self.assertTrue(s._numpy_full_loaded is False)
self.assertTrue(row._numpy_full_loaded is True)
del s
s = StorageNumpy(None, "test_loaded")
col = s[:, 0]
self.assertTrue(s._numpy_full_loaded is False)
self.assertTrue(col._numpy_full_loaded is True)
del s
s = StorageNumpy(None, "test_loaded")
block = s[22:44, 22:44]
self.assertTrue(s._numpy_full_loaded is False)
self.assertTrue(block._numpy_full_loaded is True)
# Loading ALL elements must make the object full loaded
del s
s = StorageNumpy(None, "test_loaded")
for i in range(s.shape[0]):
x = s[i,:]
self.assertTrue(s._numpy_full_loaded is True)
del s
s = StorageNumpy(None, "test_loaded")
for i in range(s.shape[1]):
x = s[:,i]
self.assertTrue(s._numpy_full_loaded is True)
# Split MUST NOT load the object
del s
s = StorageNumpy(None, "test_loaded")
rows = [ i for i in s.split(cols=False) ]
for i in rows:
self.assertTrue(i._numpy_full_loaded is False)
del s
s = StorageNumpy(None, "test_loaded")
columns = [ i for i in s.split(cols=True) ]
for i in columns:
self.assertTrue(i._numpy_full_loaded is False)
del s
s = StorageNumpy(None, "test_loaded")
blocks = [ i for i in s.split() ]
for i in blocks:
self.assertTrue(i._numpy_full_loaded is False)
def test_split_access(self):
n = np.arange(2*128).reshape(2,128) # A matrix with "some" columns
s = StorageNumpy(n, "test_split_access")
if s._build_args.metas.partition_type != 0: #This test is only valid for ZORDER
return
splits = 0
for i in s.split():
# Assuming a BLOCK_SIZE of 4096!! FIXME use an environment variable!
if splits <= 4:
self.assertTrue(np.array_equal(i[:], n[0:22, 22*splits:22*(splits+1)]))
else:
self.assertTrue(np.array_equal(i[:], n[0:22, 22*splits:22*(splits)+18]))
splits = splits + 1
def test_split(self):
n = np.arange(2*128).reshape(2,128) # A matrix with "some" columns
s = StorageNumpy(n, "test_split")
splits = 0
for i in s.split():
# Assuming a BLOCK_SIZE of 4096!! FIXME use an environment variable!
if splits <= 4:
self.assertEqual(i.shape, (2,22))
else:
self.assertEqual(i.shape, (2,18))
self.assertTrue(i[0,0] == splits*22)
splits = splits + 1
self.assertTrue(splits == 6)
def test_split_nomem(self):
n = np.arange(2*128).reshape(2,128) # A matrix with "some" columns
s = StorageNumpy(n, "test_split_nomem")
splits = 0
s.sync() # Flush values to cassandra
for i in s.split():
sid = i.storage_id
i.getID() # Store split in hecuba.istorage
del i
i = StorageNumpy(None,None,sid)
# Assuming a BLOCK_SIZE of 4096!! FIXME use an environment variable!
if splits <= 4:
self.assertEqual(i.shape, (2,22))
else:
self.assertEqual(i.shape, (2,18))
self.assertTrue(i[0,0] == splits*22)
splits = splits + 1
self.assertTrue(splits == 6)
def test_split_access_nomem(self):
n = np.arange(2*128).reshape(2,128) # A matrix with "some" columns
s = StorageNumpy(n, "test_split_access_nomem")
if s._build_args.metas.partition_type != 0: #This test is only valid for ZORDER
return
s.sync() # Flush values to cassandra
u = s.storage_id
splits = 0
for i in s.split():
sid = i.storage_id
i.getID() # Store split in hecuba.istorage
del i
i = StorageNumpy(None,None,sid)
# Assuming a BLOCK_SIZE of 4096!! FIXME use an environment variable!
if splits <= 4:
self.assertTrue(np.array_equal(i[:], n[0:22, 22*splits:22*(splits+1)]))
else:
self.assertTrue(np.array_equal(i[:], n[0:22, 22*splits:22*(splits)+18]))
splits = splits + 1
