def test_split_already_persistent(self):
bn, bm = (2, 1)
x = np.arange(100).reshape(10, -1)
blocks = []
for i in range(0, x.shape[0], bn):
row = [x[i: i + bn, j: j + bm] for j in range(0, x.shape[1], bm)]
blocks.append(row)
data = StorageNumpy(input_array=x, name="test_split_already_persistent")
data.sync() # Flush values to cassandra
for i, chunk in enumerate(data.np_split(block_size=(bn, bm))):
storage_id = chunk.storage_id
chunk.sync() #Flush data
del chunk
chunk = getByID(storage_id)
self.assertTrue(np.array_equal(list(chunk), blocks[i]))
del data
gc.collect()
data = StorageNumpy(name="test_split_already_persistent")
self.assertTrue(np.array_equal(list(data), x))
for i, chunk in enumerate(data.np_split(block_size=(bn, bm))):
storage_id = chunk.storage_id
chunk.sync() #Flush data
del chunk
chunk = getByID(storage_id)
self.assertTrue(np.array_equal(list(chunk), blocks[i]))
self.assertEqual(i + 1, len(blocks))
def test_split_by_columns(self):
"""
Tests iterating through the columns of the Hecuba array
"""
bn, bm = (10, 1)
x = np.arange(100).reshape(10, -1)
blocks = []
for i in range(0, x.shape[0], bn):
row = [x[i: i + bn, j: j + bm] for j in range(0, x.shape[1], bm)]
blocks.append(row)
data = StorageNumpy(input_array=x, name="test_split_by_columns")
data.sync() # Flush values to cassandra
for i, chunk in enumerate(data.np_split(block_size=(bn, bm))):
storage_id = chunk.storage_id
chunk.sync() #Flush data
del chunk
chunk = getByID(storage_id)
self.assertTrue(np.array_equal(list(chunk), blocks[i]))
self.assertEqual(i + 1, len(blocks))