def test_split_already_persistent(self): bn, bm = (2, 1) x = np.arange(100).reshape(10, -1) blocks = [] for i in range(0, x.shape[0], bn): row = [x[i: i + bn, j: j + bm] for j in range(0, x.shape[1], bm)] blocks.append(row) data = StorageNumpy(input_array=x, name="test_split_already_persistent") data.sync() # Flush values to cassandra for i, chunk in enumerate(data.np_split(block_size=(bn, bm))): storage_id = chunk.storage_id chunk.sync() #Flush data del chunk chunk = getByID(storage_id) self.assertTrue(np.array_equal(list(chunk), blocks[i])) del data gc.collect() data = StorageNumpy(name="test_split_already_persistent") self.assertTrue(np.array_equal(list(data), x)) for i, chunk in enumerate(data.np_split(block_size=(bn, bm))): storage_id = chunk.storage_id chunk.sync() #Flush data del chunk chunk = getByID(storage_id) self.assertTrue(np.array_equal(list(chunk), blocks[i])) self.assertEqual(i + 1, len(blocks))
def test_split_by_columns(self): """ Tests iterating through the columns of the Hecuba array """ bn, bm = (10, 1) x = np.arange(100).reshape(10, -1) blocks = [] for i in range(0, x.shape[0], bn): row = [x[i: i + bn, j: j + bm] for j in range(0, x.shape[1], bm)] blocks.append(row) data = StorageNumpy(input_array=x, name="test_split_by_columns") data.sync() # Flush values to cassandra for i, chunk in enumerate(data.np_split(block_size=(bn, bm))): storage_id = chunk.storage_id chunk.sync() #Flush data del chunk chunk = getByID(storage_id) self.assertTrue(np.array_equal(list(chunk), blocks[i])) self.assertEqual(i + 1, len(blocks))