def write_block_s3(block: Any, filename: AnyStr, grid_entry: Tuple,
grid_meta: Dict):
return np.array(
StoredArrayS3(filename,
ArrayGrid.from_meta(grid_meta)).put(grid_entry, block),
dtype=dict,
)
def test_array_rwd():
X: np.ndarray = np.random.random(3)
stored_X = StoredArrayS3("darrays/%s_X" % "__test__")
stored_X.put_grid(
ArrayGrid(shape=X.shape,
block_shape=X.shape,
dtype=np.float64.__name__))
stored_X.init_grid()
stored_X.put_array(X)
assert np.allclose(X, stored_X.get_array())
stored_X.del_array()
stored_X.delete_grid()
def delete_s3(self, filename: str):
grid = self._get_array_grid(filename, StoredArrayS3)
grid_entry = tuple(np.zeros_like(grid.shape, dtype=np.int))
result = self._fs.delete_meta_s3(filename,
syskwargs={
"grid_entry": grid_entry,
"grid_shape": grid.grid_shape
})
deleted_key = self.cm.get(result).item()["Deleted"][0]["Key"]
assert deleted_key == StoredArrayS3(filename, grid).get_meta_key()
results: BlockArray = self._delete(filename, StoredArrayS3,
self._fs.delete_block_s3)
return results
def delete_s3(self, filename: str) -> bool:
grid = self._get_array_grid(filename, StoredArrayS3)
grid_entry = tuple(np.zeros_like(grid.shape, dtype=np.int))
result = self._fs.delete_meta_s3(
filename,
syskwargs={"grid_entry": grid_entry, "grid_shape": grid.grid_shape},
)
deleted_key = self.cm.get(result).item()["Deleted"][0]["Key"]
assert deleted_key == StoredArrayS3(filename, grid).get_meta_key()
results: BlockArray = self._delete(
filename, StoredArrayS3, self._fs.delete_block_s3
)
# Test whether data has been deleted from S3.
data_deleted = True
delete_result_arr = self.get(results)
stored_array = StoredArrayS3(filename, results.grid)
for grid_entry in results.grid.get_entry_iterator():
deleted_key = delete_result_arr[grid_entry]["Deleted"][0]["Key"]
entry_deleted = deleted_key == stored_array.get_key(grid_entry)
data_deleted = data_deleted and entry_deleted
return data_deleted
def test_array_rwd():
conn = boto3.resource("s3", region_name="us-east-1")
assert conn.Bucket("darrays") not in conn.buckets.all()
conn.create_bucket(Bucket="darrays")
X: np.ndarray = np.random.random(3)
stored_X = StoredArrayS3("darrays/%s_X" % "__test__")
stored_X.put_grid(
ArrayGrid(shape=X.shape, block_shape=X.shape, dtype=np.float64.__name__)
)
stored_X.init_grid()
stored_X.put_array(X)
assert np.allclose(X, stored_X.get_array())
stored_X.del_array()
stored_X.delete_grid()
def test_rwd(app_inst: ArrayApplication):
array: np.ndarray = np.random.random(35).reshape(7, 5)
ba: BlockArray = app_inst.array(array, block_shape=(3, 4))
filename = "darrays/read_write_delete_array_test"
write_result: BlockArray = app_inst.write_s3(ba, filename)
write_result_arr = app_inst.get(write_result)
for grid_entry in write_result.grid.get_entry_iterator():
assert 'ETag' in write_result_arr[grid_entry]
ba_read: BlockArray = app_inst.read_s3(filename)
assert app_inst.get(app_inst.allclose(ba, ba_read))
delete_result: BlockArray = app_inst.delete_s3(filename)
delete_result_arr = app_inst.get(delete_result)
for grid_entry in delete_result.grid.get_entry_iterator():
deleted_key = delete_result_arr[grid_entry]["Deleted"][0]["Key"]
assert deleted_key == StoredArrayS3(
filename, delete_result.grid).get_key(grid_entry)
def test_rwd(serial_app_inst: ArrayApplication):
conn = boto3.resource('s3', region_name='us-east-1')
assert conn.Bucket('darrays') not in conn.buckets.all()
conn.create_bucket(Bucket='darrays')
array: np.ndarray = np.random.random(35).reshape(7, 5)
ba: BlockArray = serial_app_inst.array(array, block_shape=(3, 4))
filename = "darrays/read_write_delete_array_test"
write_result: BlockArray = serial_app_inst.write_s3(ba, filename)
write_result_arr = serial_app_inst.get(write_result)
for grid_entry in write_result.grid.get_entry_iterator():
assert 'ETag' in write_result_arr[grid_entry]
ba_read: BlockArray = serial_app_inst.read_s3(filename)
assert serial_app_inst.get(serial_app_inst.allclose(ba, ba_read))
delete_result: BlockArray = serial_app_inst.delete_s3(filename)
delete_result_arr = serial_app_inst.get(delete_result)
for grid_entry in delete_result.grid.get_entry_iterator():
deleted_key = delete_result_arr[grid_entry]["Deleted"][0]["Key"]
assert deleted_key == StoredArrayS3(
filename, delete_result.grid).get_key(grid_entry)
def delete_block_s3(filename: AnyStr, grid_entry: Tuple, grid_meta: Dict):
return np.array(StoredArrayS3(
filename, ArrayGrid.from_meta(grid_meta)).delete(grid_entry),
dtype=dict)
def read_block_s3(filename: AnyStr, grid_entry: Tuple, grid_meta: Dict):
return StoredArrayS3(filename,
ArrayGrid.from_meta(grid_meta)).get(grid_entry)
def delete_meta_s3(filename: AnyStr):
sa: StoredArrayS3 = StoredArrayS3(filename)
sa.init_grid()
return np.array(sa.delete_grid(), dtype=dict)
def write_meta_s3(filename: AnyStr, grid_meta: Dict):
sa: StoredArrayS3 = StoredArrayS3(filename, ArrayGrid.from_meta(grid_meta))
return np.array(sa.put_grid(sa.grid), dtype=dict)
