def write_block_s3(block: Any, filename: AnyStr, grid_entry: Tuple, grid_meta: Dict): return np.array( StoredArrayS3(filename, ArrayGrid.from_meta(grid_meta)).put(grid_entry, block), dtype=dict, )
def test_array_rwd(): X: np.ndarray = np.random.random(3) stored_X = StoredArrayS3("darrays/%s_X" % "__test__") stored_X.put_grid( ArrayGrid(shape=X.shape, block_shape=X.shape, dtype=np.float64.__name__)) stored_X.init_grid() stored_X.put_array(X) assert np.allclose(X, stored_X.get_array()) stored_X.del_array() stored_X.delete_grid()
def delete_s3(self, filename: str): grid = self._get_array_grid(filename, StoredArrayS3) grid_entry = tuple(np.zeros_like(grid.shape, dtype=np.int)) result = self._fs.delete_meta_s3(filename, syskwargs={ "grid_entry": grid_entry, "grid_shape": grid.grid_shape }) deleted_key = self.cm.get(result).item()["Deleted"][0]["Key"] assert deleted_key == StoredArrayS3(filename, grid).get_meta_key() results: BlockArray = self._delete(filename, StoredArrayS3, self._fs.delete_block_s3) return results
def delete_s3(self, filename: str) -> bool: grid = self._get_array_grid(filename, StoredArrayS3) grid_entry = tuple(np.zeros_like(grid.shape, dtype=np.int)) result = self._fs.delete_meta_s3( filename, syskwargs={"grid_entry": grid_entry, "grid_shape": grid.grid_shape}, ) deleted_key = self.cm.get(result).item()["Deleted"][0]["Key"] assert deleted_key == StoredArrayS3(filename, grid).get_meta_key() results: BlockArray = self._delete( filename, StoredArrayS3, self._fs.delete_block_s3 ) # Test whether data has been deleted from S3. data_deleted = True delete_result_arr = self.get(results) stored_array = StoredArrayS3(filename, results.grid) for grid_entry in results.grid.get_entry_iterator(): deleted_key = delete_result_arr[grid_entry]["Deleted"][0]["Key"] entry_deleted = deleted_key == stored_array.get_key(grid_entry) data_deleted = data_deleted and entry_deleted return data_deleted
def test_array_rwd(): conn = boto3.resource("s3", region_name="us-east-1") assert conn.Bucket("darrays") not in conn.buckets.all() conn.create_bucket(Bucket="darrays") X: np.ndarray = np.random.random(3) stored_X = StoredArrayS3("darrays/%s_X" % "__test__") stored_X.put_grid( ArrayGrid(shape=X.shape, block_shape=X.shape, dtype=np.float64.__name__) ) stored_X.init_grid() stored_X.put_array(X) assert np.allclose(X, stored_X.get_array()) stored_X.del_array() stored_X.delete_grid()
def test_rwd(app_inst: ArrayApplication): array: np.ndarray = np.random.random(35).reshape(7, 5) ba: BlockArray = app_inst.array(array, block_shape=(3, 4)) filename = "darrays/read_write_delete_array_test" write_result: BlockArray = app_inst.write_s3(ba, filename) write_result_arr = app_inst.get(write_result) for grid_entry in write_result.grid.get_entry_iterator(): assert 'ETag' in write_result_arr[grid_entry] ba_read: BlockArray = app_inst.read_s3(filename) assert app_inst.get(app_inst.allclose(ba, ba_read)) delete_result: BlockArray = app_inst.delete_s3(filename) delete_result_arr = app_inst.get(delete_result) for grid_entry in delete_result.grid.get_entry_iterator(): deleted_key = delete_result_arr[grid_entry]["Deleted"][0]["Key"] assert deleted_key == StoredArrayS3( filename, delete_result.grid).get_key(grid_entry)
def test_rwd(serial_app_inst: ArrayApplication): conn = boto3.resource('s3', region_name='us-east-1') assert conn.Bucket('darrays') not in conn.buckets.all() conn.create_bucket(Bucket='darrays') array: np.ndarray = np.random.random(35).reshape(7, 5) ba: BlockArray = serial_app_inst.array(array, block_shape=(3, 4)) filename = "darrays/read_write_delete_array_test" write_result: BlockArray = serial_app_inst.write_s3(ba, filename) write_result_arr = serial_app_inst.get(write_result) for grid_entry in write_result.grid.get_entry_iterator(): assert 'ETag' in write_result_arr[grid_entry] ba_read: BlockArray = serial_app_inst.read_s3(filename) assert serial_app_inst.get(serial_app_inst.allclose(ba, ba_read)) delete_result: BlockArray = serial_app_inst.delete_s3(filename) delete_result_arr = serial_app_inst.get(delete_result) for grid_entry in delete_result.grid.get_entry_iterator(): deleted_key = delete_result_arr[grid_entry]["Deleted"][0]["Key"] assert deleted_key == StoredArrayS3( filename, delete_result.grid).get_key(grid_entry)
def delete_block_s3(filename: AnyStr, grid_entry: Tuple, grid_meta: Dict): return np.array(StoredArrayS3( filename, ArrayGrid.from_meta(grid_meta)).delete(grid_entry), dtype=dict)
def read_block_s3(filename: AnyStr, grid_entry: Tuple, grid_meta: Dict): return StoredArrayS3(filename, ArrayGrid.from_meta(grid_meta)).get(grid_entry)
def delete_meta_s3(filename: AnyStr): sa: StoredArrayS3 = StoredArrayS3(filename) sa.init_grid() return np.array(sa.delete_grid(), dtype=dict)
def write_meta_s3(filename: AnyStr, grid_meta: Dict): sa: StoredArrayS3 = StoredArrayS3(filename, ArrayGrid.from_meta(grid_meta)) return np.array(sa.put_grid(sa.grid), dtype=dict)