def _create_position_array(self, pos):
"""
maps all of the tiff data into a virtual zarr store in memory for a given position
Parameters
----------
pos: (int) index of the position to create array under
Returns
-------
"""
# intialize virtual zarr store and save it under positions
timepoints, channels, slices = self._get_dimensions(pos)
self.position_arrays[pos] = zarr.empty(
shape=(timepoints, channels, slices, self.height, self.width),
chunks=(1, 1, 1, self.height, self.width),
dtype=self.dtype)
# add all the images with this specific dimension. Will be blank images if dataset
# is incomplete
for p, t, c, z in self.coord_map.keys():
if p == pos:
self.position_arrays[pos][t, c, z, :, :] = self.get_image(
pos, t, c, z)
def test_empty_like():
z = empty(100, 10)
z2 = empty_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.cname, z2.cname)
eq(z.clevel, z2.clevel)
eq(z.shuffle, z2.shuffle)
eq(z.fill_value, z2.fill_value)
def test_empty_like():
z = empty(100, 10)
z2 = empty_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.cname, z2.cname)
eq(z.clevel, z2.clevel)
eq(z.shuffle, z2.shuffle)
eq(z.fill_value, z2.fill_value)
def _zarr_empty(shape, store_or_group, chunks, dtype, name=None, **kwargs):
# wrapper that maybe creates the array within a group
if name is not None:
assert isinstance(store_or_group, zarr.hierarchy.Group)
return store_or_group.empty(
name, shape=shape, chunks=chunks, dtype=dtype, **kwargs
)
else:
return zarr.empty(
shape, chunks=chunks, dtype=dtype, store=store_or_group, **kwargs
)
def submit_job(job_id):
import s3fs
client = cluster.get_client()
fs = s3fs.S3FileSystem(
use_ssl=True,
client_kwargs=dict(
endpoint_url="https://js2.jetstream-cloud.org:8001/",
region_name="RegionOne",
),
)
store = s3fs.S3Map(root=f"gateway-results/{job_id}",
s3=fs) # , check=False)
z = zarr.empty(shape=(1000, 1000),
chunks=(100, 100),
dtype="f4",
store=store,
compression=None)
x = da.random.random(size=z.shape, chunks=z.chunks).astype(z.dtype)
x.store(z, lock=False)
return "Submitted job {}\n".format(job_id)
def rechunk_zarr2zarr_w_dask(source_array,
target_chunks,
max_mem,
target_store,
temp_store=None,
source_storage_options={},
temp_storage_options={},
target_storage_options={}):
shape = source_array.shape
source_chunks = source_array.chunks
dtype = source_array.dtype
itemsize = dtype.itemsize
read_chunks, int_chunks, write_chunks = rechunking_plan(
shape, source_chunks, target_chunks, itemsize, max_mem)
source_read = dsa.from_zarr(source_array,
chunks=read_chunks,
storage_options=source_storage_options)
# create target
target_array = zarr.empty(shape,
chunks=target_chunks,
dtype=dtype,
store=target_store)
target_array.attrs.update(source_array.attrs)
if int_chunks == target_chunks:
target_store_delayed = dsa.store(source_read,
target_array,
lock=False,
compute=False)
print("One step rechunking plan")
return target_store_delayed
else:
# do intermediate store
assert temp_store is not None
int_array = zarr.empty(shape,
chunks=int_chunks,
dtype=dtype,
store=temp_store)
intermediate_store_delayed = dsa.store(source_read,
int_array,
lock=False,
compute=False)
int_read = dsa.from_zarr(int_array,
chunks=write_chunks,
storage_options=temp_storage_options)
target_store_delayed = dsa.store(int_read,
target_array,
lock=False,
compute=False)
# now do some hacking to chain these together into a single graph.
# get the two graphs as dicts
int_dsk = dask.utils.ensure_dict(intermediate_store_delayed.dask)
target_dsk = dask.utils.ensure_dict(target_store_delayed.dask)
# find the root store key representing the read
root_keys = []
for key in target_dsk:
if isinstance(key, str):
if key.startswith('from-zarr'):
root_keys.append(key)
assert len(root_keys) == 1
root_key = root_keys[0]
# now rewrite the graph
target_dsk[root_key] = (lambda a, *b: a, target_dsk[root_key],
*int_dsk[intermediate_store_delayed.key])
target_dsk.update(int_dsk)
# fuse
dsk_fused, deps = fuse(target_dsk)
delayed_fused = Delayed(target_store_delayed.key, dsk_fused)
print("Two step rechunking plan")
return delayed_fused
for block, info in a.iter_write():
block[:] = content[info.slice]
acratio = a.cratio
if persistent:
del a
t1 = time()
print("Time for filling array (caterva, iter): %.3fs ; CRatio: %.1fx" % ((t1 - t0), acratio))
# Create and fill a zarr array
t0 = time()
compressor = numcodecs.Blosc(cname=cname, clevel=clevel, shuffle=filter, blocksize=blocksize)
numcodecs.blosc.set_nthreads(nthreads)
if persistent:
z = zarr.open(fname_zarr, mode='w', shape=shape, chunks=chunkshape, dtype=dtype, compressor=compressor)
else:
z = zarr.empty(shape=shape, chunks=chunkshape, dtype=dtype, compressor=compressor)
z[:] = content
zratio = z.nbytes / z.nbytes_stored
if persistent:
del z
t1 = time()
print("Time for filling array (zarr): %.3fs ; CRatio: %.1fx" % ((t1 - t0), zratio))
# Create and fill a hdf5 array
t0 = time()
filters = tables.Filters(complevel=clevel, complib="blosc:%s" % cname, shuffle=True)
tables.set_blosc_max_threads(nthreads)
if persistent:
h5f = tables.open_file(fname_h5, 'w')
else:
h5f = tables.open_file(fname_h5, 'w', driver='H5FD_CORE', driver_core_backing_store=0)
def test_empty():
z = empty(100, 10)
eq((100, ), z.shape)
eq((10, ), z.chunks)
def test_empty():
z = empty(100, 10)
eq((100,), z.shape)
eq((10,), z.chunks)