def test_ndarray_fs():
location = 'cloudvolume-shm-test-ndarray'
array_like, array = shm.ndarray_fs(shape=(2, 2, 2),
dtype=np.uint8,
location=location,
lock=None,
emulate_shm=True)
assert np.all(array == np.zeros(shape=(2, 2, 2), dtype=np.uint8))
array[:] = 100
array_like.close()
array_like, array = shm.ndarray_fs(shape=(2, 2, 2),
dtype=np.uint8,
location=location,
lock=None,
emulate_shm=True)
assert np.all(array[:] == 100)
array_like.close()
full_loc = os.path.join(shm.EMULATED_SHM_DIRECTORY, location)
assert os.path.exists(full_loc)
assert os.path.getsize(full_loc) == 8
assert shm.unlink_fs(location) == True
assert shm.unlink_fs(location) == False
try:
array_like, array = shm.ndarray_fs(shape=(2, 2, 2),
dtype=np.uint8,
location=location,
lock=None,
readonly=True,
emulate_shm=True)
assert False
except shm.SharedMemoryReadError:
pass
array_like, array = shm.ndarray_fs(shape=(2, 2, 2),
dtype=np.uint8,
location=location,
lock=None,
emulate_shm=True)
try:
array_like, array = shm.ndarray_fs(shape=(200, 200, 200),
dtype=np.uint8,
location=location,
lock=None,
readonly=True,
emulate_shm=True)
assert False
except shm.SharedMemoryReadError:
pass
assert shm.unlink_fs(location) == True
assert shm.unlink_fs(location) == False
assert not os.path.exists(full_loc)
def child_process_download(meta, cache, mip, compress_cache, dest_bbox,
fill_missing, progress, location, use_shared_memory,
green, cloudpaths):
reset_connection_pools() # otherwise multi-process hangs
shape = list(dest_bbox.size3()) + [meta.num_channels]
if use_shared_memory:
array_like, dest_img = shm.ndarray(shape,
dtype=meta.dtype,
location=location,
lock=fs_lock)
else:
array_like, dest_img = shm.ndarray_fs(shape,
dtype=meta.dtype,
location=location,
emulate_shm=False,
lock=fs_lock)
def process(src_img, src_bbox):
shade(dest_img, dest_bbox, src_img, src_bbox)
download_chunks_threaded(meta,
cache,
mip,
cloudpaths,
fn=process,
fill_missing=fill_missing,
progress=progress,
compress_cache=compress_cache,
green=green)
array_like.close()
def multiprocess_download(requested_bbox,
mip,
cloudpaths,
meta,
cache,
compress_cache,
fill_missing,
progress,
parallel,
location,
retain,
use_shared_memory,
order,
green,
secrets=None,
background_color=0):
cloudpaths_by_process = []
length = int(math.ceil(len(cloudpaths) / float(parallel)) or 1)
for i in range(0, len(cloudpaths), length):
cloudpaths_by_process.append(cloudpaths[i:i + length])
cpd = partial(child_process_download, meta, cache, mip, compress_cache,
requested_bbox, fill_missing, progress, location,
use_shared_memory, green, secrets, background_color)
parallel_execution(cpd,
cloudpaths_by_process,
parallel,
cleanup_shm=location)
shape = list(requested_bbox.size3()) + [meta.num_channels]
if use_shared_memory:
mmap_handle, renderbuffer = shm.ndarray(shape,
dtype=meta.dtype,
order=order,
location=location,
lock=fs_lock)
else:
handle, renderbuffer = shm.ndarray_fs(shape,
dtype=meta.dtype,
order=order,
location=location,
lock=fs_lock,
emulate_shm=False)
if not retain:
if use_shared_memory:
shm.unlink(location)
else:
os.unlink(location)
return mmap_handle, renderbuffer
def multiprocess_download(
requested_bbox, mip, cloudpaths,
meta, cache, lru, compress_cache,
fill_missing, progress,
parallel, location,
retain, use_shared_memory, order,
green, secrets=None, background_color=0,
):
cpd = partial(child_process_download,
meta, cache,
mip, compress_cache, requested_bbox,
fill_missing, progress,
location, use_shared_memory,
green, secrets, background_color
)
if lru.size > 0:
for path in cloudpaths:
lru.pop(path, None)
parallel_execution(
cpd, cloudpaths, parallel,
progress=progress,
desc="Download",
cleanup_shm=location,
block_size=750,
)
shape = list(requested_bbox.size3()) + [ meta.num_channels ]
if use_shared_memory:
mmap_handle, renderbuffer = shm.ndarray(
shape, dtype=meta.dtype, order=order,
location=location, lock=fs_lock
)
else:
handle, renderbuffer = shm.ndarray_fs(
shape, dtype=meta.dtype, order=order,
location=location, lock=fs_lock,
emulate_shm=False
)
if meta.encoding(mip) == "raw":
repopulate_lru_from_shm(meta, mip, lru, renderbuffer, requested_bbox)
if not retain:
if use_shared_memory:
shm.unlink(location)
else:
os.unlink(location)
return mmap_handle, renderbuffer
def child_process_download(
meta, cache,
mip, compress_cache, dest_bbox,
fill_missing, progress,
location, use_shared_memory, green,
secrets, background_color, cloudpaths
):
reset_connection_pools() # otherwise multi-process hangs
shape = list(dest_bbox.size3()) + [ meta.num_channels ]
if use_shared_memory:
array_like, dest_img = shm.ndarray(
shape, dtype=meta.dtype,
location=location, lock=fs_lock
)
else:
array_like, dest_img = shm.ndarray_fs(
shape, dtype=meta.dtype,
location=location, emulate_shm=False,
lock=fs_lock
)
if background_color != 0:
dest_img[dest_bbox.to_slices()] = background_color
def process(src_img, src_bbox):
shade(dest_img, dest_bbox, src_img, src_bbox)
if progress:
# This is not good programming practice, but
# I could not find a clean way to do this that
# did not result in warnings about leaked semaphores.
# progress_queue is created in common.py:initialize_progress_queue
# as a global for this module.
progress_queue.put(1)
download_chunks_threaded(
meta, cache, None, mip, cloudpaths,
fn=process, decode_fn=decode, fill_missing=fill_missing,
progress=False, compress_cache=compress_cache,
green=green, secrets=secrets, background_color=background_color
)
array_like.close()
return len(cloudpaths)
def download(requested_bbox,
mip,
meta,
cache,
fill_missing,
progress,
parallel,
location,
retain,
use_shared_memory,
use_file,
compress,
order='F',
green=False,
secrets=None,
renumber=False,
background_color=0):
"""Cutout a requested bounding box from storage and return it as a numpy array."""
full_bbox = requested_bbox.expand_to_chunk_size(
meta.chunk_size(mip), offset=meta.voxel_offset(mip))
full_bbox = Bbox.clamp(full_bbox, meta.bounds(mip))
cloudpaths = list(
chunknames(full_bbox,
meta.bounds(mip),
meta.key(mip),
meta.chunk_size(mip),
protocol=meta.path.protocol))
shape = list(requested_bbox.size3()) + [meta.num_channels]
compress_cache = should_compress(meta.encoding(mip),
compress,
cache,
iscache=True)
handle = None
if renumber and (parallel != 1):
raise ValueError("renumber is not supported for parallel operation.")
if use_shared_memory and use_file:
raise ValueError(
"use_shared_memory and use_file are mutually exclusive arguments.")
dtype = np.uint16 if renumber else meta.dtype
if parallel == 1:
if use_shared_memory: # write to shared memory
handle, renderbuffer = shm.ndarray(shape,
dtype=dtype,
order=order,
location=location,
lock=fs_lock)
if not retain:
shm.unlink(location)
elif use_file: # write to ordinary file
handle, renderbuffer = shm.ndarray_fs(shape,
dtype=dtype,
order=order,
location=location,
lock=fs_lock,
emulate_shm=False)
if not retain:
os.unlink(location)
else:
renderbuffer = np.full(shape=shape,
fill_value=background_color,
dtype=dtype,
order=order)
def process(img3d, bbox):
shade(renderbuffer, requested_bbox, img3d, bbox)
remap = {background_color: background_color}
lock = threading.Lock()
N = 1
def process_renumber(img3d, bbox):
nonlocal N
nonlocal lock
nonlocal remap
nonlocal renderbuffer
img_labels = fastremap.unique(img3d)
with lock:
for lbl in img_labels:
if lbl not in remap:
remap[lbl] = N
N += 1
if N > np.iinfo(renderbuffer.dtype).max:
renderbuffer = fastremap.refit(renderbuffer,
value=N,
increase_only=True)
fastremap.remap(img3d, remap, in_place=True)
shade(renderbuffer, requested_bbox, img3d, bbox)
fn = process
if renumber and not (use_file or use_shared_memory):
fn = process_renumber
download_chunks_threaded(meta,
cache,
mip,
cloudpaths,
fn=fn,
fill_missing=fill_missing,
progress=progress,
compress_cache=compress_cache,
green=green,
secrets=secrets,
background_color=background_color)
else:
handle, renderbuffer = multiprocess_download(
requested_bbox,
mip,
cloudpaths,
meta,
cache,
compress_cache,
fill_missing,
progress,
parallel,
location,
retain,
use_shared_memory=(use_file == False),
order=order,
green=green,
secrets=secrets,
background_color=background_color)
out = VolumeCutout.from_volume(meta,
mip,
renderbuffer,
requested_bbox,
handle=handle)
if renumber:
return (out, remap)
return out
def download(requested_bbox,
mip,
meta,
cache,
fill_missing,
progress,
parallel,
location,
retain,
use_shared_memory,
use_file,
compress,
order='F',
green=False):
"""Cutout a requested bounding box from storage and return it as a numpy array."""
full_bbox = requested_bbox.expand_to_chunk_size(
meta.chunk_size(mip), offset=meta.voxel_offset(mip))
full_bbox = Bbox.clamp(full_bbox, meta.bounds(mip))
cloudpaths = list(
chunknames(full_bbox,
meta.bounds(mip),
meta.key(mip),
meta.chunk_size(mip),
protocol=meta.path.protocol))
shape = list(requested_bbox.size3()) + [meta.num_channels]
compress_cache = should_compress(meta.encoding(mip),
compress,
cache,
iscache=True)
handle = None
if use_shared_memory and use_file:
raise ValueError(
"use_shared_memory and use_file are mutually exclusive arguments.")
if parallel == 1:
if use_shared_memory: # write to shared memory
handle, renderbuffer = shm.ndarray(shape,
dtype=meta.dtype,
order=order,
location=location,
lock=fs_lock)
if not retain:
shm.unlink(location)
elif use_file: # write to ordinary file
handle, renderbuffer = shm.ndarray_fs(shape,
dtype=meta.dtype,
order=order,
location=location,
lock=fs_lock,
emulate_shm=False)
if not retain:
os.unlink(location)
else:
renderbuffer = np.zeros(shape=shape, dtype=meta.dtype, order=order)
def process(img3d, bbox):
shade(renderbuffer, requested_bbox, img3d, bbox)
download_chunks_threaded(meta,
cache,
mip,
cloudpaths,
fn=process,
fill_missing=fill_missing,
progress=progress,
compress_cache=compress_cache,
green=green)
else:
handle, renderbuffer = multiprocess_download(
requested_bbox,
mip,
cloudpaths,
meta,
cache,
compress_cache,
fill_missing,
progress,
parallel,
location,
retain,
use_shared_memory=(use_file == False),
order=order,
green=green,
)
return VolumeCutout.from_volume(meta,
mip,
renderbuffer,
requested_bbox,
handle=handle)
