def test_concatenate_and_rechunk__shape_mismatch():
z1 = zarr.zeros((5, 3), chunks=(2, 3), dtype="i4")
z2 = zarr.zeros((5, 4), chunks=(2, 4), dtype="i4")
zarrs = [z1, z2]
with pytest.raises(ValueError, match="Zarr arrays must have matching shapes"):
concatenate_and_rechunk(zarrs)
def test_run_simulator_with_threads_and_zarr_memory_store():
"""
If the store is in memory (here a Zarr MemoryStore) and the Dask workers
share memory with the client (i.e. we have a threads-based cluster),
collect_in_memory can be set to False.
"""
cluster = LocalCluster(n_workers=2, processes=False, threads_per_worker=1)
simulator = Simulator(model, sim_shapes=dict(x=(10, )), cluster=cluster)
pars = zarr.zeros((100, 2))
pars[:, :] = np.random.random(pars.shape)
x = zarr.zeros((100, 10))
sims = dict(x=x.oindex)
sim_status = zarr.full(100, SimulationStatus.RUNNING, dtype="int")
# the following is non-blocking (it immediately returns)
simulator.run(
pars=pars,
sims=sims,
sim_status=sim_status.oindex,
indices=np.arange(100, dtype=np.int),
collect_in_memory=False,
batch_size=20,
)
# need to wait for tasks to be completed
_wait_for_all_tasks()
assert np.all(sim_status[:] == SimulationStatus.FINISHED)
assert not np.all(np.isclose(sims["x"][:, :].sum(axis=1), 0.0))
simulator.client.close()
cluster.close()
def test_run_simulator_with_processes_and_zarr_memory_store():
"""
If the store is in memory (here a Zarr MemoryStore) and the Dask workers do
not share memory with the client (i.e. we have a processes-based cluster),
collect_in_memory must be set to True.
"""
cluster = LocalCluster(n_workers=2, processes=True, threads_per_worker=1)
simulator = Simulator(model, sim_shapes=dict(x=(10, )), cluster=cluster)
pars = zarr.zeros((100, 2))
pars[:, :] = np.random.random(pars.shape)
x = zarr.zeros((100, 10))
sims = dict(x=x.oindex)
sim_status = zarr.full(100, SimulationStatus.RUNNING, dtype="int")
simulator.run(
pars=pars,
sims=sims,
sim_status=sim_status.oindex,
indices=np.arange(100, dtype=np.int),
collect_in_memory=True,
batch_size=20,
)
assert np.all(sim_status[:] == SimulationStatus.FINISHED)
assert not np.all(np.isclose(sims["x"][:, :].sum(axis=1), 0.0))
simulator.client.close()
cluster.close()
def test_fancy_indexing_fallback_on_get_setitem():
z = zarr.zeros((20, 20))
z[[1, 2, 3], [1, 2, 3]] = 1
np.testing.assert_array_equal(
z[:4, :4],
[
[0, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
],
)
np.testing.assert_array_equal(
z[[1, 2, 3], [1, 2, 3]], 1
)
# test broadcasting
np.testing.assert_array_equal(
z[1, [1, 2, 3]], [1, 0, 0]
)
# test 1D fancy indexing
z2 = zarr.zeros(5)
z2[[1, 2, 3]] = 1
np.testing.assert_array_equal(
z2, [0, 1, 1, 1, 0]
)
def _create_stores(self, p):
"""
extract all singlepage tiffs at each coordinate and place them in a zarr array
coordinates are of shape = (pos, time, channel, z)
arrays are of shape = (time, channel, z, height, width)
Parameters
----------
p: (int) position (aka ome-tiff scene) to extract
Returns
-------
"""
self.log.info("")
z = zarr.zeros(shape=(self.frames, self.channels, self.slices,
self.height, self.width),
chunks=(1, 1, 1, self.height, self.width))
for c, fn in self.coord_to_filename.items():
if c[0] == p:
self.log.info(f"reading coord = {c} from filename = {fn}")
z[c[1], c[2], c[3]] = zarr.open(tiff.imread(fn, aszarr=True))
# check that the array was assigned
if z == zarr.zeros(shape=(self.frames, self.channels, self.slices,
self.height, self.width),
chunks=(1, 1, 1, self.height, self.width)):
raise IOError(f"array at position {p} can not be found")
self.positions[p] = z
def test__assert_calendars_same(source_attr, target_attr, expected_error):
source_array = zarr.zeros((5))
for k, v in source_attr.items():
source_array.attrs[k] = v
target_array = zarr.zeros((5))
for k, v in target_attr.items():
target_array.attrs[k] = v
with pytest.raises(expected_error):
append._assert_calendars_same(source_array, target_array)
def reset_states(self, input_shape=None):
"""Initialize the state space
This method initializes the layer and resets any previously held data.
The zarr array is initialized in this method.
Args:
input_shape (TensorShape,tuple, list): Shape of the input.
"""
if not isinstance(input_shape, type(None)):
self._input_shape = input_shape
if self._input_shape == None:
raise ValueError(
'The input_shape is None, and no previous input ' +
'shape information was provided. The first time ' +
'reset_states is called, an input_shape must be ' +
'provided.')
# Try to keep chunks limited to 16MB
ncols = int(np.ceil(self._input_shape[self._channel_index] / 8))
nrows = 2**22 // ncols
# Initialize internal variables related to state space
self._state_ids = None
self._edges = None
self._index = None
self._counts = None
self._entropy = None
self._threads = []
self._chunk_size = (nrows, ncols)
self._state_shape = list(self._chunk_size)
self._state_count = 0
if self._raw_states != None:
# Zero out states and resize if zarr already open
self._raw_states.resize(self._state_shape)
self._raw_states[:] = 0
else:
# Initialize the zarr array
if self._zarr_path != None:
if self._zarr_path.is_file():
self._zarr_path.unlink()
self._raw_states = zarr.zeros(
shape=self._state_shape,
chunks=self._chunk_size,
dtype='B',
synchronizer=zarr.ThreadSynchronizer(),
store=str(self._zarr_path.absolute()))
else:
self._raw_states = zarr.zeros(
shape=self._state_shape,
chunks=self._chunk_size,
dtype='B',
synchronizer=zarr.ThreadSynchronizer())
def test_concatenate_and_rechunk__2d():
z1 = zarr.zeros((5, 3), chunks=(2, 3), dtype="i4")
z1[:] = np.arange(15).reshape(5, 3)
z2 = zarr.zeros((5, 3), chunks=(2, 3), dtype="i4")
z2[:] = np.arange(15, 30).reshape(5, 3)
zarrs = [z1, z2]
out = concatenate_and_rechunk(zarrs)
assert out.chunks == ((2, 2, 2, 2, 2), (3,))
np.testing.assert_array_equal(out.compute(), np.arange(30).reshape(10, 3))
def test_concatenate_and_rechunk__1d():
z1 = zarr.zeros(5, chunks=2, dtype="i4")
z1[:] = np.arange(5)
z2 = zarr.zeros(5, chunks=2, dtype="i4")
z2[:] = np.arange(5, 10)
zarrs = [z1, z2]
out = concatenate_and_rechunk(zarrs)
assert out.chunks == ((2, 2, 2, 2, 2),)
np.testing.assert_array_equal(out.compute(), np.arange(10))
def _start_flood_fill(self, pos):
self._stop_flood_fill()
inf_results = zarr.zeros(
self.gt_vol.bounds.to_list()[3:][::-1], chunks=(64, 64, 64), dtype=np.uint8)
inf_volume = neuroglancer.LocalVolume(
data=inf_results, voxel_size=list(self.gt_vol.resolution))
with self.viewer.txn() as s:
s.layers['points'] = neuroglancer.AnnotationLayer()
s.layers['inference'] = neuroglancer.ImageLayer(
source=inf_volume,
shader='''
void main() {
float v = toNormalized(getDataValue(0));
vec4 rgba = vec4(0,0,0,0);
if (v != 0.0) {
rgba = vec4(colormapJet(v), 1.0);
}
emitRGBA(rgba);
}
''',
)
self.flood_fill_event = threading.Event()
t = threading.Thread(
target=self._do_flood_fill,
kwargs=dict(
initial_pos=pos,
inf_results=inf_results,
inf_volume=inf_volume,
event=self.flood_fill_event,
))
t.daemon = True
t.start()
def test_cache():
store = SlowStore()
store = Cache(store, max_size=1000000)
store.flush()
for i in range(10):
z = zarr.zeros(
(1000, 1000),
chunks=(100, 100),
path=posixpath.realpath(f"./data/test/test_cache/first{i}"),
store=store,
overwrite=True,
)
z[...] = i
store.invalidate()
t1 = time.time()
z[...]
t2 = time.time()
z[...]
t3 = time.time()
assert z[0, 0] == i
# print(t2 - t1, t3 - t2)
assert t2 - t1 > t3 - t2
store.close()
def _start_flood_fill(self, pos):
self._stop_flood_fill()
inf_results = zarr.zeros(self.gt_vol.bounds.to_list()[3:],
chunks=(64, 64, 64),
dtype=np.uint8)
inf_volume = neuroglancer.LocalVolume(data=inf_results,
dimensions=self.dimensions)
with self.viewer.txn() as s:
s.layers['points'] = neuroglancer.LocalAnnotationLayer(
self.dimensions)
s.layers['inference'] = neuroglancer.ImageLayer(
source=inf_volume,
shader='''
void main() {
float v = toNormalized(getDataValue(0));
vec4 rgba = vec4(0,0,0,0);
if (v != 0.0) {
rgba = vec4(colormapJet(v), 1.0);
}
emitRGBA(rgba);
}
''',
)
self.flood_fill_event = threading.Event()
t = threading.Thread(target=self._do_flood_fill,
kwargs=dict(
initial_pos=pos,
inf_results=inf_results,
inf_volume=inf_volume,
event=self.flood_fill_event,
))
t.daemon = True
t.start()
def test_fancy_indexing_doesnt_mix_with_slicing():
z = zarr.zeros((20, 20))
with pytest.raises(IndexError):
z[[1, 2, 3], :] = 2
with pytest.raises(IndexError):
np.testing.assert_array_equal(
z[[1, 2, 3], :], 0
)
def test_concatenate_and_rechunk__tiny_file():
z1 = zarr.zeros(4, chunks=3, dtype="i4")
z1[:] = np.arange(4)
# this zarr array lies entirely within the second chunk
z2 = zarr.zeros(1, chunks=3, dtype="i4")
z2[:] = np.arange(4, 5)
z3 = zarr.zeros(5, chunks=3, dtype="i4")
z3[:] = np.arange(5, 10)
zarrs = [z1, z2, z3]
out = concatenate_and_rechunk(zarrs)
assert out.chunks == ((3, 3, 3, 1),)
np.testing.assert_array_equal(out.compute(), np.arange(10))
def test_zarr_dask_2D(viewer_factory):
"""Test adding 2D dask image."""
view, viewer = viewer_factory()
data = zarr.zeros((200, 100), chunks=(40, 20))
data[53:63, 10:20] = 1
zdata = da.from_zarr(data)
viewer.add_image(zdata)
assert np.all(viewer.layers[0].data == zdata)
def test_zarr_nD():
"""Test adding nD zarr image."""
viewer = ViewerModel()
data = zarr.zeros((200, 100, 50), chunks=(40, 20, 10))
data[53:63, 10:20, :] = 1
# If passing a zarr file directly, must pass contrast_limits
viewer.add_image(data, contrast_limits=[0, 1])
assert np.all(viewer.layers[0].data == data)
def test_zarr_2D(viewer_factory):
"""Test adding 2D zarr image."""
view, viewer = viewer_factory()
data = zarr.zeros((200, 100), chunks=(40, 20))
data[53:63, 10:20] = 1
# If passing a zarr file directly, must pass contrast_limits
viewer.add_image(data, contrast_limits=[0, 1])
assert np.all(viewer.layers[0].data == data)
def setUp(self):
self.z = zarr.zeros((25, 25), chunks=(5, 5), dtype=np.float32)
self.z[:] = np.arange(25**2).reshape((25, 25))
self.f_zarr = double_elements
self.shm = shm
with shm.txn() as arr:
arr[:] = self.z[:]
self.f_shm = double_elements_shm
def test_zarr_dask_nD():
"""Test adding nD zarr image."""
viewer = ViewerModel()
data = zarr.zeros((200, 100, 50), chunks=(40, 20, 10))
data[53:63, 10:20, :] = 1
zdata = da.from_zarr(data)
viewer.add_image(zdata)
assert np.all(viewer.layers[0].data == zdata)
def test_zeros_like():
z = zeros(100, 10)
z2 = zeros_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.cname, z2.cname)
eq(z.clevel, z2.clevel)
eq(z.shuffle, z2.shuffle)
eq(z.fill_value, z2.fill_value)
def test_zeros_like():
z = zeros(100, 10)
z2 = zeros_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.cname, z2.cname)
eq(z.clevel, z2.clevel)
eq(z.shuffle, z2.shuffle)
eq(z.fill_value, z2.fill_value)
def test_ensure_dask_array():
np_arr = np.zeros((128, 128, 3), dtype=np.uint8)
da_arr = da.zeros((128, 128, 3), dtype=np.uint8)
za_arr = zarr.zeros((128, 128, 3), dtype=np.uint8)
np_out = ensure_dask_array(np_arr)
da_out = ensure_dask_array(da_arr)
za_out = ensure_dask_array(za_arr)
assert isinstance(np_out, da.Array)
assert isinstance(da_out, da.Array)
assert isinstance(za_out, da.Array)
def write_n5(path, shape, block_size, compressor):
store = zarr.N5Store(path)
data = np.arange(np.prod(shape), dtype=np.uint16)
data = data.reshape(shape)
data_transpose = data.transpose()
z = zarr.zeros(
data_transpose.shape,
chunks=block_size[::-1],
store=store,
dtype=data.dtype,
overwrite=True,
compressor=compressor)
z[...] = data_transpose
def test_zarr_nD(qtbot):
"""Test adding nD zarr image."""
viewer = Viewer()
view = viewer.window.qt_viewer
qtbot.addWidget(view)
data = zarr.zeros((200, 100, 50), chunks=(40, 20, 10))
data[53:63, 10:20, :] = 1
# If passing a zarr file directly, must pass contrast_limits
viewer.add_image(data, contrast_limits=[0, 1])
assert np.all(viewer.layers[0].data == data)
# Close the viewer
viewer.window.close()
def test_zarr_dask_nD(qtbot):
"""Test adding nD zarr image."""
viewer = Viewer()
view = viewer.window.qt_viewer
qtbot.addWidget(view)
data = zarr.zeros((200, 100, 50), chunks=(40, 20, 10))
data[53:63, 10:20, :] = 1
zdata = da.from_zarr(data)
viewer.add_image(zdata)
assert np.all(viewer.layers[0].data == zdata)
# Close the viewer
viewer.window.close()
def dummy_matrix_sequential(self):
"""Convert COO matrix format in Pytables to zarr array with
One-Hot-Encoding or Dummy matrix format
BAD PERFORMANCE
Args:
None
Returns:
zarr.array
"""
def assign(za, chunk):
for row in chunk:
za[row['row'], row['col']] = 1
h5file = pt.open_file(self.pytables_filepath, mode='r')
ncol = h5file.root.kmers.nrows
nrow = h5file.root.genomes.nrows
idxtable = h5file.root.indexes
print(nrow, ncol)
za = zarr.zeros((nrow, ncol), chunks=(nrow, 1000), dtype='u1')
bsize = 1000
t = h5file.root.indexes.nrows
t0 = time.time()
for b in range(0, t, bsize):
assign(za, idxtable[b:(bsize + b)])
if b % 100000 == 0:
t1 = time.time()
diff = int(t1 - t0)
minutes, seconds = diff // 60, diff % 60
print("{} complete.\tElapsed time {}:{}".format(
round(b / t * 100, 1), str(minutes),
str(seconds).zfill(2)))
# Write to file
# store = zarr.DirectoryStore(self.zarr_filepath)
# group=zarr.hierarchy.group(store=store,overwrite=True,synchronizer=zarr.ThreadSynchronizer())
# za2=group.empty('ohe',shape=za.shape,dtype=za.dtype,chunks=za.chunks)
# za2[...]=za[...]
print(za[:10, :10])
return za
def mergeTstats(randout, tstatout, contrast):
import numpy as np
from glob import glob
import re, zarr
zarrf = '%s/TestTval_%s.zarr' % (tstatout, contrast)
print('Merging Test-T Values from contrast: %s' % contrast)
# randomise results: randout/rowSlice/chunk/chunk*
# TODO: This line seems RAM heavy:
testTvals = glob('%s/*/*/*[0-9]_tstat%s.nii.gz' % (randout, contrast))
# key: tstat filename
# value: list of chunk coordinates (SourceStart, SourceEnd, TargetStart, TargetEnd)
filedict = {}
for ifile in testTvals:
# RowStart-RowEnd_ColStart-ColEnd (inclusive)
research = re.search('.*/(.*?)-(.*?)_(.*?)-(.*?)_done/', ifile)
chunk_coords = [
research.group(1),
research.group(2),
research.group(3),
research.group(4)
]
chunk_coords = [int(k) for k in chunk_coords]
filedict[ifile] = chunk_coords
# Find max Source/Target based on created niftis:
tmp = np.array(list(filedict.values()))
tmp = np.max(tmp, axis=0)
shape = [
tmp[i] + 1 for i in [1, 3]
] # shape of final array needs to include maximums for source and target
print('Contrast %s Final Shape: %s' % (contrast, shape))
# create zarr file to output final matrix (2D)
testTvals = zarr.zeros(store=zarrf, shape=shape, chunks=(128, 128))
# loop through dictionary of input files to assign final values to testT:
# i: tstat filename
# j: chunk coords
for i, j in filedict.items():
tmpnii = getNII(i)
# Assaign nifti Test T-values to testTvals:
# Add one because chunk coordinates from filename are inclusive:
testTvals.oindex[j[0]:j[1] + 1, j[2]:j[3] + 1] = tmpnii
print('### Test T-values Merged: %s' % zarrf)
def delayed_assign(chunk, ncol, j):
zrow = zarr.zeros((1, ncol), chunks=(1, 1000), dtype='u1')
t = chunk.shape[0]
t0 = time.time()
i = 0
for row in chunk:
zrow[0, row['col']] = 1
i += 1
if i % 100000 == 0 and j == 0:
t1 = time.time()
diff = int(t1 - t0)
minutes, seconds = diff // 60, diff % 60
print("Job {} is {} complete.\tElapsed time {}:{}".format(
j, round(i / t * 100, 1), str(minutes),
str(seconds).zfill(2)))
return (zrow)
def __init__(self):
viewer = self.viewer = neuroglancer.Viewer()
viewer.actions.add('inference', self._do_inference)
self.gt_vol = cloudvolume.CloudVolume(
'https://storage.googleapis.com/neuroglancer-public-data/flyem_fib-25/ground_truth',
mip=0,
bounded=True,
progress=False,
provenance={})
self.dimensions = neuroglancer.CoordinateSpace(
names=['x', 'y', 'z'],
units='nm',
scales=self.gt_vol.resolution,
)
self.inf_results = zarr.zeros(self.gt_vol.bounds.to_list()[3:],
chunks=(64, 64, 64),
dtype=np.uint8)
self.inf_volume = neuroglancer.LocalVolume(data=self.inf_results,
dimensions=self.dimensions)
with viewer.config_state.txn() as s:
s.input_event_bindings.data_view['shift+mousedown0'] = 'inference'
with viewer.txn() as s:
s.layers['image'] = neuroglancer.ImageLayer(
source=
'precomputed://gs://neuroglancer-public-data/flyem_fib-25/image',
)
s.layers['ground_truth'] = neuroglancer.SegmentationLayer(
source=
'precomputed://gs://neuroglancer-public-data/flyem_fib-25/ground_truth',
)
s.layers['ground_truth'].visible = False
s.layers['inference'] = neuroglancer.ImageLayer(
source=self.inf_volume,
shader='''
void main() {
float v = toNormalized(getDataValue(0));
vec4 rgba = vec4(0,0,0,0);
if (v != 0.0) {
rgba = vec4(colormapJet(v), 1.0);
}
emitRGBA(rgba);
}
''',
)
def new_zarr(path, shape, chunks, dtype, in_memory=False, **kwargs):
"""
Create new Zarr NestedDirectoryStore at `path`.
**NOTE:** Persistent Zarr arrays are stored on disk. To avoid data loss, be careful when calling `new_zarr`
on a path with an existing array.
Parameters
----------
path : str
Path to new zarr array
shape : tuple
Overall shape of the zarr array
chunks : tuple
Shape of each chunk for the zarr array
dtype : str
Data type of for the zarr array
kwargs : dict
Keyword args to passs to zarr.open()
Returns
-------
arr : zarr Array
Reference to open zarr array
"""
compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.BITSHUFFLE)
if in_memory:
z_arr_out = zarr.zeros(shape=shape,
chunks=chunks,
dtype=dtype,
compressor=compressor,
**kwargs)
else:
store = zarr.NestedDirectoryStore(path)
z_arr_out = zarr.open(store,
mode='w',
shape=shape,
chunks=chunks,
dtype=dtype,
compressor=compressor,
**kwargs)
return z_arr_out
def test_fancy_indexing_doesnt_mix_with_implicit_slicing():
z2 = zarr.zeros((5, 5, 5))
with pytest.raises(IndexError):
z2[[1, 2, 3], [1, 2, 3]] = 2
with pytest.raises(IndexError):
np.testing.assert_array_equal(
z2[[1, 2, 3], [1, 2, 3]], 0
)
with pytest.raises(IndexError):
z2[[1, 2, 3]] = 2
with pytest.raises(IndexError):
np.testing.assert_array_equal(
z2[[1, 2, 3]], 0
)
with pytest.raises(IndexError):
z2[..., [1, 2, 3]] = 2
with pytest.raises(IndexError):
np.testing.assert_array_equal(
z2[..., [1, 2, 3]], 0
)
def __init__(
self,
url: str,
shape: typing.Tuple[int, ...] = None,
dtype="float32",
creds=None,
memcache: float = None,
):
if shape is not None:
self._zarr = zarr.zeros(
shape,
dtype=dtype,
chunks=self._determine_chunksizes(shape, dtype),
store=hub.areal.store.get_storage_map(url, creds, memcache),
overwrite=True,
)
else:
self._zarr = zarr.open_array(
hub.areal.store.get_storage_map(url, creds, memcache))
self._shape = self._zarr.shape
self._chunks = self._zarr.chunks
self._dtype = self._zarr.dtype
self._memcache = memcache
def __init__(self):
viewer = self.viewer = neuroglancer.Viewer()
viewer.actions.add('inference', self._do_inference)
self.gt_vol = cloudvolume.CloudVolume(
'https://storage.googleapis.com/neuroglancer-public-data/flyem_fib-25/ground_truth',
mip=0,
bounded=True,
progress=False,
provenance={})
self.inf_results = zarr.zeros(
self.gt_vol.bounds.to_list()[3:][::-1], chunks=(64, 64, 64), dtype=np.uint8)
self.inf_volume = neuroglancer.LocalVolume(
data=self.inf_results, voxel_size=list(self.gt_vol.resolution))
with viewer.config_state.txn() as s:
s.input_event_bindings.data_view['shift+mousedown0'] = 'inference'
with viewer.txn() as s:
s.layers['image'] = neuroglancer.ImageLayer(
source='precomputed://gs://neuroglancer-public-data/flyem_fib-25/image',
)
s.layers['ground_truth'] = neuroglancer.SegmentationLayer(
source='precomputed://gs://neuroglancer-public-data/flyem_fib-25/ground_truth',
)
s.layers['ground_truth'].visible = False
s.layers['inference'] = neuroglancer.ImageLayer(
source=self.inf_volume,
shader='''
void main() {
float v = toNormalized(getDataValue(0));
vec4 rgba = vec4(0,0,0,0);
if (v != 0.0) {
rgba = vec4(colormapJet(v), 1.0);
}
emitRGBA(rgba);
}
''',
)
def test_zeros():
z = zeros(100, 10)
eq((100,), z.shape)
eq((10,), z.chunks)
assert_array_equal(np.zeros(100), z[:])
def _do_flood_fill(self, initial_pos, inf_results, inf_volume, event):
initial_pos = (int(initial_pos[0]), int(initial_pos[1]), int(initial_pos[2]))
gt_vol_zarr = zarr.zeros(
self.gt_vol.bounds.to_list()[3:][::-1], chunks=(64, 64, 64), dtype=np.uint64)
gt_blocks_seen = set()
block_size = np.array((64, 64, 64), np.int64)
def fetch_gt_block(block):
spos = block * block_size
epos = spos + block_size
slice_expr = np.s_[int(spos[0]):int(epos[0]),
int(spos[1]):int(epos[1]),
int(spos[2]):int(epos[2])]
rev_slice_expr = np.s_[int(spos[2]):int(epos[2]),
int(spos[1]):int(epos[1]),
int(spos[0]):int(epos[0])]
gt_data = np.transpose(self.gt_vol[slice_expr][..., 0], (2, 1, 0))
gt_vol_zarr[rev_slice_expr] = gt_data
def get_patch(spos, epos):
spos = np.array(spos)
epos = np.array(epos)
sblock = spos // block_size
eblock = (epos - 1) // block_size
for blockoff in np.ndindex(tuple(eblock - sblock + 1)):
block = np.array(blockoff) + sblock
block_tuple = tuple(block)
if block_tuple in gt_blocks_seen: continue
gt_blocks_seen.add(block_tuple)
fetch_gt_block(block)
rev_slice_expr = np.s_[int(spos[2]):int(epos[2]),
int(spos[1]):int(epos[1]),
int(spos[0]):int(epos[0])]
result = gt_vol_zarr[rev_slice_expr]
return result
segment_id = self.gt_vol[initial_pos][0]
patch_size = np.array((33, ) * 3, np.int64)
lower_bound = patch_size // 2
upper_bound = np.array(self.gt_vol.bounds.to_list()[3:]) - patch_size + patch_size // 2
d = 8
seen = set()
q = []
last_invalidate = [time.time()]
invalidate_interval = 3
def enqueue(pos):
if np.any(pos < lower_bound) or np.any(pos >= upper_bound): return
if pos in seen: return
seen.add(pos)
q.append(pos)
def update_view():
if event.is_set():
return
cur_time = time.time()
if cur_time < last_invalidate[0] + invalidate_interval:
return
last_invalidate[0] = cur_time
inf_volume.invalidate()
with self.viewer.txn() as s:
s.layers['points'].annotations = [
neuroglancer.PointAnnotation(id=repr(pos), point=pos) for pos in list(seen)
]
def process_pos(pos):
spos = pos - patch_size // 2
epos = spos + patch_size
rev_slice_expr = np.s_[int(spos[2]):int(epos[2]),
int(spos[1]):int(epos[1]),
int(spos[0]):int(epos[0])]
gt_data = get_patch(spos, epos)
mask = gt_data == segment_id
for offset in ((0, 0, d), (0, 0, -d), (0, d, 0), (0, -d, 0), (d, 0, 0), (-d, 0, 0)):
if not mask[tuple(patch_size // 2 + offset)[::-1]]: continue
new_pos = np.array(pos) + np.array(offset)
enqueue(tuple(new_pos))
dist_transform = scipy.ndimage.morphology.distance_transform_edt(~mask)
inf_results[rev_slice_expr] = 1 + np.cast[np.uint8](
np.minimum(dist_transform, 5) / 5.0 * 254)
self.viewer.defer_callback(update_view)
enqueue(initial_pos)
while len(q) > 0 and not event.is_set():
i = random.randint(0, len(q) - 1)
pos = q[i]
q[i] = q[-1]
del q[-1]
process_pos(pos)
self.viewer.defer_callback(update_view)
