def test_2d_renumber():
for dtype in DTYPES:
data = np.array([
[5, 5, 5, 2],
[3, 5, 5, 0],
[1, 2, 4, 1],
[20, 19, 20, 1],
],
dtype=dtype)
data2 = np.copy(data, order='C')
data2, remapdict = fastremap.renumber(data2, preserve_zero=True)
assert np.all(data2 == [
[1, 1, 1, 2],
[3, 1, 1, 0],
[4, 2, 5, 4],
[6, 7, 6, 4],
])
data2 = np.copy(data, order='F')
data2, remapdict = fastremap.renumber(data2, preserve_zero=True)
assert np.all(data2 == [
[1, 1, 1, 5],
[2, 1, 1, 0],
[3, 5, 7, 3],
[4, 6, 4, 3],
])
def test_1d_renumber():
for dtype in DTYPES:
print(dtype)
data = np.arange(8).astype(dtype)
data = np.flip(data)
data2 = np.copy(data)
data2, remapdict = fastremap.renumber(data2, preserve_zero=False)
assert np.all(data2 == np.arange(1, 9))
assert len(remapdict) > 0
data2 = np.copy(data)
data2, remapdict = fastremap.renumber(data2, preserve_zero=True)
assert data2[-1] == 0
assert np.all(data2 == [1, 2, 3, 4, 5, 6, 7, 0])
assert len(remapdict) > 0
data = np.arange(8).astype(np.bool)
data = np.flip(data)
data2 = np.copy(data)
data2, remapdict = fastremap.renumber(data2, preserve_zero=False)
assert np.all(data2 == [1, 1, 1, 1, 1, 1, 1, 2])
assert len(remapdict) > 0
data2 = np.copy(data)
data2, remapdict = fastremap.renumber(data2, preserve_zero=True)
assert np.all(data2 == [1, 1, 1, 1, 1, 1, 1, 0])
assert len(remapdict) > 0
def test_empty_renumber():
for dtype in DTYPES:
data = np.array([], dtype=dtype)
data2, remapdict = fastremap.renumber(data, preserve_zero=False)
assert np.all(data2 == [])
assert remapdict == {}
def postprocessing(raster, S):
"""Post processing function to enforce connectivity.
:param raster: Labelled image.
:type raster: numpy.ndarray
:param S: Spacing between superpixels.
:type S: int
:returns final: Labelled image with connectivity enforced.
"""
import cc3d
import fastremap
from rasterio import features
for i in range(10):
raster, remapping = fastremap.renumber(raster, in_place=True)
# Remove spourious regions generated during segmentation
cc = cc3d.connected_components(raster.astype(dtype=numpy.uint16),
connectivity=6)
T = int((S**2) / 2)
# Use Connectivity as 4 to avoid undesired connections
raster = features.sieve(cc.astype(dtype=rasterio.int32),
T,
out=numpy.zeros(cc.shape,
dtype=rasterio.int32),
connectivity=4)
return raster
def build_batch(data: torch_geometric.data.Batch,
id2graphlet: Dict[int, Subgraph],
common_file=None) -> "Batch":
# Check if graphlet_id == 0 exists in x because of how remap works
graphlet_id_zero = (data.x == 0).any().item()
# remap graphlet_ids to (0..len(data.x.unique()))
remapped_graphlet_ids, mapping = renumber(
data.x.numpy(),
start=0 + int(graphlet_id_zero),
in_place=False,
preserve_zero=graphlet_id_zero)
batch_graphlet_indices = torch.tensor(remapped_graphlet_ids.flatten(),
dtype=torch.int64)
graphlet_ids_sorted_by_new_id = (
list(
map(
lambda e: e[0], # get key of
# (key, value) sorted by value
sorted(mapping.items(), key=lambda e: e[1]))))
xs = []
edge_indices = []
# create list of xs and edge_indices where
# xs[i] are the features of the graphlet that was mapped to
# new_id == i etc.
for i, graphlet_id in enumerate(graphlet_ids_sorted_by_new_id):
graphlet = id2graphlet[graphlet_id]
xs.append(graphlet.x)
edge_indices.append(graphlet.edge_index + i * graphlet.x.size(0))
if common_file is not None:
common = np.loadtxt(str(common_file), dtype=np.int64)
common = torch.tensor(
list(map(lambda x: mapping.get(x, -100), common)))
common = (batch_graphlet_indices == common.reshape(-1, 1)).any(0)
data.estimates[~common] = 0
# Sparse matrix where each row represents a graph
# and each column a graphlet where
# m[graph][graphlet] == count of graphlet in graph
graph_has_graphlet = SparseTensor(row=data.batch,
col=batch_graphlet_indices,
value=data.estimates)
if graph_has_graphlet.density(
) > .75: # FIXME: update parameter if necessary
graph_has_graphlet = graph_has_graphlet.to_dense()
return Batch(x=torch.cat(xs, dim=0),
edge_index=torch.cat(edge_indices, dim=1),
graph_has_graphlet=graph_has_graphlet,
graphlet_ids=graphlet_ids_sorted_by_new_id,
y=data.y)
def test_3d_renumber():
for dtype in DTYPES:
bits = np.dtype(dtype).itemsize * 8
big = (2**(bits - 1)) - 1 # cover ints and uints
data = np.array([
[
[big, 0],
[2, big],
],
[
[big - 5, big - 1],
[big - 7, big - 3],
],
],
dtype=dtype)
data2 = np.copy(data, order='C')
data2, remapdict = fastremap.renumber(data2, preserve_zero=False)
assert np.all(data2 == [
[[1, 2], [3, 1]],
[
[4, 5],
[6, 7],
],
])
data2 = np.copy(data, order='F')
data2, remapdict = fastremap.renumber(data2, preserve_zero=False)
assert np.all(data2 == [
[[1, 5], [3, 1]],
[
[2, 6],
[4, 7],
],
])
big = np.random.randint(0, (2**64) - 1,
size=(512, 512, 100),
dtype=np.uint64)
big, remapdict = fastremap.renumber(big, preserve_zero=True, in_place=True)
assert np.dtype(big.dtype).itemsize <= 4
assert np.dtype(big.dtype).itemsize > 1
def execute(self):
self._volume = CloudVolume(
self.layer_path, self.options['mip'], bounded=False,
parallel=self.options['parallel_download'],
fill_missing=self.options['fill_missing']
)
self._bounds = Bbox(self.offset, self.shape + self.offset)
self._bounds = Bbox.clamp(self._bounds, self._volume.bounds)
self.progress = bool(self.options['progress'])
self._mesher = zmesh.Mesher(self._volume.resolution)
# Marching cubes loves its 1vx overlaps.
# This avoids lines appearing between
# adjacent chunks.
data_bounds = self._bounds.clone()
data_bounds.minpt -= self.options['low_padding']
data_bounds.maxpt += self.options['high_padding']
self._mesh_dir = self.get_mesh_dir()
if self.options['encoding'] == 'draco':
self.draco_encoding_settings = draco_encoding_settings(
shape=(self.shape + self.options['low_padding'] + self.options['high_padding']),
offset=self.offset,
resolution=self._volume.resolution,
compression_level=self.options["draco_compression_level"],
create_metadata=self.options['draco_create_metadata'],
uses_new_draco_bin_size=False,
)
# chunk_position includes the overlap specified by low_padding/high_padding
# agglomerate, timestamp, stop_layer only applies to graphene volumes,
# no-op for precomputed
data = self._volume.download(
data_bounds,
agglomerate=self.options['agglomerate'],
timestamp=self.options['timestamp'],
stop_layer=self.options['stop_layer']
)
if not np.any(data):
if self.options['spatial_index']:
self._upload_spatial_index(self._bounds, {})
return
data = self._remove_dust(data, self.options['dust_threshold'])
data = self._remap(data)
if self.options['object_ids']:
data = fastremap.mask_except(data, self.options['object_ids'], in_place=True)
data, renumbermap = fastremap.renumber(data, in_place=True)
renumbermap = { v:k for k,v in renumbermap.items() }
self.compute_meshes(data, renumbermap)
def compute_cc_labels(all_labels):
tmp_labels = all_labels
if np.dtype(all_labels.dtype).itemsize > 1:
tmp_labels, remapping = fastremap.renumber(all_labels, in_place=False)
cc_labels = cc3d.connected_components(tmp_labels)
cc_labels = fastremap.refit(cc_labels)
del tmp_labels
remapping = kimimaro.skeletontricks.get_mapping(all_labels, cc_labels)
return cc_labels, remapping
def _masks_to_gui(parent, masks, outlines=None):
""" masks loaded into GUI """
# get unique values
shape = masks.shape
fastremap.renumber(masks, in_place=True)
masks = np.reshape(masks, shape)
masks = masks.astype(
np.uint16) if masks.max() < 2**16 - 1 else masks.astype(np.uint32)
parent.cellpix = masks
# get outlines
if outlines is None: # parent.outlinesOn
parent.outpix = np.zeros_like(masks)
for z in range(parent.NZ):
outlines = utils.masks_to_outlines(masks[z])
parent.outpix[z] = outlines * masks[z]
if z % 50 == 0 and parent.NZ > 1:
print('GUI_INFO: plane %d outlines processed' % z)
else:
parent.outpix = outlines
shape = parent.outpix.shape
_, parent.outpix = np.unique(parent.outpix, return_inverse=True)
parent.outpix = np.reshape(parent.outpix, shape)
parent.ncells = parent.cellpix.max()
colors = parent.colormap[:parent.ncells, :3]
parent.cellcolors = list(
np.concatenate((np.array([[255, 255, 255]]), colors),
axis=0).astype(np.uint8))
parent.draw_masks()
parent.redraw_masks(masks=parent.masksOn, outlines=parent.outlinesOn
) # add to obey outline/mask setting upon recomputing
if parent.ncells > 0:
parent.toggle_mask_ops()
parent.ismanual = np.zeros(parent.ncells, bool)
parent.zdraw = list(-1 * np.ones(parent.ncells, np.int16))
parent.update_plot()
def engage_avocado_protection(cc_labels, all_dbf, remapping,
soma_detection_threshold, edtfn, progress):
orig_cc_labels = np.copy(cc_labels, order='F')
unchanged = set()
max_iterations = max(fastremap.unique(cc_labels))
# This loop handles nested avocados
# Unless there are deeply nested double avocados,
# this should complete in 2-3 passes. We limit it
# to 20 just to make sure this loop terminates no matter what.
# Avocados aren't the end of the world.
for _ in tqdm(range(20), disable=(not progress), desc="Avocado Pass"):
# Note: Divide soma_detection_threshold by a bit more than 2 because the nucleii are going to be
# about a factor of 2 or less smaller than what we'd expect from a cell. For example,
# in an avocado I saw, the DBF of the nucleus was 499 when the detection threshold was
# set to 1100.
candidates = set(
fastremap.unique(cc_labels *
(all_dbf > soma_detection_threshold / 2.5)))
candidates -= unchanged
candidates.discard(0)
cc_labels, unchanged_this_cycle, changes = engage_avocado_protection_single_pass(
cc_labels,
all_dbf,
candidates=candidates,
progress=progress,
)
unchanged |= unchanged_this_cycle
if len(changes) == 0:
break
all_dbf = edtfn(cc_labels)
# Downstream logic assumes cc_labels is contigiously numbered
cc_labels, _ = fastremap.renumber(cc_labels, in_place=True)
cc_remapping = kimimaro.skeletontricks.get_mapping(orig_cc_labels,
cc_labels)
adjusted_remapping = {}
for new_cc, cc in cc_remapping.items():
if cc in remapping:
adjusted_remapping[new_cc] = remapping[cc]
return cc_labels, all_dbf, adjusted_remapping
def compute_cc_labels(all_labels, cc_safety_factor):
if cc_safety_factor <= 0 or cc_safety_factor > 1:
raise ValueError(
"cc_safety_factor must be greater than zero and less than or equal to one. Got: "
+ str(cc_safety_factor))
tmp_labels = all_labels
if np.dtype(all_labels.dtype).itemsize > 1:
tmp_labels, remapping = fastremap.renumber(all_labels, in_place=True)
cc_labels = cc3d.connected_components(tmp_labels,
max_labels=int(tmp_labels.size *
cc_safety_factor))
del tmp_labels
remapping = kimimaro.skeletontricks.get_mapping(all_labels, cc_labels)
return cc_labels, remapping
def labels_to_flows(labels, files=None, use_gpu=False, device=None, redo_flows=False):
""" convert labels (list of masks or flows) to flows for training model
if files is not None, flows are saved to files to be reused
Parameters
--------------
labels: list of ND-arrays
labels[k] can be 2D or 3D, if [3 x Ly x Lx] then it is assumed that flows were precomputed.
Otherwise labels[k][0] or labels[k] (if 2D) is used to create flows and cell probabilities.
Returns
--------------
flows: list of [4 x Ly x Lx] arrays
flows[k][0] is labels[k], flows[k][1] is cell distance transform, flows[k][2] is Y flow,
flows[k][3] is X flow, and flows[k][4] is heat distribution
"""
nimg = len(labels)
if labels[0].ndim < 3:
labels = [labels[n][np.newaxis,:,:] for n in range(nimg)]
if labels[0].shape[0] == 1 or labels[0].ndim < 3 or redo_flows: # flows need to be recomputed
dynamics_logger.info('computing flows for labels')
# compute flows; labels are fixed here to be unique, so they need to be passed back
# make sure labels are unique!
labels = [fastremap.renumber(label, in_place=True)[0] for label in labels]
veci = [masks_to_flows(labels[n][0],use_gpu=use_gpu, device=device) for n in trange(nimg)]
# concatenate labels, distance transform, vector flows, heat (boundary and mask are computed in augmentations)
flows = [np.concatenate((labels[n], labels[n]>0.5, veci[n]), axis=0).astype(np.float32)
for n in range(nimg)]
if files is not None:
for flow, file in zip(flows, files):
file_name = os.path.splitext(file)[0]
tifffile.imsave(file_name+'_flows.tif', flow)
else:
dynamics_logger.info('flows precomputed')
flows = [labels[n].astype(np.float32) for n in range(nimg)]
return flows
def cc3d_test(labels):
labels, remap = fastremap.renumber(labels)
res = cc3d.connected_components(labels)
N = np.max(labels)
for segid in tqdm(range(1, N + 1)):
extracted = (res == segid)
def remap(self, start_id: int):
fastremap.renumber(self.array, preserve_zero=True, in_place=True)
seg = self.astype(np.uint64)
seg.array[seg.array > 0] += start_id
start_id = seg.max()
return seg, start_id
def get_masks(p, iscell=None, rpad=20, flows=None, threshold=0.4, use_gpu=False, device=None):
""" create masks using pixel convergence after running dynamics
Makes a histogram of final pixel locations p, initializes masks
at peaks of histogram and extends the masks from the peaks so that
they include all pixels with more than 2 final pixels p. Discards
masks with flow errors greater than the threshold.
Parameters
----------------
p: float32, 3D or 4D array
final locations of each pixel after dynamics,
size [axis x Ly x Lx] or [axis x Lz x Ly x Lx].
iscell: bool, 2D or 3D array
if iscell is not None, set pixels that are
iscell False to stay in their original location.
rpad: int (optional, default 20)
histogram edge padding
threshold: float (optional, default 0.4)
masks with flow error greater than threshold are discarded
(if flows is not None)
flows: float, 3D or 4D array (optional, default None)
flows [axis x Ly x Lx] or [axis x Lz x Ly x Lx]. If flows
is not None, then masks with inconsistent flows are removed using
`remove_bad_flow_masks`.
Returns
---------------
M0: int, 2D or 3D array
masks with inconsistent flow masks removed,
0=NO masks; 1,2,...=mask labels,
size [Ly x Lx] or [Lz x Ly x Lx]
"""
pflows = []
edges = []
shape0 = p.shape[1:]
dims = len(p)
if iscell is not None:
if dims==3:
inds = np.meshgrid(np.arange(shape0[0]), np.arange(shape0[1]),
np.arange(shape0[2]), indexing='ij')
elif dims==2:
inds = np.meshgrid(np.arange(shape0[0]), np.arange(shape0[1]),
indexing='ij')
for i in range(dims):
p[i, ~iscell] = inds[i][~iscell]
for i in range(dims):
pflows.append(p[i].flatten().astype('int32'))
edges.append(np.arange(-.5-rpad, shape0[i]+.5+rpad, 1))
h,_ = np.histogramdd(tuple(pflows), bins=edges)
hmax = h.copy()
for i in range(dims):
hmax = maximum_filter1d(hmax, 5, axis=i)
seeds = np.nonzero(np.logical_and(h-hmax>-1e-6, h>10))
Nmax = h[seeds]
isort = np.argsort(Nmax)[::-1]
for s in seeds:
s = s[isort]
pix = list(np.array(seeds).T)
shape = h.shape
if dims==3:
expand = np.nonzero(np.ones((3,3,3)))
else:
expand = np.nonzero(np.ones((3,3)))
for e in expand:
e = np.expand_dims(e,1)
for iter in range(5):
for k in range(len(pix)):
if iter==0:
pix[k] = list(pix[k])
newpix = []
iin = []
for i,e in enumerate(expand):
epix = e[:,np.newaxis] + np.expand_dims(pix[k][i], 0) - 1
epix = epix.flatten()
iin.append(np.logical_and(epix>=0, epix<shape[i]))
newpix.append(epix)
iin = np.all(tuple(iin), axis=0)
for p in newpix:
p = p[iin]
newpix = tuple(newpix)
igood = h[newpix]>2
for i in range(dims):
pix[k][i] = newpix[i][igood]
if iter==4:
pix[k] = tuple(pix[k])
M = np.zeros(h.shape, np.uint32)
for k in range(len(pix)):
M[pix[k]] = 1+k
for i in range(dims):
pflows[i] = pflows[i] + rpad
M0 = M[tuple(pflows)]
# remove big masks
uniq, counts = fastremap.unique(M0, return_counts=True)
big = np.prod(shape0) * 0.4
bigc = uniq[counts > big]
if len(bigc) > 0 and (len(bigc)>1 or bigc[0]!=0):
M0 = fastremap.mask(M0, bigc)
fastremap.renumber(M0, in_place=True) #convenient to guarantee non-skipped labels
M0 = np.reshape(M0, shape0)
return M0
