def map_to_volume(fw, h5path_in, h5path_out, h5path_mask=''):
"""Map the prediction to a labelvolume."""
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
a = ds_in[:]
h5file_in.close()
# myelinated axon labels
h5file_out, ds_out = utils.h5_write(None, a.shape, a.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
mask = fw[a]
a[~mask] = 0
ds_out[:] = a
h5file_out.close()
# myelinated axon mask
if not h5path_mask:
return
h5file_out, ds_out = utils.h5_write(None, a.shape, 'bool',
h5path_mask,
element_size_um=elsize,
axislabels=axlab)
ds_out[:] = a.astype('bool')
h5file_out.close()
def combine_vols(
h5path_in,
vol_idxs=[0, 2, 4, 7],
h5path_out='',
protective=False,
):
"""Combine volumes by addition."""
if bool(h5path_out) and ('.h5' in h5path_out):
status, info = utils.h5_check(h5path_out, protective)
print(info)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, es, al = utils.load(h5path_in)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape[:3], ds_in.dtype,
h5path_out,
element_size_um=es[:3],
axislabels=al[:3])
out = np.zeros(ds_out.shape, dtype=ds_out.dtype)
for volnr in vol_idxs:
out += ds_in[:, :, :, volnr]
ds_out[:] = out
# close and return
h5file_in.close()
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def CC_2Dto3D(
h5path_in,
h5path_out='',
protective=False,
):
"""Label connected components in 3D from the 2D-generated mask."""
# check output path
if h5path_out:
status, info = utils.h5_check(h5path_out, protective)
print(info)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, 'uint32',
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# NOTE: scipy has much lower memory consumption than scikit-image
# TODO: input mask directly, not labelimage?
ds_out[:] = label(ds_in[:, :, :] != 0)
# ds_out[:] = scipy_label(ds_in[:, :, :] != 0)[0]
# close and return
try:
h5file_in.close()
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def write_block(ds_in, elsize, axlab, block):
"""Write the block to file."""
shape = list(block['size'])
if ds_in.ndim == 4:
shape += [ds_in.shape[3]]
chunks = ds_in.chunks
if any(np.array(chunks) > np.array(shape)):
chunks = True
h5file_out, ds_out = utils.h5_write(
data=None,
shape=shape,
dtype=ds_in.dtype,
h5path_full=block['h5path'],
chunks=chunks,
element_size_um=elsize,
axislabels=axlab,
)
slcs = block['slc']
if ds_in.ndim == 3:
ds_out[:] = ds_in[slcs[0], slcs[1], slcs[2]]
elif ds_in.ndim == 4:
ds_out[:] = ds_in[slcs[0], slcs[1], slcs[2], :]
h5file_out.close()
def downsample_blockwise(
h5path_in,
blockreduce=[3, 3, 3],
func='np.amax',
dataslices=None,
h5path_out='',
save_steps=False,
protective=False,
):
"""Downsample volume by blockwise reduction."""
# Check if any output paths already exist.
outpaths = {'out': h5path_out}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# Open the inputfile for reading.
# TODO: option to get the input data passed
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
# Get the matrix size and resolution of the outputdata.
outsize, elsize = get_new_sizes(func, blockreduce, ds_in.shape, elsize)
# Open the outputfile for writing and create the dataset or output array.
h5file_out, ds_out = utils.h5_write(None,
outsize,
ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# Get the slice objects for the input data.
slices = utils.get_slice_objects_prc(dataslices, ds_in.shape)
# Reformat the data to the outputsize.
if func == 'expand':
out = ds_in[slices[0], ...]
for axis in range(0, ds_out.ndim):
out = np.repeat(out, blockreduce[axis], axis=axis)
ds_out[slices[0], ...] = out
else:
""" TODO: flexible mapping from in to out
now:
the reduction factor of the first axis must be 1;
the extent of the remaining axes must be full
"""
ds_out[slices[0], ...] = block_reduce(ds_in[slices[0], ...],
block_size=tuple(blockreduce),
func=eval(func))
# Close the h5 files or return the output array.
try:
h5file_in.close()
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def map_labels(
h5path_in,
h5path_out='',
save_steps=False,
protective=False,
):
"""Map groups of labels to a single label."""
# check output path
outpaths = {'out': h5path_out, 'stitched': ''}
root, ds_main = outpaths['out'].split('.h5')
for dsname, outpath in outpaths.items():
grpname = ds_main + "_steps"
outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname)
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# load the pickled set of neighbours
lsroot = h5path_out.split('.h5')[0]
lspath = '{}_{}.pickle'.format(lsroot, ds_out.name[1:])
with open(lspath, "r") as f:
labelsets = pickle.load(f)
# apply forward map
ulabels = np.unique(ds_in[:])
maxlabel = np.amax(ulabels)
fw = [l if l in ulabels else 0 for l in range(0, maxlabel + 1)]
labels = utils.forward_map(np.array(fw), ds_in[:], labelsets)
if save_steps:
fw = np.zeros(maxlabel + 1, dtype='i')
MAlabels = utils.forward_map(np.array(fw), ds_in[:], labelsets)
utils.save_step(outpaths, 'stitched', MAlabels, elsize, axlab)
ds_out[:, :, :] = labels
# close and return
try:
h5file_in.close()
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def CC_2Dprops(
h5path_labels,
basename,
map_propnames,
usempi=False,
h5path_out='',
protective=False,
):
"""Map the labels/properties."""
# check output paths
if '.h5' in h5path_out:
for propname in map_propnames:
h5path_prop = os.path.join(h5path_out, propname)
status, info = utils.h5_check(h5path_out, protective)
print(info)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_labels)
# prepare mpi
n_props = len(map_propnames)
series = np.array(range(0, n_props), dtype=int)
mpi_info = utils.get_mpi_info(usempi)
if mpi_info['enabled']:
series = utils.scatter_series(mpi_info, series)[0]
fws = {}
for i in series:
propname = map_propnames[i]
print("processing prop %s" % propname)
nppath = '{}_{}.npy'.format(basename, propname)
fws[propname] = np.load(nppath)
# open data for writing
h5path_prop = os.path.join(h5path_out, propname)
h5file_prop, ds_prop = utils.h5_write(None, ds_in.shape,
fws[propname].dtype,
h5path_prop,
element_size_um=elsize,
axislabels=axlab,
comm=mpi_info['comm'])
ds_prop[:] = fws[propname][ds_in[:]]
h5file_prop.close()
# close and return
h5file_in.close()
def filter_NoR(
h5path_in,
h5path_2D,
h5path_out='',
save_steps=False,
protective=False,
):
"""Filter nodes of ranvier."""
# check output paths
outpaths = {'out': h5path_out}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
h5file_2D, ds_2D, _, _ = utils.h5_load(h5path_2D)
# open data for writing
h5file_out, ds_out = utils.h5_write(ds_in, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
labelsets = {i: set(np.unique(ds_2D[i, :, :]))
for i in range(ds_2D.shape[0])}
ulabels = np.unique(ds_in)
m = {l: np.array([True if l in lsv else False
for _, lsv in labelsets.items()])
for l in ulabels}
rp = regionprops(ds_in)
for prop in rp:
z, y, x, Z, Y, X = tuple(prop.bbox)
mask = prop.image
mask[m[prop.label][z:Z], :, :] = 0
ds_out[z:Z, y:Y, x:X][mask] = 0
# close and return
h5file_in.close()
h5file_2D.close()
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def nodes_of_ranvier(
h5path_in,
min_labelsize=0,
remove_small_labels=False,
h5path_boundarymask='',
merge_methods=['neighbours'],
overlap_threshold=20,
h5path_data='',
h5path_mmm='',
searchradius=[100, 30, 30],
h5path_out='',
save_steps=False,
protective=False,
):
"""Find labels that do not traverse through the volume."""
# check output paths
outpaths = {'out': h5path_out,
'largelabels': '', 'smalllabelmask': '',
'boundarymask': '',
'labels_nt': '', 'labels_tv': '',
'filled': '',
}
root, ds_main = outpaths['out'].split('.h5')
for dsname, outpath in outpaths.items():
grpname = ds_main + "_steps"
outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname)
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
labels = ds_in[:] # FIXME: do we make a copy, or use ds_out?
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# start with the set of all labels
ulabels = np.unique(labels)
maxlabel = np.amax(ulabels)
labelset = set(ulabels)
print("number of labels in labelvolume: {}".format(len(labelset)))
# get the labelsets that touch the borders
sidesmask = get_boundarymask(h5path_boundarymask, 'invdil')
ls_bot = set(np.unique(labels[:4, :, :]))
ls_top = set(np.unique(labels[-4:, :, :]))
ls_sides = set(np.unique(labels[sidesmask]))
ls_border = ls_bot | ls_top | ls_sides
ls_centre = labelset - ls_border
# get the labels that do not touch the border twice
ls_bts = (ls_bot ^ ls_top) ^ ls_sides
ls_tbs = (ls_top ^ ls_bot) ^ ls_sides
ls_sbt = (ls_sides ^ ls_bot) ^ ls_top
ls_nt = ls_centre | ls_bts | ls_tbs | ls_sbt
# filter labels on size
root = os.path.splitext(h5file_out.filename)[0]
ls_small = utils.filter_on_size(labels, labelset, min_labelsize,
remove_small_labels,
save_steps, root, ds_out.name[1:],
outpaths, elsize, axlab)[2]
labelset -= ls_small
ls_nt -= ls_small
ls_short = filter_on_heigth(labels, 5)
labelset -= ls_short
ls_nt -= ls_short
ls_tv = labelset - ls_nt
print('number of large, long labels: {}'.format(len(labelset)))
print('number of large, long in-volume labels: {}'.format(len(ls_nt)))
print('number of large, long through-volume labels: {}'.format(len(ls_tv)))
# map the large labels that don't traverse the volume
fw_nt = np.zeros(maxlabel + 1, dtype='i')
for l in ls_nt:
fw_nt[l] = l
labels_nt = fw_nt[labels]
# automated label merge
labelsets = {}
min_labelsize = 10
labelsets, filled = merge_labels(labels_nt, labelsets, merge_methods,
overlap_threshold,
h5path_data, h5path_mmm,
min_labelsize,
searchradius)
fw = np.zeros(maxlabel + 1, dtype='i')
ds_out[:] = utils.forward_map(np.array(fw), labels, labelsets)
if save_steps:
utils.save_step(outpaths, 'boundarymask', sidesmask, elsize, axlab)
utils.save_step(outpaths, 'labels_nt', labels_nt, elsize, axlab)
fw_tv = np.zeros(maxlabel + 1, dtype='i')
for l in ls_tv:
fw_tv[l] = l
labels_tv = fw_tv[labels]
utils.save_step(outpaths, 'labels_tv', labels_tv, elsize, axlab)
if filled is not None:
fw = np.zeros(maxlabel + 1, dtype='i')
filled = utils.forward_map(np.array(fw), filled, labelsets)
utils.save_step(outpaths, 'filled', filled, elsize, axlab)
filestem = '{}_{}_automerged'.format(root, ds_out.name[1:])
utils.write_labelsets(labelsets, filestem,
filetypes=['txt', 'pickle'])
# close and return
h5file_in.close()
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def watershed_ics(
h5path_in,
masks=[],
h5path_seeds='',
seed_size=64,
lower_threshold=None,
upper_threshold=None,
invert=False,
h5path_out='',
save_steps=False,
protective=False,
):
"""Perform watershed on the intracellular space compartments."""
# check output paths
outpaths = {'out': h5path_out, 'seeds': h5path_seeds, 'mask': ''}
root, ds_main = outpaths['out'].split('.h5')
for dsname, _ in outpaths.items():
grpname = ds_main + "_steps"
outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname)
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
# open data for writing
h5file_out, ds_out = utils.h5_write(None,
ds_in.shape[:3],
'uint32',
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# load/generate the seeds
if h5path_seeds:
h5file_sds, ds_sds, _, _ = utils.h5_load(h5path_seeds)
else:
h5file_sds, ds_sds = utils.h5_write(None,
ds_in.shape[:3],
'uint32',
outpaths['seeds'],
element_size_um=elsize,
axislabels=axlab)
lower_threshold = lower_threshold or np.amin(ds_in[:]) - 1
upper_threshold = upper_threshold or np.amax(ds_in[:]) + 1
ds_sds[:] = np.logical_and(ds_in[:] > lower_threshold,
ds_in[:] <= upper_threshold)
ds_sds[:], _ = label(ds_sds[:])
ds_sds[:] = remove_small_objects(ds_sds[:], min_size=seed_size)
ds_sds[:] = relabel_sequential(ds_sds[:])[0]
""" NOTE:
numpy/scipy/skimage inplace is not written when using hdf5
Therefore, we cannot use:
np.logical_and(ds_in[:] > lower_threshold,
ds_in[:] <= upper_threshold, ds_sds[:])
num = label(ds_sds[:], output=ds_sds[:])
remove_small_objects(ds_sds[:], min_size=seed_size, in_place=True)
"""
# determine the mask
mask = np.ones(ds_in.shape[:3], dtype='bool')
mask = utils.string_masks(masks, mask)
h5file_mask, ds_mask = utils.h5_write(None,
ds_in.shape[:3],
'uint8',
outpaths['mask'],
element_size_um=elsize,
axislabels=axlab)
ds_mask[:] = mask
# ds_mask[:].fill(1)
# ds_mask[:] = utils.string_masks(masks, ds_mask[:])
# perform the watershed
if invert:
ds_out[:] = watershed(-ds_in[:], ds_sds[:], mask=ds_mask[:])
else:
ds_out[:] = watershed(ds_in[:], ds_sds[:], mask=ds_mask[:])
# close and return
h5file_in.close()
try:
h5file_out.close()
h5file_sds.close()
h5file_mask.close()
except (ValueError, AttributeError):
return ds_out, ds_sds, ds_mask
def filter_labels(
h5path_in,
h5path_mm='',
min_labelsize=0,
close=None,
relabel_from=0,
h5path_out='',
save_steps=False,
protective=False,
):
"""Map groups of labels to a single label."""
# check output path
outpaths = {'out': h5path_out, 'closed': ''}
root, ds_main = outpaths['out'].split('.h5')
for dsname, outpath in outpaths.items():
grpname = ds_main + "_steps"
outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname)
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
labels = ds_in[:]
# if min_labelsize:
# remove_small_objects(labels, min_size=min_labelsize, in_place=True)
if close is not None:
labels = close_labels(labels, close)
if h5path_mm:
print('removing voxels in mask')
h5file_mm, ds_mm, _, _ = utils.h5_load(h5path_mm)
labels[ds_mm[:].astype('bool')] = 0
h5file_mm.close()
if save_steps:
utils.save_step(outpaths, 'closed', labels, elsize, axlab)
if min_labelsize:
print('removing small labels')
remove_small_objects(labels, min_size=min_labelsize, in_place=True)
# if save_steps:
# utils.save_step(outpaths, 'small', smalllabels, elsize, axlab)
if relabel_from > 1:
print('relabeling from {}'.format(relabel_from))
labels = relabel_sequential(labels, relabel_from)[0]
# TODO: save mapping?
ds_out[:, :, :] = labels
# close and return
try:
h5file_in.close()
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def stack2stack(
inputfile,
outputfile,
dset_name='',
blockoffset=[],
datatype='',
uint8conv=False,
inlayout='',
outlayout='',
elsize=[],
chunksize=[],
additional_outputs=[],
nzfills=5,
dataslices=None,
save_steps=False,
protective=False,
):
"""Convert/select/downscale/transpose/... an hdf5 dataset."""
# output root and exts
root, ext = split_filepath(outputfile)
outexts = list(set(additional_outputs + [ext]))
# Check if any output paths already exist.
outpaths = {'out': outputfile, 'addext': (root, additional_outputs)}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
h5file_in, ds_in, h5elsize, h5axlab = utils.h5_load(inputfile)
try:
ndim = ds_in.ndim
except AttributeError:
ndim = len(ds_in.dims)
# data layout
# FIXME: h5axlab not necessarily xyzct!
# FIXME: this forces a possibly erroneous outlayout when inlayout is not found
inlayout = inlayout or ''.join(h5axlab) or 'zyxct'[0:ndim]
outlayout = outlayout or inlayout
in2out = [inlayout.index(l) for l in outlayout]
# element size
elsize = elsize or (h5elsize[in2out]
if h5elsize is not None else None)
# chunksize
if chunksize is not None:
chunksize = tuple(chunksize) or (
True if not any(chunksize)
else (tuple([ds_in.chunks[i] for i in in2out])
if ds_in.chunks else None
)
)
# datatype
datatype = datatype or ds_in.dtype
if dset_name:
_, x, X, y, Y, z, Z = utils.split_filename(dset_name, blockoffset)
slices = {'x': [x, X, 1], 'y': [y, Y, 1], 'z': [z, Z, 1]}
if ndim > 3:
C = ds_in.shape[inlayout.index('c')]
slices['c'] = [0, C, 1]
sliceslist = [slices[dim] for dim in inlayout]
dataslices = [item for sl in sliceslist for item in sl]
# get the selected and transformed data
# TODO: most memory-efficient solution
data = utils.load_dataset(ds_in, elsize, inlayout, outlayout,
datatype, dataslices, uint8conv)[0]
h5file_in.close()
# write the data
for ext in outexts:
if '.nii' in ext:
if data.dtype == 'float16':
data = data.astype('float')
utils.write_to_nifti(root + '.nii.gz', data, elsize)
if '.h5' in ext:
utils.h5_write(data, data.shape, data.dtype,
outputfile,
element_size_um=elsize,
axislabels=outlayout,
chunks=chunksize)
if (('.tif' in ext) |
('.png' in ext) |
('.jpg' in ext)) & (data.ndim < 4):
if data.ndim == 2:
data = data.atleast_3d()
outlayout += 'z'
utils.write_to_img(root, data, outlayout, nzfills, ext)
return data
m = fwf > 3000 # always exclude
fw[m] = 0
# In[35]:
## map to volume
h5file_in, ds_in, elsize, axlab = utils.h5_load(labelpath)
a = ds_in[:]
h5file_in.close()
# myelinated axon labels
h5path_out = os.path.join(labelfile, '{}_prediction'.format(labeldset))
h5file_out, ds_out = utils.h5_write(None,
a.shape,
a.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
mask = fw[a]
a[~mask] = 0
ds_out[:] = a
h5file_out.close()
# myelinated axon mask
h5path_out = os.path.join(datadir, '{}_masks_maskPRED.h5'.format(dataset),
'maskMA')
h5file_out, ds_out = utils.h5_write(None,
a.shape,
'bool',
h5path_out,
def fill_holes(
h5path_in,
methods,
selem,
h5path_mask='',
h5path_md='',
h5path_mm='',
h5path_mx='',
h5path_out='',
h5path_out_holes='',
h5path_out_ma='',
h5path_out_mm='',
protective=False,
):
"""Fill holes in labels."""
# check output path # TODO check other output volumes (holes, ma, mm)
for outpath in [h5path_out,
h5path_out_holes, h5path_out_ma, h5path_out_mm]:
if '.h5' in outpath:
status, info = utils.h5_check(outpath, protective)
print(info)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
if h5path_mask:
h5file_ml, ds_ml, _, _ = utils.h5_load(h5path_mask)
ds_in[~ds_ml[:].astype('bool')] = 0
h5file_ml.close()
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# fill holes
ds_out[:] = ds_in[:] # TODO: simply make copy h5file_in and open it?
for m in methods:
if m == '1':
ds_out = fill_holes_m1(ds_out, selem)
if m == '2':
ds_out = fill_holes_m2(ds_out, selem)
if m == '3':
ds_out = fill_holes_m3(ds_out, selem)
if m == '4':
ds_out = fill_holes_m4(ds_out, h5path_md, h5path_mm, h5path_mx)
# additional output: holes filled and updated masks
if h5path_out_holes:
h5file_ho, ds_ho, _, _ = utils.h5_write(None,
ds_out.shape, ds_out.dtype,
h5path_out_holes,
element_size_um=elsize,
axislabels=axlab)
ds_ho[:] = ds_out[:]
ds_ho[ds_in > 0] = 0
h5file_ho.close()
if h5path_out_ma:
ds_ma = utils.h5_write(ds_out.astype('bool'), ds_out.shape, 'uint8',
h5path_out_ma,
element_size_um=elsize,
axislabels=axlab)[1]
if h5path_out_mm:
ds_mm = utils.h5_load(h5path_mm, load_data=True)
ds_mm[ds_ho > 0] = 0
ds_mm = utils.h5_write(ds_mm.astype('bool'), ds_mm.shape, 'uint8',
h5path_out_mm,
element_size_um=elsize,
axislabels=axlab)
# close and return
try:
h5file_in.close()
h5file_out.close()
except (ValueError, AttributeError):
return ds_out, ds_ho, ds_ma, ds_mm
def prob2mask(
h5path_in,
dataslices=None,
lower_threshold=0,
upper_threshold=1,
size=0,
dilation=0,
h5path_mask=None,
go2D=False,
h5path_out='',
save_steps=False,
protective=False,
):
"""Create thresholded hard segmentation."""
# Check if any output paths already exist.
outpaths = {'out': h5path_out, 'raw': '', 'mito': '', 'dil': ''}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
root, ds_main = outpaths['out'].split('.h5')
grpname = ds_main + "_steps"
for dsname, outpath in outpaths.items():
if ((dsname != 'out') and save_steps and (not outpath)):
outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname)
# Load the (sliced) data.
h5file_in, ds_in, es, al = utils.load(h5path_in)
data, _, _, slices_out = utils.load_dataset(ds_in, dataslices=dataslices)
inmask = utils.load(h5path_mask,
load_data=True,
dtype='bool',
dataslices=dataslices)[0]
if data.ndim == 4: # FIXME: generalize
# data = np.squeeze(data)
data = data[:, :, :, 0]
outshape = ds_in.shape[:3]
es = es[:3]
al = al[:3]
slices_out = slices_out[:3]
else:
outshape = ds_in.shape
# Open the outputfile(s) for writing
# and create the dataset(s) or output array(s).
h5file_out, ds_out = utils.h5_write(None,
outshape,
'uint8',
outpaths['out'],
element_size_um=es,
axislabels=al)
if save_steps:
stepnames = ['raw', 'mito', 'dil']
steps = [
utils.h5_write(None,
outshape,
'uint8',
outpaths[out],
element_size_um=es,
axislabels=al) for out in stepnames
]
dss = [np.zeros_like(data)] * 3
else:
stepnames, steps, dss = [], [], []
# Threshold (and dilate and filter) the data.
if go2D:
# process slicewise
mask = np.zeros_like(data)
for i, slc in enumerate(data):
print('processing slice: {}'.format(i))
smf, sdss = process_slice(slc, lower_threshold, upper_threshold,
size, dilation, disk, save_steps)
mask[i, :, :] = smf
for ds, sds in zip(dss, sdss):
ds[i, :, :] = sds
else:
# process full input
mask, dss = process_slice(data, lower_threshold, upper_threshold, size,
dilation, ball, save_steps)
# Apply additional mask.
if inmask is not None:
mask[~inmask] = False
# Write the mask(s) to file.
if '.h5' in h5path_out:
utils.write_to_h5ds(ds_out, mask.astype('uint8'), slices_out)
for step, ds in zip(steps, dss):
utils.write_to_h5ds(step[1], ds.astype('uint8'), slices_out)
else: # write as tif/png/jpg series
root, ext = os.path.splitext(outpaths['out'])
utils.write_to_img(root, mask.astype('uint8'), al, 5, ext, 0)
for stepname, ds in zip(stepnames, dss):
root, ext = os.path.splitext(outpaths[stepname])
utils.write_to_img(root, ds.astype('uint8'), al, 5, ext, 0)
# Close the h5 file(s) or return the output array(s).
try:
h5file_in.close()
h5file_out.close()
for step in steps:
step[0].close()
except (ValueError, AttributeError):
return mask, dss
def downsample_slices(
inputdir,
outputdir,
regex='*.tif',
ds_factor=4,
dataslices=None,
usempi=False,
protective=False,
):
"""Downsample a series of 2D images."""
if '.h5' in outputdir:
status, info = utils.h5_check(outputdir, protective)
print(info)
if status == "CANCELLED":
return
if '.h5' in inputdir: # FIXME: assumed zyx for now
h5file_in, ds_in, elsize, axlab = utils.h5_load(inputdir)
zyxdims = ds_in.shape
else:
# Get the list of input filepaths.
files = sorted(glob.glob(os.path.join(inputdir, regex)))
zyxdims = [len(files)] + list(io.imread(files[0]).shape)
axlab = 'zyx'
if '.h5' in outputdir:
elsize[1] = elsize[1] / ds_factor
elsize[2] = elsize[2] / ds_factor
outsize = [ds_in.shape[0],
ds_in.shape[1] / ds_factor,
ds_in.shape[2] / ds_factor]
h5file_out, ds_out = utils.h5_write(None, outsize, ds_in.dtype,
outputdir,
element_size_um=elsize,
axislabels=axlab)
else:
# Get the list of output filepaths.
utils.mkdir_p(outputdir)
outpaths = []
for fpath in files:
root, ext = os.path.splitext(fpath)
tail = os.path.split(root)[1]
outpaths.append(os.path.join(outputdir, tail + ext))
# Check if any output paths already exist.
status = utils.output_check_dir(outpaths, protective)
if status == "CANCELLED":
return
# Get the slice objects for the input data.
slices = utils.get_slice_objects_prc(dataslices, zyxdims)
# Prepare for processing with MPI.
mpi_info = utils.get_mpi_info(usempi)
series = np.array(range(slices[0].start,
slices[0].stop,
slices[0].step), dtype=int)
if mpi_info['enabled']:
series = utils.scatter_series(mpi_info, series)[0]
# Downsample and save the images.
for slc in series:
if '.h5' in inputdir:
sub = ds_in[slc, slices[1], slices[2]]
else:
sub = io.imread(files[slc])[slices[1], slices[2]]
img_ds = resize(sub, (sub.shape[0] / ds_factor,
sub.shape[1] / ds_factor))
if '.h5' in outputdir:
ds_out[slc, :, :] = img_ds
else:
imsave(outpaths[slc], img_ds)
# downsample_image(outpaths[slc], sub, ds_factor)
try:
h5file_in.close()
h5file_out.close()
except (ValueError, AttributeError):
pass
def mergeblocks(
h5paths_in,
blockoffset=[0, 0, 0],
blocksize=[],
margin=[0, 0, 0],
fullsize=[],
is_labelimage=False,
relabel=False,
neighbourmerge=False,
save_fwmap=False,
blockreduce=[],
func='np.amax',
datatype='',
usempi=False,
h5path_out='',
save_steps=False,
protective=False,
):
"""Merge blocks of data into a single hdf5 file."""
# prepare mpi
mpi_info = utils.get_mpi_info(usempi)
series = np.array(range(0, len(h5paths_in)), dtype=int)
if mpi_info['enabled']:
series = utils.scatter_series(mpi_info, series)[0]
# TODO: save_steps
# check output paths
outpaths = {'out': h5path_out}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5paths_in[0],
comm=mpi_info['comm'])
try:
ndim = ds_in.ndim
except AttributeError:
ndim = len(ds_in.dims)
# get the size of the outputfile
# TODO: option to derive fullsize from dset_names?
if blockreduce:
datasize = np.subtract(fullsize, blockoffset)
outsize = [
int(np.ceil(d / np.float(b)))
for d, b in zip(datasize, blockreduce)
]
elsize = [e * b for e, b in zip(elsize, blockreduce)]
else: # FIXME: 'zyx(c)' stack assumed
outsize = np.subtract(fullsize, blockoffset)
if ndim == 4:
outsize = list(outsize) + [ds_in.shape[3]] # TODO: flexible insert
datatype = datatype or ds_in.dtype
chunks = ds_in.chunks or None
h5file_in.close()
# open data for writing
h5file_out, ds_out = utils.h5_write(data=None,
shape=outsize,
dtype=datatype,
h5path_full=h5path_out,
chunks=chunks,
element_size_um=elsize,
axislabels=axlab,
comm=mpi_info['comm'])
# merge the datasets
maxlabel = 0
for i in series:
h5path_in = h5paths_in[i]
try:
maxlabel = process_block(h5path_in, ndim, blockreduce, func,
blockoffset, blocksize, margin, fullsize,
ds_out, is_labelimage, relabel,
neighbourmerge, save_fwmap, maxlabel,
usempi, mpi_info)
print('processed block {:03d}: {}'.format(i, h5path_in))
except Exception as e:
print('failed block {:03d}: {}'.format(i, h5path_in))
print(e)
# close and return
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def series2stack(
inputdir,
regex='*.tif',
element_size_um=[None, None, None],
outlayout='zyx',
datatype='',
chunksize=[20, 20, 20],
dataslices=None,
usempi=False,
outputformats=['.h5'],
outputpath='',
save_steps=False,
protective=False,
):
""""Convert a directory of tifs to an hdf5 stack."""
# Check if any output paths already exist.
if '.h5' in outputformats:
outpaths = {'out': outputpath}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# Get the list of input filepaths.
files = sorted(glob.glob(os.path.join(inputdir, regex)))
# Get some metadata from the inputfiles
zyxdims, datatype, element_size_um = get_metadata(files,
datatype,
outlayout,
element_size_um)
# (plane, row, column) indexing to outlayout (where prc -> zyx).
in2out = ['zyx'.index(o) for o in outlayout]
# Get the properties of the output dataset.
slices = utils.get_slice_objects_prc(dataslices, zyxdims) # prc-order
files = files[slices[0]]
datashape_out_prc = (len(files),
len(range(*slices[1].indices(slices[1].stop))),
len(range(*slices[2].indices(slices[2].stop))))
datashape_out = [datashape_out_prc[i] for i in in2out]
# Reshape the file list into a list of blockwise file lists.
scs = chunksize[outlayout.index('z')] # chunksize slice dimension
files_blocks = zip(* [iter(files)] * scs)
rem = len(files) % scs
if rem:
files_blocks += [tuple(files[-rem:])]
# Get slice objects for every output block.
slices_out_prc = [[slice(bnr * scs, bnr * scs + scs),
slice(0, datashape_out_prc[1]),
slice(0, datashape_out_prc[2])]
for bnr in range(0, len(files_blocks))]
slices_out = [[sliceset_prc[i] for i in in2out]
for sliceset_prc in slices_out_prc]
# Prepare for processing with MPI.
mpi_info = utils.get_mpi_info(usempi)
series = np.array(range(0, len(files_blocks)), dtype=int)
if mpi_info['enabled']:
series = utils.scatter_series(mpi_info, series)[0]
# Open the outputfile for writing and create the dataset or output array.
if '.h5' in outputformats:
h5file_out, ds_out = utils.h5_write(None, datashape_out, datatype,
outputpath,
element_size_um=element_size_um,
axislabels=outlayout,
chunks=tuple(chunksize),
comm=mpi_info['comm'])
outdir = os.path.dirname(outputpath.split('.h5')[0])
else:
ds_out = None
outdir = outputpath
# Write blocks of 2D images to the outputfile(s).
for blocknr in series:
if '.h5' in outputformats:
ds_out = process_block(files_blocks[blocknr], ds_out,
slices, slices_out[blocknr], in2out,
outputformats, outdir)
else:
process_slices(files_blocks[blocknr],
slices, slices_out[blocknr],
outputformats, outdir, datatype)
# Close the h5 files or return the output array.
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
except UnboundLocalError:
pass
def separate_sheaths(
h5path_in,
h5path_lmm='',
h5path_wsmask='',
h5path_mask='',
h5path_mmm='',
MAdilation=0,
h5path_dist='',
sigmoidweighting=0,
margin=50,
h5path_out='',
save_steps=False,
protective=False,
):
"""Separate the myelin compartment into individual myelin sheaths."""
# check output paths
outpaths = {
'out': h5path_out,
'wsmask': '',
'madil{:02d}'.format(MAdilation): '',
'distance_simple': '',
'sheaths_simple': '',
'distance_sigmod': '',
}
root, ds_main = outpaths['out'].split('.h5')
for dsname, outpath in outpaths.items():
grpname = ds_main + "_steps"
outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname)
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
ds_mma = ds_in[:] != 0
# open data for writing
h5file_out, ds_out = utils.h5_write(None,
ds_in.shape,
ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# load/calculate a mask to constrain the watershed in
mask_mmregion = get_wsmask(outpaths, ds_mma, h5path_wsmask, h5path_mask,
h5path_mmm, MAdilation, elsize, axlab)
elsize_abs = np.absolute(elsize)
seeds = grey_dilation(ds_in, size=(3, 3, 3))
# load/calculate the distance transform
if h5path_dist:
h5file_dist, ds_dist, _, _ = utils.h5_load(h5path_dist)
else:
if sigmoidweighting:
if h5path_lmm:
ds_lmm = utils.h5_load(h5path_lmm, load_data=True)[0]
else:
ds_dist = distance_transform_edt(~ds_mma, sampling=elsize_abs)
ds_dist = img_as_float(ds_dist)
utils.save_step(outpaths, 'distance_simple', ds_dist, elsize,
axlab)
ds_lmm = watershed(ds_dist, seeds, mask=mask_mmregion)
utils.save_step(outpaths, 'sheaths_simple', ds_lmm, elsize,
axlab)
ds_dist, _ = distance_transform_sw(ds_in, ds_lmm, elsize_abs,
sigmoidweighting, margin)
utils.save_step(outpaths, 'distance_sigmod', ds_dist, elsize,
axlab)
ds_out[:] = watershed(ds_dist, seeds, mask=mask_mmregion)
# TODO: save median widths mw
else:
ds_dist = distance_transform_edt(~ds_mma, sampling=elsize_abs)
utils.save_step(outpaths, 'distance_simple', ds_dist, elsize,
axlab)
ds_out[:] = watershed(ds_dist, seeds, mask=mask_mmregion)
# close and return
h5file_in.close()
if h5path_dist:
h5file_dist.close()
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
y[l] = True
y[0] = False
np.save(gtpath, y)
# In[11]:
# map the groundtruth labels to a volume
h5file_in, ds_in, elsize, axlab = utils.h5_load(labelpath)
h5path_out = os.path.join(datadir, '{}_gt.h5'.format(dataset))
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, 'uint8',
os.path.join(h5path_out, 'class0'),
element_size_um=elsize,
axislabels=axlab)
ds_out[:] = ~y[ds_in[:]]
h5file_out.close()
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, 'uint8',
os.path.join(h5path_out, 'class1'),
element_size_um=elsize,
axislabels=axlab)
ds_out[:] = y[ds_in[:]]
h5file_out.close()
h5file_in.close()
# convert to nifti
niipath_out = os.path.join(datadir, '{}_gt_class0.nii.gz'.format(dataset))
def CC_3D(
h5path_in,
h5path_mask='',
min_size_maskMM=0,
min_area=0,
h5path_out='',
protective=False,
):
"""Label connected components in a 3D stack."""
# check output path
if '.h5' in h5path_out:
status, info = utils.h5_check(h5path_out, protective)
print(info)
if status == "CANCELLED":
return
# open data for reading
h5file_mm, ds_mm, elsize, axlab = utils.h5_load(h5path_in)
if h5path_mask:
h5file_md, ds_md, _, _ = utils.h5_load(h5path_mask)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_mm.shape, 'uint32',
h5path_out,
element_size_um=elsize,
axislabels=axlab)
# 3D labeling with label size constraints
# NOTE: could save memory here by applying the constraints to input before
# if h5path_mask:
# mask = np.logical_or(binary_dilation(ds_mm[:]), ~ds_md[:])
# else:
# mask = binary_dilation(ds_mm[:])
if min_size_maskMM:
mask = label(ds_mm[:], return_num=False, connectivity=None)
remove_small_objects(mask, min_size_maskMM, in_place=True)
mask = ~mask.astype('bool')
else:
mask = ~ds_mm[:]
labels = label(mask, return_num=False, connectivity=None)
# shuffle_labels = True
# if shuffle_labels:
# fw = utils.shuffle_labels(labels)
# labels = fw[labels]
if min_area:
remove_small_objects(labels, min_area, in_place=True)
# remove the largest label (assumed unmyelinated axon compartment)
rp = regionprops(labels)
areas = [prop.area for prop in rp]
labs = [prop.label for prop in rp]
llab = labs[np.argmax(areas)]
labels[labels == llab] = 0
labels = relabel_sequential(labels)[0]
print('maxlabel: {}'.format(np.amax(labels)))
ds_out[:] = labels
# close and return
try:
h5file_mm.close()
h5file_out.close()
if h5path_mask:
h5file_md.close()
except (ValueError, AttributeError):
return labels
def remap_labels(
inputfile,
delete_labels=[],
delete_files=[],
except_files=[],
merge_labels=[],
merge_files=[],
split_labels=[],
split_files=[],
aux_labelvolume='',
min_labelsize=0,
min_segmentsize=0,
keep_only_largest=False,
conncomp=False,
nifti_output=False,
nifti_transpose=False,
h5path_out='',
save_steps=False,
protective=False,
):
"""Delete/split/merge/... labels in a labelvolume."""
# check output paths
outpaths = {'out': h5path_out, 'deleted': '', 'split': '', 'merged': ''}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.load(inputfile)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
ds_out[:] = ds_in[:]
# filter labels on size
root = os.path.splitext(h5file_out.filename)[0]
ls_small = utils.filter_on_size(ds_out[:], min_labelsize, False,
save_steps, root, ds_out.name[1:],
outpaths, elsize, axlab)[1]
delete_labels = set(delete_labels) | ls_small
# delete labels
delete_labelsets(ds_out, delete_labels, delete_files, except_files)
if save_steps: # FIXME!
save_diff(ds_in[:], ds_out[:], 'deleted', outpaths, elsize, axlab)
# split labels
split_labelsets(ds_out, split_labels, split_files, aux_labelvolume, conncomp)
if save_steps:
save_diff(ds_in[:], ds_out[:], 'split', outpaths, elsize, axlab)
# merge labels
merge_labelsets(ds_out, merge_labels, merge_files)
if save_steps:
save_diff(ds_in[:], ds_out[:], 'merged', outpaths, elsize, axlab)
# remove small, non-contiguous segments of labels
if min_segmentsize or keep_only_largest:
filter_segments(ds_out[:], min_segmentsize, keep_only_largest)
if nifti_output:
if nifti_transpose:
ds_out[:] = np.transpose(ds_out[:])
elsize = elsize[::-1]
axlab = axlab[::-1]
fpath = '{}_{}.nii.gz'.format(root, ds_out.name[1:])
utils.write_to_nifti(fpath, ds_out[:], elsize)
# close and return
h5file_in.close()
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
def CC_2D(
h5path_in,
h5path_mask='',
slicedim=0,
usempi=False,
h5path_out='',
protective=False,
):
"""Label connected components in all slices."""
# check output path
if '.h5' in h5path_out:
status, info = utils.h5_check(h5path_out, protective)
print(info)
if status == "CANCELLED":
return
# open data for reading
h5file_mm, ds_mm, elsize, axlab = utils.h5_load(h5path_in)
if h5path_mask:
h5file_md, ds_md, _, _ = utils.h5_load(h5path_mask)
# prepare mpi # TODO: could allow selection of slices/subset here
mpi_info = utils.get_mpi_info(usempi)
n_slices = ds_mm.shape[slicedim]
series = np.array(range(0, n_slices), dtype=int)
if mpi_info['enabled']:
series = utils.scatter_series(mpi_info, series)[0]
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_mm.shape, 'uint32',
h5path_out,
element_size_um=elsize,
axislabels=axlab,
comm=mpi_info['comm'])
# slicewise labeling
maxlabel = 0
for i in series:
slcMM = utils.get_slice(ds_mm, i, slicedim, 'bool')
if h5path_mask:
slcMD = utils.get_slice(ds_md, i, slicedim, 'bool')
labels, num = label(np.logical_and(~slcMM, slcMD), return_num=True)
else:
labels, num = label(~slcMM, return_num=True)
print("found %d labels in slice %d" % (num, i))
if mpi_info['enabled']:
# NOTE: assumed max number of labels in slice is 10000
labels[~slcMM] += 10000 * i
if i == n_slices - 1:
maxlabel = np.amax(labels)
else:
labels[~slcMM] += maxlabel
maxlabel += num
if slicedim == 0:
ds_out[i, :, :] = labels
elif slicedim == 1:
ds_out[:, i, :] = labels
elif slicedim == 2:
ds_out[:, :, i] = labels
# save the maximum labelvalue in the dataset
print("found %d labels" % (maxlabel))
if mpi_info['rank'] == mpi_info['size'] - 1:
root = h5path_out.split('.h5')[0]
fpath = root + '.npy'
np.save(fpath, np.array([maxlabel]))
# close and return
try:
h5file_mm.close()
h5file_out.close()
if h5path_mask:
h5file_md.close()
except (ValueError, AttributeError):
return ds_out
def agglo_from_labelmask(
h5path_in,
h5path_lv='',
ratio_threshold=0,
h5path_out='',
save_steps=False,
protective=False,
):
"""Apply mapping of labelsets to a labelvolume."""
# check output paths
outpaths = {'out': h5path_out}
status = utils.output_check(outpaths, save_steps, protective)
if status == "CANCELLED":
return
# open data for reading
h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in)
h5file_lv, ds_lv, _, _ = utils.h5_load(h5path_lv)
# open data for writing
h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype,
h5path_out,
element_size_um=elsize,
axislabels=axlab)
ulabels = np.unique(ds_in)
maxlabel = np.amax(ulabels)
print("number of labels in watershed: {:d}".format(maxlabel))
fwmap = np.zeros(maxlabel + 1, dtype='i')
areas_ws = np.bincount(ds_in.ravel())
labelsets = {}
rp_lw = regionprops(ds_lv, ds_in)
for prop in rp_lw:
maskedregion = prop.intensity_image[prop.image]
counts = np.bincount(maskedregion)
svoxs_in_label = [l for sl in np.argwhere(counts) for l in sl]
ratios_svox_in_label = [float(counts[svox]) / float(areas_ws[svox])
for svox in svoxs_in_label]
fwmask = np.greater(ratios_svox_in_label, ratio_threshold)
labelset = np.array(svoxs_in_label)[fwmask]
labelsets[prop.label] = set(labelset) - set([0])
basepath = h5path_in.split('.h5/')[0]
utils.write_labelsets(labelsets, basepath + "_svoxsets",
filetypes=['pickle'])
ds_out[:] = utils.forward_map(np.array(fwmap), ds_in, labelsets)
# close and return
h5file_in.close()
h5file_lv.close()
try:
h5file_out.close()
except (ValueError, AttributeError):
return ds_out
