def map_to_volume(fw, h5path_in, h5path_out, h5path_mask=''): """Map the prediction to a labelvolume.""" h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) a = ds_in[:] h5file_in.close() # myelinated axon labels h5file_out, ds_out = utils.h5_write(None, a.shape, a.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) mask = fw[a] a[~mask] = 0 ds_out[:] = a h5file_out.close() # myelinated axon mask if not h5path_mask: return h5file_out, ds_out = utils.h5_write(None, a.shape, 'bool', h5path_mask, element_size_um=elsize, axislabels=axlab) ds_out[:] = a.astype('bool') h5file_out.close()
def combine_vols( h5path_in, vol_idxs=[0, 2, 4, 7], h5path_out='', protective=False, ): """Combine volumes by addition.""" if bool(h5path_out) and ('.h5' in h5path_out): status, info = utils.h5_check(h5path_out, protective) print(info) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, es, al = utils.load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape[:3], ds_in.dtype, h5path_out, element_size_um=es[:3], axislabels=al[:3]) out = np.zeros(ds_out.shape, dtype=ds_out.dtype) for volnr in vol_idxs: out += ds_in[:, :, :, volnr] ds_out[:] = out # close and return h5file_in.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def CC_2Dto3D( h5path_in, h5path_out='', protective=False, ): """Label connected components in 3D from the 2D-generated mask.""" # check output path if h5path_out: status, info = utils.h5_check(h5path_out, protective) print(info) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, 'uint32', h5path_out, element_size_um=elsize, axislabels=axlab) # NOTE: scipy has much lower memory consumption than scikit-image # TODO: input mask directly, not labelimage? ds_out[:] = label(ds_in[:, :, :] != 0) # ds_out[:] = scipy_label(ds_in[:, :, :] != 0)[0] # close and return try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def write_block(ds_in, elsize, axlab, block): """Write the block to file.""" shape = list(block['size']) if ds_in.ndim == 4: shape += [ds_in.shape[3]] chunks = ds_in.chunks if any(np.array(chunks) > np.array(shape)): chunks = True h5file_out, ds_out = utils.h5_write( data=None, shape=shape, dtype=ds_in.dtype, h5path_full=block['h5path'], chunks=chunks, element_size_um=elsize, axislabels=axlab, ) slcs = block['slc'] if ds_in.ndim == 3: ds_out[:] = ds_in[slcs[0], slcs[1], slcs[2]] elif ds_in.ndim == 4: ds_out[:] = ds_in[slcs[0], slcs[1], slcs[2], :] h5file_out.close()
def downsample_blockwise( h5path_in, blockreduce=[3, 3, 3], func='np.amax', dataslices=None, h5path_out='', save_steps=False, protective=False, ): """Downsample volume by blockwise reduction.""" # Check if any output paths already exist. outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # Open the inputfile for reading. # TODO: option to get the input data passed h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # Get the matrix size and resolution of the outputdata. outsize, elsize = get_new_sizes(func, blockreduce, ds_in.shape, elsize) # Open the outputfile for writing and create the dataset or output array. h5file_out, ds_out = utils.h5_write(None, outsize, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # Get the slice objects for the input data. slices = utils.get_slice_objects_prc(dataslices, ds_in.shape) # Reformat the data to the outputsize. if func == 'expand': out = ds_in[slices[0], ...] for axis in range(0, ds_out.ndim): out = np.repeat(out, blockreduce[axis], axis=axis) ds_out[slices[0], ...] = out else: """ TODO: flexible mapping from in to out now: the reduction factor of the first axis must be 1; the extent of the remaining axes must be full """ ds_out[slices[0], ...] = block_reduce(ds_in[slices[0], ...], block_size=tuple(blockreduce), func=eval(func)) # Close the h5 files or return the output array. try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def map_labels( h5path_in, h5path_out='', save_steps=False, protective=False, ): """Map groups of labels to a single label.""" # check output path outpaths = {'out': h5path_out, 'stitched': ''} root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # load the pickled set of neighbours lsroot = h5path_out.split('.h5')[0] lspath = '{}_{}.pickle'.format(lsroot, ds_out.name[1:]) with open(lspath, "r") as f: labelsets = pickle.load(f) # apply forward map ulabels = np.unique(ds_in[:]) maxlabel = np.amax(ulabels) fw = [l if l in ulabels else 0 for l in range(0, maxlabel + 1)] labels = utils.forward_map(np.array(fw), ds_in[:], labelsets) if save_steps: fw = np.zeros(maxlabel + 1, dtype='i') MAlabels = utils.forward_map(np.array(fw), ds_in[:], labelsets) utils.save_step(outpaths, 'stitched', MAlabels, elsize, axlab) ds_out[:, :, :] = labels # close and return try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def CC_2Dprops( h5path_labels, basename, map_propnames, usempi=False, h5path_out='', protective=False, ): """Map the labels/properties.""" # check output paths if '.h5' in h5path_out: for propname in map_propnames: h5path_prop = os.path.join(h5path_out, propname) status, info = utils.h5_check(h5path_out, protective) print(info) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_labels) # prepare mpi n_props = len(map_propnames) series = np.array(range(0, n_props), dtype=int) mpi_info = utils.get_mpi_info(usempi) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] fws = {} for i in series: propname = map_propnames[i] print("processing prop %s" % propname) nppath = '{}_{}.npy'.format(basename, propname) fws[propname] = np.load(nppath) # open data for writing h5path_prop = os.path.join(h5path_out, propname) h5file_prop, ds_prop = utils.h5_write(None, ds_in.shape, fws[propname].dtype, h5path_prop, element_size_um=elsize, axislabels=axlab, comm=mpi_info['comm']) ds_prop[:] = fws[propname][ds_in[:]] h5file_prop.close() # close and return h5file_in.close()
def filter_NoR( h5path_in, h5path_2D, h5path_out='', save_steps=False, protective=False, ): """Filter nodes of ranvier.""" # check output paths outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) h5file_2D, ds_2D, _, _ = utils.h5_load(h5path_2D) # open data for writing h5file_out, ds_out = utils.h5_write(ds_in, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) labelsets = {i: set(np.unique(ds_2D[i, :, :])) for i in range(ds_2D.shape[0])} ulabels = np.unique(ds_in) m = {l: np.array([True if l in lsv else False for _, lsv in labelsets.items()]) for l in ulabels} rp = regionprops(ds_in) for prop in rp: z, y, x, Z, Y, X = tuple(prop.bbox) mask = prop.image mask[m[prop.label][z:Z], :, :] = 0 ds_out[z:Z, y:Y, x:X][mask] = 0 # close and return h5file_in.close() h5file_2D.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def nodes_of_ranvier( h5path_in, min_labelsize=0, remove_small_labels=False, h5path_boundarymask='', merge_methods=['neighbours'], overlap_threshold=20, h5path_data='', h5path_mmm='', searchradius=[100, 30, 30], h5path_out='', save_steps=False, protective=False, ): """Find labels that do not traverse through the volume.""" # check output paths outpaths = {'out': h5path_out, 'largelabels': '', 'smalllabelmask': '', 'boundarymask': '', 'labels_nt': '', 'labels_tv': '', 'filled': '', } root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) labels = ds_in[:] # FIXME: do we make a copy, or use ds_out? # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # start with the set of all labels ulabels = np.unique(labels) maxlabel = np.amax(ulabels) labelset = set(ulabels) print("number of labels in labelvolume: {}".format(len(labelset))) # get the labelsets that touch the borders sidesmask = get_boundarymask(h5path_boundarymask, 'invdil') ls_bot = set(np.unique(labels[:4, :, :])) ls_top = set(np.unique(labels[-4:, :, :])) ls_sides = set(np.unique(labels[sidesmask])) ls_border = ls_bot | ls_top | ls_sides ls_centre = labelset - ls_border # get the labels that do not touch the border twice ls_bts = (ls_bot ^ ls_top) ^ ls_sides ls_tbs = (ls_top ^ ls_bot) ^ ls_sides ls_sbt = (ls_sides ^ ls_bot) ^ ls_top ls_nt = ls_centre | ls_bts | ls_tbs | ls_sbt # filter labels on size root = os.path.splitext(h5file_out.filename)[0] ls_small = utils.filter_on_size(labels, labelset, min_labelsize, remove_small_labels, save_steps, root, ds_out.name[1:], outpaths, elsize, axlab)[2] labelset -= ls_small ls_nt -= ls_small ls_short = filter_on_heigth(labels, 5) labelset -= ls_short ls_nt -= ls_short ls_tv = labelset - ls_nt print('number of large, long labels: {}'.format(len(labelset))) print('number of large, long in-volume labels: {}'.format(len(ls_nt))) print('number of large, long through-volume labels: {}'.format(len(ls_tv))) # map the large labels that don't traverse the volume fw_nt = np.zeros(maxlabel + 1, dtype='i') for l in ls_nt: fw_nt[l] = l labels_nt = fw_nt[labels] # automated label merge labelsets = {} min_labelsize = 10 labelsets, filled = merge_labels(labels_nt, labelsets, merge_methods, overlap_threshold, h5path_data, h5path_mmm, min_labelsize, searchradius) fw = np.zeros(maxlabel + 1, dtype='i') ds_out[:] = utils.forward_map(np.array(fw), labels, labelsets) if save_steps: utils.save_step(outpaths, 'boundarymask', sidesmask, elsize, axlab) utils.save_step(outpaths, 'labels_nt', labels_nt, elsize, axlab) fw_tv = np.zeros(maxlabel + 1, dtype='i') for l in ls_tv: fw_tv[l] = l labels_tv = fw_tv[labels] utils.save_step(outpaths, 'labels_tv', labels_tv, elsize, axlab) if filled is not None: fw = np.zeros(maxlabel + 1, dtype='i') filled = utils.forward_map(np.array(fw), filled, labelsets) utils.save_step(outpaths, 'filled', filled, elsize, axlab) filestem = '{}_{}_automerged'.format(root, ds_out.name[1:]) utils.write_labelsets(labelsets, filestem, filetypes=['txt', 'pickle']) # close and return h5file_in.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def watershed_ics( h5path_in, masks=[], h5path_seeds='', seed_size=64, lower_threshold=None, upper_threshold=None, invert=False, h5path_out='', save_steps=False, protective=False, ): """Perform watershed on the intracellular space compartments.""" # check output paths outpaths = {'out': h5path_out, 'seeds': h5path_seeds, 'mask': ''} root, ds_main = outpaths['out'].split('.h5') for dsname, _ in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape[:3], 'uint32', h5path_out, element_size_um=elsize, axislabels=axlab) # load/generate the seeds if h5path_seeds: h5file_sds, ds_sds, _, _ = utils.h5_load(h5path_seeds) else: h5file_sds, ds_sds = utils.h5_write(None, ds_in.shape[:3], 'uint32', outpaths['seeds'], element_size_um=elsize, axislabels=axlab) lower_threshold = lower_threshold or np.amin(ds_in[:]) - 1 upper_threshold = upper_threshold or np.amax(ds_in[:]) + 1 ds_sds[:] = np.logical_and(ds_in[:] > lower_threshold, ds_in[:] <= upper_threshold) ds_sds[:], _ = label(ds_sds[:]) ds_sds[:] = remove_small_objects(ds_sds[:], min_size=seed_size) ds_sds[:] = relabel_sequential(ds_sds[:])[0] """ NOTE: numpy/scipy/skimage inplace is not written when using hdf5 Therefore, we cannot use: np.logical_and(ds_in[:] > lower_threshold, ds_in[:] <= upper_threshold, ds_sds[:]) num = label(ds_sds[:], output=ds_sds[:]) remove_small_objects(ds_sds[:], min_size=seed_size, in_place=True) """ # determine the mask mask = np.ones(ds_in.shape[:3], dtype='bool') mask = utils.string_masks(masks, mask) h5file_mask, ds_mask = utils.h5_write(None, ds_in.shape[:3], 'uint8', outpaths['mask'], element_size_um=elsize, axislabels=axlab) ds_mask[:] = mask # ds_mask[:].fill(1) # ds_mask[:] = utils.string_masks(masks, ds_mask[:]) # perform the watershed if invert: ds_out[:] = watershed(-ds_in[:], ds_sds[:], mask=ds_mask[:]) else: ds_out[:] = watershed(ds_in[:], ds_sds[:], mask=ds_mask[:]) # close and return h5file_in.close() try: h5file_out.close() h5file_sds.close() h5file_mask.close() except (ValueError, AttributeError): return ds_out, ds_sds, ds_mask
def filter_labels( h5path_in, h5path_mm='', min_labelsize=0, close=None, relabel_from=0, h5path_out='', save_steps=False, protective=False, ): """Map groups of labels to a single label.""" # check output path outpaths = {'out': h5path_out, 'closed': ''} root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) labels = ds_in[:] # if min_labelsize: # remove_small_objects(labels, min_size=min_labelsize, in_place=True) if close is not None: labels = close_labels(labels, close) if h5path_mm: print('removing voxels in mask') h5file_mm, ds_mm, _, _ = utils.h5_load(h5path_mm) labels[ds_mm[:].astype('bool')] = 0 h5file_mm.close() if save_steps: utils.save_step(outpaths, 'closed', labels, elsize, axlab) if min_labelsize: print('removing small labels') remove_small_objects(labels, min_size=min_labelsize, in_place=True) # if save_steps: # utils.save_step(outpaths, 'small', smalllabels, elsize, axlab) if relabel_from > 1: print('relabeling from {}'.format(relabel_from)) labels = relabel_sequential(labels, relabel_from)[0] # TODO: save mapping? ds_out[:, :, :] = labels # close and return try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def stack2stack( inputfile, outputfile, dset_name='', blockoffset=[], datatype='', uint8conv=False, inlayout='', outlayout='', elsize=[], chunksize=[], additional_outputs=[], nzfills=5, dataslices=None, save_steps=False, protective=False, ): """Convert/select/downscale/transpose/... an hdf5 dataset.""" # output root and exts root, ext = split_filepath(outputfile) outexts = list(set(additional_outputs + [ext])) # Check if any output paths already exist. outpaths = {'out': outputfile, 'addext': (root, additional_outputs)} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return h5file_in, ds_in, h5elsize, h5axlab = utils.h5_load(inputfile) try: ndim = ds_in.ndim except AttributeError: ndim = len(ds_in.dims) # data layout # FIXME: h5axlab not necessarily xyzct! # FIXME: this forces a possibly erroneous outlayout when inlayout is not found inlayout = inlayout or ''.join(h5axlab) or 'zyxct'[0:ndim] outlayout = outlayout or inlayout in2out = [inlayout.index(l) for l in outlayout] # element size elsize = elsize or (h5elsize[in2out] if h5elsize is not None else None) # chunksize if chunksize is not None: chunksize = tuple(chunksize) or ( True if not any(chunksize) else (tuple([ds_in.chunks[i] for i in in2out]) if ds_in.chunks else None ) ) # datatype datatype = datatype or ds_in.dtype if dset_name: _, x, X, y, Y, z, Z = utils.split_filename(dset_name, blockoffset) slices = {'x': [x, X, 1], 'y': [y, Y, 1], 'z': [z, Z, 1]} if ndim > 3: C = ds_in.shape[inlayout.index('c')] slices['c'] = [0, C, 1] sliceslist = [slices[dim] for dim in inlayout] dataslices = [item for sl in sliceslist for item in sl] # get the selected and transformed data # TODO: most memory-efficient solution data = utils.load_dataset(ds_in, elsize, inlayout, outlayout, datatype, dataslices, uint8conv)[0] h5file_in.close() # write the data for ext in outexts: if '.nii' in ext: if data.dtype == 'float16': data = data.astype('float') utils.write_to_nifti(root + '.nii.gz', data, elsize) if '.h5' in ext: utils.h5_write(data, data.shape, data.dtype, outputfile, element_size_um=elsize, axislabels=outlayout, chunks=chunksize) if (('.tif' in ext) | ('.png' in ext) | ('.jpg' in ext)) & (data.ndim < 4): if data.ndim == 2: data = data.atleast_3d() outlayout += 'z' utils.write_to_img(root, data, outlayout, nzfills, ext) return data
m = fwf > 3000 # always exclude fw[m] = 0 # In[35]: ## map to volume h5file_in, ds_in, elsize, axlab = utils.h5_load(labelpath) a = ds_in[:] h5file_in.close() # myelinated axon labels h5path_out = os.path.join(labelfile, '{}_prediction'.format(labeldset)) h5file_out, ds_out = utils.h5_write(None, a.shape, a.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) mask = fw[a] a[~mask] = 0 ds_out[:] = a h5file_out.close() # myelinated axon mask h5path_out = os.path.join(datadir, '{}_masks_maskPRED.h5'.format(dataset), 'maskMA') h5file_out, ds_out = utils.h5_write(None, a.shape, 'bool', h5path_out,
def fill_holes( h5path_in, methods, selem, h5path_mask='', h5path_md='', h5path_mm='', h5path_mx='', h5path_out='', h5path_out_holes='', h5path_out_ma='', h5path_out_mm='', protective=False, ): """Fill holes in labels.""" # check output path # TODO check other output volumes (holes, ma, mm) for outpath in [h5path_out, h5path_out_holes, h5path_out_ma, h5path_out_mm]: if '.h5' in outpath: status, info = utils.h5_check(outpath, protective) print(info) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) if h5path_mask: h5file_ml, ds_ml, _, _ = utils.h5_load(h5path_mask) ds_in[~ds_ml[:].astype('bool')] = 0 h5file_ml.close() # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # fill holes ds_out[:] = ds_in[:] # TODO: simply make copy h5file_in and open it? for m in methods: if m == '1': ds_out = fill_holes_m1(ds_out, selem) if m == '2': ds_out = fill_holes_m2(ds_out, selem) if m == '3': ds_out = fill_holes_m3(ds_out, selem) if m == '4': ds_out = fill_holes_m4(ds_out, h5path_md, h5path_mm, h5path_mx) # additional output: holes filled and updated masks if h5path_out_holes: h5file_ho, ds_ho, _, _ = utils.h5_write(None, ds_out.shape, ds_out.dtype, h5path_out_holes, element_size_um=elsize, axislabels=axlab) ds_ho[:] = ds_out[:] ds_ho[ds_in > 0] = 0 h5file_ho.close() if h5path_out_ma: ds_ma = utils.h5_write(ds_out.astype('bool'), ds_out.shape, 'uint8', h5path_out_ma, element_size_um=elsize, axislabels=axlab)[1] if h5path_out_mm: ds_mm = utils.h5_load(h5path_mm, load_data=True) ds_mm[ds_ho > 0] = 0 ds_mm = utils.h5_write(ds_mm.astype('bool'), ds_mm.shape, 'uint8', h5path_out_mm, element_size_um=elsize, axislabels=axlab) # close and return try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out, ds_ho, ds_ma, ds_mm
def prob2mask( h5path_in, dataslices=None, lower_threshold=0, upper_threshold=1, size=0, dilation=0, h5path_mask=None, go2D=False, h5path_out='', save_steps=False, protective=False, ): """Create thresholded hard segmentation.""" # Check if any output paths already exist. outpaths = {'out': h5path_out, 'raw': '', 'mito': '', 'dil': ''} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return root, ds_main = outpaths['out'].split('.h5') grpname = ds_main + "_steps" for dsname, outpath in outpaths.items(): if ((dsname != 'out') and save_steps and (not outpath)): outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) # Load the (sliced) data. h5file_in, ds_in, es, al = utils.load(h5path_in) data, _, _, slices_out = utils.load_dataset(ds_in, dataslices=dataslices) inmask = utils.load(h5path_mask, load_data=True, dtype='bool', dataslices=dataslices)[0] if data.ndim == 4: # FIXME: generalize # data = np.squeeze(data) data = data[:, :, :, 0] outshape = ds_in.shape[:3] es = es[:3] al = al[:3] slices_out = slices_out[:3] else: outshape = ds_in.shape # Open the outputfile(s) for writing # and create the dataset(s) or output array(s). h5file_out, ds_out = utils.h5_write(None, outshape, 'uint8', outpaths['out'], element_size_um=es, axislabels=al) if save_steps: stepnames = ['raw', 'mito', 'dil'] steps = [ utils.h5_write(None, outshape, 'uint8', outpaths[out], element_size_um=es, axislabels=al) for out in stepnames ] dss = [np.zeros_like(data)] * 3 else: stepnames, steps, dss = [], [], [] # Threshold (and dilate and filter) the data. if go2D: # process slicewise mask = np.zeros_like(data) for i, slc in enumerate(data): print('processing slice: {}'.format(i)) smf, sdss = process_slice(slc, lower_threshold, upper_threshold, size, dilation, disk, save_steps) mask[i, :, :] = smf for ds, sds in zip(dss, sdss): ds[i, :, :] = sds else: # process full input mask, dss = process_slice(data, lower_threshold, upper_threshold, size, dilation, ball, save_steps) # Apply additional mask. if inmask is not None: mask[~inmask] = False # Write the mask(s) to file. if '.h5' in h5path_out: utils.write_to_h5ds(ds_out, mask.astype('uint8'), slices_out) for step, ds in zip(steps, dss): utils.write_to_h5ds(step[1], ds.astype('uint8'), slices_out) else: # write as tif/png/jpg series root, ext = os.path.splitext(outpaths['out']) utils.write_to_img(root, mask.astype('uint8'), al, 5, ext, 0) for stepname, ds in zip(stepnames, dss): root, ext = os.path.splitext(outpaths[stepname]) utils.write_to_img(root, ds.astype('uint8'), al, 5, ext, 0) # Close the h5 file(s) or return the output array(s). try: h5file_in.close() h5file_out.close() for step in steps: step[0].close() except (ValueError, AttributeError): return mask, dss
def downsample_slices( inputdir, outputdir, regex='*.tif', ds_factor=4, dataslices=None, usempi=False, protective=False, ): """Downsample a series of 2D images.""" if '.h5' in outputdir: status, info = utils.h5_check(outputdir, protective) print(info) if status == "CANCELLED": return if '.h5' in inputdir: # FIXME: assumed zyx for now h5file_in, ds_in, elsize, axlab = utils.h5_load(inputdir) zyxdims = ds_in.shape else: # Get the list of input filepaths. files = sorted(glob.glob(os.path.join(inputdir, regex))) zyxdims = [len(files)] + list(io.imread(files[0]).shape) axlab = 'zyx' if '.h5' in outputdir: elsize[1] = elsize[1] / ds_factor elsize[2] = elsize[2] / ds_factor outsize = [ds_in.shape[0], ds_in.shape[1] / ds_factor, ds_in.shape[2] / ds_factor] h5file_out, ds_out = utils.h5_write(None, outsize, ds_in.dtype, outputdir, element_size_um=elsize, axislabels=axlab) else: # Get the list of output filepaths. utils.mkdir_p(outputdir) outpaths = [] for fpath in files: root, ext = os.path.splitext(fpath) tail = os.path.split(root)[1] outpaths.append(os.path.join(outputdir, tail + ext)) # Check if any output paths already exist. status = utils.output_check_dir(outpaths, protective) if status == "CANCELLED": return # Get the slice objects for the input data. slices = utils.get_slice_objects_prc(dataslices, zyxdims) # Prepare for processing with MPI. mpi_info = utils.get_mpi_info(usempi) series = np.array(range(slices[0].start, slices[0].stop, slices[0].step), dtype=int) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] # Downsample and save the images. for slc in series: if '.h5' in inputdir: sub = ds_in[slc, slices[1], slices[2]] else: sub = io.imread(files[slc])[slices[1], slices[2]] img_ds = resize(sub, (sub.shape[0] / ds_factor, sub.shape[1] / ds_factor)) if '.h5' in outputdir: ds_out[slc, :, :] = img_ds else: imsave(outpaths[slc], img_ds) # downsample_image(outpaths[slc], sub, ds_factor) try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): pass
def mergeblocks( h5paths_in, blockoffset=[0, 0, 0], blocksize=[], margin=[0, 0, 0], fullsize=[], is_labelimage=False, relabel=False, neighbourmerge=False, save_fwmap=False, blockreduce=[], func='np.amax', datatype='', usempi=False, h5path_out='', save_steps=False, protective=False, ): """Merge blocks of data into a single hdf5 file.""" # prepare mpi mpi_info = utils.get_mpi_info(usempi) series = np.array(range(0, len(h5paths_in)), dtype=int) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] # TODO: save_steps # check output paths outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5paths_in[0], comm=mpi_info['comm']) try: ndim = ds_in.ndim except AttributeError: ndim = len(ds_in.dims) # get the size of the outputfile # TODO: option to derive fullsize from dset_names? if blockreduce: datasize = np.subtract(fullsize, blockoffset) outsize = [ int(np.ceil(d / np.float(b))) for d, b in zip(datasize, blockreduce) ] elsize = [e * b for e, b in zip(elsize, blockreduce)] else: # FIXME: 'zyx(c)' stack assumed outsize = np.subtract(fullsize, blockoffset) if ndim == 4: outsize = list(outsize) + [ds_in.shape[3]] # TODO: flexible insert datatype = datatype or ds_in.dtype chunks = ds_in.chunks or None h5file_in.close() # open data for writing h5file_out, ds_out = utils.h5_write(data=None, shape=outsize, dtype=datatype, h5path_full=h5path_out, chunks=chunks, element_size_um=elsize, axislabels=axlab, comm=mpi_info['comm']) # merge the datasets maxlabel = 0 for i in series: h5path_in = h5paths_in[i] try: maxlabel = process_block(h5path_in, ndim, blockreduce, func, blockoffset, blocksize, margin, fullsize, ds_out, is_labelimage, relabel, neighbourmerge, save_fwmap, maxlabel, usempi, mpi_info) print('processed block {:03d}: {}'.format(i, h5path_in)) except Exception as e: print('failed block {:03d}: {}'.format(i, h5path_in)) print(e) # close and return try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def series2stack( inputdir, regex='*.tif', element_size_um=[None, None, None], outlayout='zyx', datatype='', chunksize=[20, 20, 20], dataslices=None, usempi=False, outputformats=['.h5'], outputpath='', save_steps=False, protective=False, ): """"Convert a directory of tifs to an hdf5 stack.""" # Check if any output paths already exist. if '.h5' in outputformats: outpaths = {'out': outputpath} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # Get the list of input filepaths. files = sorted(glob.glob(os.path.join(inputdir, regex))) # Get some metadata from the inputfiles zyxdims, datatype, element_size_um = get_metadata(files, datatype, outlayout, element_size_um) # (plane, row, column) indexing to outlayout (where prc -> zyx). in2out = ['zyx'.index(o) for o in outlayout] # Get the properties of the output dataset. slices = utils.get_slice_objects_prc(dataslices, zyxdims) # prc-order files = files[slices[0]] datashape_out_prc = (len(files), len(range(*slices[1].indices(slices[1].stop))), len(range(*slices[2].indices(slices[2].stop)))) datashape_out = [datashape_out_prc[i] for i in in2out] # Reshape the file list into a list of blockwise file lists. scs = chunksize[outlayout.index('z')] # chunksize slice dimension files_blocks = zip(* [iter(files)] * scs) rem = len(files) % scs if rem: files_blocks += [tuple(files[-rem:])] # Get slice objects for every output block. slices_out_prc = [[slice(bnr * scs, bnr * scs + scs), slice(0, datashape_out_prc[1]), slice(0, datashape_out_prc[2])] for bnr in range(0, len(files_blocks))] slices_out = [[sliceset_prc[i] for i in in2out] for sliceset_prc in slices_out_prc] # Prepare for processing with MPI. mpi_info = utils.get_mpi_info(usempi) series = np.array(range(0, len(files_blocks)), dtype=int) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] # Open the outputfile for writing and create the dataset or output array. if '.h5' in outputformats: h5file_out, ds_out = utils.h5_write(None, datashape_out, datatype, outputpath, element_size_um=element_size_um, axislabels=outlayout, chunks=tuple(chunksize), comm=mpi_info['comm']) outdir = os.path.dirname(outputpath.split('.h5')[0]) else: ds_out = None outdir = outputpath # Write blocks of 2D images to the outputfile(s). for blocknr in series: if '.h5' in outputformats: ds_out = process_block(files_blocks[blocknr], ds_out, slices, slices_out[blocknr], in2out, outputformats, outdir) else: process_slices(files_blocks[blocknr], slices, slices_out[blocknr], outputformats, outdir, datatype) # Close the h5 files or return the output array. try: h5file_out.close() except (ValueError, AttributeError): return ds_out except UnboundLocalError: pass
def separate_sheaths( h5path_in, h5path_lmm='', h5path_wsmask='', h5path_mask='', h5path_mmm='', MAdilation=0, h5path_dist='', sigmoidweighting=0, margin=50, h5path_out='', save_steps=False, protective=False, ): """Separate the myelin compartment into individual myelin sheaths.""" # check output paths outpaths = { 'out': h5path_out, 'wsmask': '', 'madil{:02d}'.format(MAdilation): '', 'distance_simple': '', 'sheaths_simple': '', 'distance_sigmod': '', } root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) ds_mma = ds_in[:] != 0 # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # load/calculate a mask to constrain the watershed in mask_mmregion = get_wsmask(outpaths, ds_mma, h5path_wsmask, h5path_mask, h5path_mmm, MAdilation, elsize, axlab) elsize_abs = np.absolute(elsize) seeds = grey_dilation(ds_in, size=(3, 3, 3)) # load/calculate the distance transform if h5path_dist: h5file_dist, ds_dist, _, _ = utils.h5_load(h5path_dist) else: if sigmoidweighting: if h5path_lmm: ds_lmm = utils.h5_load(h5path_lmm, load_data=True)[0] else: ds_dist = distance_transform_edt(~ds_mma, sampling=elsize_abs) ds_dist = img_as_float(ds_dist) utils.save_step(outpaths, 'distance_simple', ds_dist, elsize, axlab) ds_lmm = watershed(ds_dist, seeds, mask=mask_mmregion) utils.save_step(outpaths, 'sheaths_simple', ds_lmm, elsize, axlab) ds_dist, _ = distance_transform_sw(ds_in, ds_lmm, elsize_abs, sigmoidweighting, margin) utils.save_step(outpaths, 'distance_sigmod', ds_dist, elsize, axlab) ds_out[:] = watershed(ds_dist, seeds, mask=mask_mmregion) # TODO: save median widths mw else: ds_dist = distance_transform_edt(~ds_mma, sampling=elsize_abs) utils.save_step(outpaths, 'distance_simple', ds_dist, elsize, axlab) ds_out[:] = watershed(ds_dist, seeds, mask=mask_mmregion) # close and return h5file_in.close() if h5path_dist: h5file_dist.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
y[l] = True y[0] = False np.save(gtpath, y) # In[11]: # map the groundtruth labels to a volume h5file_in, ds_in, elsize, axlab = utils.h5_load(labelpath) h5path_out = os.path.join(datadir, '{}_gt.h5'.format(dataset)) h5file_out, ds_out = utils.h5_write(None, ds_in.shape, 'uint8', os.path.join(h5path_out, 'class0'), element_size_um=elsize, axislabels=axlab) ds_out[:] = ~y[ds_in[:]] h5file_out.close() h5file_out, ds_out = utils.h5_write(None, ds_in.shape, 'uint8', os.path.join(h5path_out, 'class1'), element_size_um=elsize, axislabels=axlab) ds_out[:] = y[ds_in[:]] h5file_out.close() h5file_in.close() # convert to nifti niipath_out = os.path.join(datadir, '{}_gt_class0.nii.gz'.format(dataset))
def CC_3D( h5path_in, h5path_mask='', min_size_maskMM=0, min_area=0, h5path_out='', protective=False, ): """Label connected components in a 3D stack.""" # check output path if '.h5' in h5path_out: status, info = utils.h5_check(h5path_out, protective) print(info) if status == "CANCELLED": return # open data for reading h5file_mm, ds_mm, elsize, axlab = utils.h5_load(h5path_in) if h5path_mask: h5file_md, ds_md, _, _ = utils.h5_load(h5path_mask) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_mm.shape, 'uint32', h5path_out, element_size_um=elsize, axislabels=axlab) # 3D labeling with label size constraints # NOTE: could save memory here by applying the constraints to input before # if h5path_mask: # mask = np.logical_or(binary_dilation(ds_mm[:]), ~ds_md[:]) # else: # mask = binary_dilation(ds_mm[:]) if min_size_maskMM: mask = label(ds_mm[:], return_num=False, connectivity=None) remove_small_objects(mask, min_size_maskMM, in_place=True) mask = ~mask.astype('bool') else: mask = ~ds_mm[:] labels = label(mask, return_num=False, connectivity=None) # shuffle_labels = True # if shuffle_labels: # fw = utils.shuffle_labels(labels) # labels = fw[labels] if min_area: remove_small_objects(labels, min_area, in_place=True) # remove the largest label (assumed unmyelinated axon compartment) rp = regionprops(labels) areas = [prop.area for prop in rp] labs = [prop.label for prop in rp] llab = labs[np.argmax(areas)] labels[labels == llab] = 0 labels = relabel_sequential(labels)[0] print('maxlabel: {}'.format(np.amax(labels))) ds_out[:] = labels # close and return try: h5file_mm.close() h5file_out.close() if h5path_mask: h5file_md.close() except (ValueError, AttributeError): return labels
def remap_labels( inputfile, delete_labels=[], delete_files=[], except_files=[], merge_labels=[], merge_files=[], split_labels=[], split_files=[], aux_labelvolume='', min_labelsize=0, min_segmentsize=0, keep_only_largest=False, conncomp=False, nifti_output=False, nifti_transpose=False, h5path_out='', save_steps=False, protective=False, ): """Delete/split/merge/... labels in a labelvolume.""" # check output paths outpaths = {'out': h5path_out, 'deleted': '', 'split': '', 'merged': ''} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.load(inputfile) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) ds_out[:] = ds_in[:] # filter labels on size root = os.path.splitext(h5file_out.filename)[0] ls_small = utils.filter_on_size(ds_out[:], min_labelsize, False, save_steps, root, ds_out.name[1:], outpaths, elsize, axlab)[1] delete_labels = set(delete_labels) | ls_small # delete labels delete_labelsets(ds_out, delete_labels, delete_files, except_files) if save_steps: # FIXME! save_diff(ds_in[:], ds_out[:], 'deleted', outpaths, elsize, axlab) # split labels split_labelsets(ds_out, split_labels, split_files, aux_labelvolume, conncomp) if save_steps: save_diff(ds_in[:], ds_out[:], 'split', outpaths, elsize, axlab) # merge labels merge_labelsets(ds_out, merge_labels, merge_files) if save_steps: save_diff(ds_in[:], ds_out[:], 'merged', outpaths, elsize, axlab) # remove small, non-contiguous segments of labels if min_segmentsize or keep_only_largest: filter_segments(ds_out[:], min_segmentsize, keep_only_largest) if nifti_output: if nifti_transpose: ds_out[:] = np.transpose(ds_out[:]) elsize = elsize[::-1] axlab = axlab[::-1] fpath = '{}_{}.nii.gz'.format(root, ds_out.name[1:]) utils.write_to_nifti(fpath, ds_out[:], elsize) # close and return h5file_in.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def CC_2D( h5path_in, h5path_mask='', slicedim=0, usempi=False, h5path_out='', protective=False, ): """Label connected components in all slices.""" # check output path if '.h5' in h5path_out: status, info = utils.h5_check(h5path_out, protective) print(info) if status == "CANCELLED": return # open data for reading h5file_mm, ds_mm, elsize, axlab = utils.h5_load(h5path_in) if h5path_mask: h5file_md, ds_md, _, _ = utils.h5_load(h5path_mask) # prepare mpi # TODO: could allow selection of slices/subset here mpi_info = utils.get_mpi_info(usempi) n_slices = ds_mm.shape[slicedim] series = np.array(range(0, n_slices), dtype=int) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_mm.shape, 'uint32', h5path_out, element_size_um=elsize, axislabels=axlab, comm=mpi_info['comm']) # slicewise labeling maxlabel = 0 for i in series: slcMM = utils.get_slice(ds_mm, i, slicedim, 'bool') if h5path_mask: slcMD = utils.get_slice(ds_md, i, slicedim, 'bool') labels, num = label(np.logical_and(~slcMM, slcMD), return_num=True) else: labels, num = label(~slcMM, return_num=True) print("found %d labels in slice %d" % (num, i)) if mpi_info['enabled']: # NOTE: assumed max number of labels in slice is 10000 labels[~slcMM] += 10000 * i if i == n_slices - 1: maxlabel = np.amax(labels) else: labels[~slcMM] += maxlabel maxlabel += num if slicedim == 0: ds_out[i, :, :] = labels elif slicedim == 1: ds_out[:, i, :] = labels elif slicedim == 2: ds_out[:, :, i] = labels # save the maximum labelvalue in the dataset print("found %d labels" % (maxlabel)) if mpi_info['rank'] == mpi_info['size'] - 1: root = h5path_out.split('.h5')[0] fpath = root + '.npy' np.save(fpath, np.array([maxlabel])) # close and return try: h5file_mm.close() h5file_out.close() if h5path_mask: h5file_md.close() except (ValueError, AttributeError): return ds_out
def agglo_from_labelmask( h5path_in, h5path_lv='', ratio_threshold=0, h5path_out='', save_steps=False, protective=False, ): """Apply mapping of labelsets to a labelvolume.""" # check output paths outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) h5file_lv, ds_lv, _, _ = utils.h5_load(h5path_lv) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) ulabels = np.unique(ds_in) maxlabel = np.amax(ulabels) print("number of labels in watershed: {:d}".format(maxlabel)) fwmap = np.zeros(maxlabel + 1, dtype='i') areas_ws = np.bincount(ds_in.ravel()) labelsets = {} rp_lw = regionprops(ds_lv, ds_in) for prop in rp_lw: maskedregion = prop.intensity_image[prop.image] counts = np.bincount(maskedregion) svoxs_in_label = [l for sl in np.argwhere(counts) for l in sl] ratios_svox_in_label = [float(counts[svox]) / float(areas_ws[svox]) for svox in svoxs_in_label] fwmask = np.greater(ratios_svox_in_label, ratio_threshold) labelset = np.array(svoxs_in_label)[fwmask] labelsets[prop.label] = set(labelset) - set([0]) basepath = h5path_in.split('.h5/')[0] utils.write_labelsets(labelsets, basepath + "_svoxsets", filetypes=['pickle']) ds_out[:] = utils.forward_map(np.array(fwmap), ds_in, labelsets) # close and return h5file_in.close() h5file_lv.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out