def convert_dm3( filepaths, outdir='', save_steps=False, protective=False, ): """Convert dm3 files to other image formats.""" # check output paths outpaths = {'out': outdir} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return if not os.path.exists(outdir): os.makedirs(savedir) for filepath in filepaths: filename = os.path.split(filepath)[1] fileref = os.path.splitext(filename)[0] im, dm3f = read_dm3_as_im(filepath) if dumptags: dm3f.dumpTags(outdir) if '.tif' in outexts: outfilepath = os.path.join(outdir, fileref + '.tif') im.save(outfilepath)
def downsample_blockwise( h5path_in, blockreduce=[3, 3, 3], func='np.amax', dataslices=None, h5path_out='', save_steps=False, protective=False, ): """Downsample volume by blockwise reduction.""" # Check if any output paths already exist. outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # Open the inputfile for reading. # TODO: option to get the input data passed h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # Get the matrix size and resolution of the outputdata. outsize, elsize = get_new_sizes(func, blockreduce, ds_in.shape, elsize) # Open the outputfile for writing and create the dataset or output array. h5file_out, ds_out = utils.h5_write(None, outsize, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # Get the slice objects for the input data. slices = utils.get_slice_objects_prc(dataslices, ds_in.shape) # Reformat the data to the outputsize. if func == 'expand': out = ds_in[slices[0], ...] for axis in range(0, ds_out.ndim): out = np.repeat(out, blockreduce[axis], axis=axis) ds_out[slices[0], ...] = out else: """ TODO: flexible mapping from in to out now: the reduction factor of the first axis must be 1; the extent of the remaining axes must be full """ ds_out[slices[0], ...] = block_reduce(ds_in[slices[0], ...], block_size=tuple(blockreduce), func=eval(func)) # Close the h5 files or return the output array. try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def map_labels( h5path_in, h5path_out='', save_steps=False, protective=False, ): """Map groups of labels to a single label.""" # check output path outpaths = {'out': h5path_out, 'stitched': ''} root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # load the pickled set of neighbours lsroot = h5path_out.split('.h5')[0] lspath = '{}_{}.pickle'.format(lsroot, ds_out.name[1:]) with open(lspath, "r") as f: labelsets = pickle.load(f) # apply forward map ulabels = np.unique(ds_in[:]) maxlabel = np.amax(ulabels) fw = [l if l in ulabels else 0 for l in range(0, maxlabel + 1)] labels = utils.forward_map(np.array(fw), ds_in[:], labelsets) if save_steps: fw = np.zeros(maxlabel + 1, dtype='i') MAlabels = utils.forward_map(np.array(fw), ds_in[:], labelsets) utils.save_step(outpaths, 'stitched', MAlabels, elsize, axlab) ds_out[:, :, :] = labels # close and return try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def filter_NoR( h5path_in, h5path_2D, h5path_out='', save_steps=False, protective=False, ): """Filter nodes of ranvier.""" # check output paths outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) h5file_2D, ds_2D, _, _ = utils.h5_load(h5path_2D) # open data for writing h5file_out, ds_out = utils.h5_write(ds_in, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) labelsets = {i: set(np.unique(ds_2D[i, :, :])) for i in range(ds_2D.shape[0])} ulabels = np.unique(ds_in) m = {l: np.array([True if l in lsv else False for _, lsv in labelsets.items()]) for l in ulabels} rp = regionprops(ds_in) for prop in rp: z, y, x, Z, Y, X = tuple(prop.bbox) mask = prop.image mask[m[prop.label][z:Z], :, :] = 0 ds_out[z:Z, y:Y, x:X][mask] = 0 # close and return h5file_in.close() h5file_2D.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def CC_clf_train( clfpath, scalerpath, groundtruthpath, basename, map_propnames, save_steps=False, protective=False, ): """Label connected components in a 3D stack.""" # check output paths outpaths = {'out': clfpath, 'scaler': scalerpath} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # Load the ground truth labels. y_train = np.load(groundtruthpath) y_train = y_train[1:] # not the background label # Load the training data. X_train = np.zeros([y_train.shape[0], len(map_propnames) - 1]) for i, propname in enumerate(map_propnames[1:]): nppath = '{}_{}.npy'.format(basename, propname) X_train[:, i] = np.load(nppath)[1:] # not the background label # Scale the training data. scaler = preprocessing.MaxAbsScaler() X_train_scaled = scaler.fit_transform(X_train) # Fit the support vector classifier. clf = svm.SVC() clf.fit(X_train_scaled, y_train) # Write the classifier and scaler. joblib.dump(clf, clfpath) joblib.dump(scaler, scalerpath)
def prob2mask( h5path_in, dataslices=None, lower_threshold=0, upper_threshold=1, size=0, dilation=0, h5path_mask=None, go2D=False, h5path_out='', save_steps=False, protective=False, ): """Create thresholded hard segmentation.""" # Check if any output paths already exist. outpaths = {'out': h5path_out, 'raw': '', 'mito': '', 'dil': ''} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return root, ds_main = outpaths['out'].split('.h5') grpname = ds_main + "_steps" for dsname, outpath in outpaths.items(): if ((dsname != 'out') and save_steps and (not outpath)): outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) # Load the (sliced) data. h5file_in, ds_in, es, al = utils.load(h5path_in) data, _, _, slices_out = utils.load_dataset(ds_in, dataslices=dataslices) inmask = utils.load(h5path_mask, load_data=True, dtype='bool', dataslices=dataslices)[0] if data.ndim == 4: # FIXME: generalize # data = np.squeeze(data) data = data[:, :, :, 0] outshape = ds_in.shape[:3] es = es[:3] al = al[:3] slices_out = slices_out[:3] else: outshape = ds_in.shape # Open the outputfile(s) for writing # and create the dataset(s) or output array(s). h5file_out, ds_out = utils.h5_write(None, outshape, 'uint8', outpaths['out'], element_size_um=es, axislabels=al) if save_steps: stepnames = ['raw', 'mito', 'dil'] steps = [ utils.h5_write(None, outshape, 'uint8', outpaths[out], element_size_um=es, axislabels=al) for out in stepnames ] dss = [np.zeros_like(data)] * 3 else: stepnames, steps, dss = [], [], [] # Threshold (and dilate and filter) the data. if go2D: # process slicewise mask = np.zeros_like(data) for i, slc in enumerate(data): print('processing slice: {}'.format(i)) smf, sdss = process_slice(slc, lower_threshold, upper_threshold, size, dilation, disk, save_steps) mask[i, :, :] = smf for ds, sds in zip(dss, sdss): ds[i, :, :] = sds else: # process full input mask, dss = process_slice(data, lower_threshold, upper_threshold, size, dilation, ball, save_steps) # Apply additional mask. if inmask is not None: mask[~inmask] = False # Write the mask(s) to file. if '.h5' in h5path_out: utils.write_to_h5ds(ds_out, mask.astype('uint8'), slices_out) for step, ds in zip(steps, dss): utils.write_to_h5ds(step[1], ds.astype('uint8'), slices_out) else: # write as tif/png/jpg series root, ext = os.path.splitext(outpaths['out']) utils.write_to_img(root, mask.astype('uint8'), al, 5, ext, 0) for stepname, ds in zip(stepnames, dss): root, ext = os.path.splitext(outpaths[stepname]) utils.write_to_img(root, ds.astype('uint8'), al, 5, ext, 0) # Close the h5 file(s) or return the output array(s). try: h5file_in.close() h5file_out.close() for step in steps: step[0].close() except (ValueError, AttributeError): return mask, dss
def CC_clf_test( clfpath, scalerpath, basename, map_propnames, thr_mi=0, thr_area=0, h5path_in='', h5path_out='', h5path_mask='', save_steps=False, protective=False, ): """Label connected components in a 3D stack.""" # check output paths predpath = '{}_{}.npy'.format(basename, 'pred') predpath_thr = '{}_{}.npy'.format(basename, 'pred_thr') outpaths = {'pred': predpath} if h5path_out: outpaths['out'] = h5path_out if h5path_mask: outpaths['maskMA'] = h5path_mask elif save_steps: root, ds_main = outpaths['out'].split('.h5') grpname = ds_main + "_steps" outpaths['maskMA'] = os.path.join(root + '.h5' + grpname, 'maskMA') if thr_mi or thr_area: outpaths['pred_thr'] = predpath_thr status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # Load the scaler and classifier. scaler = joblib.load(scalerpath) clf = joblib.load(clfpath) # Scale the test data. tmppath = '{}_{}.npy'.format(basename, map_propnames[0]) tmp = np.load(tmppath)[1:] X_test = np.zeros([tmp.shape[0], len(map_propnames) - 1]) for i, propname in enumerate(map_propnames[1:]): nppath = '{}_{}.npy'.format(basename, propname) X_test[:, i] = np.load(nppath)[1:] X_test_scaled = scaler.transform(X_test) # Predict the test data. pred = clf.predict(X_test_scaled) fw = np.insert(pred, 0, [False]) # reinsert the background label # Save the results. np.save(predpath, fw) # Apply additional criteria. fw = apply_additional_criteria(basename, fw, thr_mi, thr_area) # Map the predicted labels to a volume. if (h5path_in and h5path_out): map_to_volume(fw, h5path_in, h5path_out, h5path_mask)
def watershed_ics( h5path_in, masks=[], h5path_seeds='', seed_size=64, lower_threshold=None, upper_threshold=None, invert=False, h5path_out='', save_steps=False, protective=False, ): """Perform watershed on the intracellular space compartments.""" # check output paths outpaths = {'out': h5path_out, 'seeds': h5path_seeds, 'mask': ''} root, ds_main = outpaths['out'].split('.h5') for dsname, _ in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape[:3], 'uint32', h5path_out, element_size_um=elsize, axislabels=axlab) # load/generate the seeds if h5path_seeds: h5file_sds, ds_sds, _, _ = utils.h5_load(h5path_seeds) else: h5file_sds, ds_sds = utils.h5_write(None, ds_in.shape[:3], 'uint32', outpaths['seeds'], element_size_um=elsize, axislabels=axlab) lower_threshold = lower_threshold or np.amin(ds_in[:]) - 1 upper_threshold = upper_threshold or np.amax(ds_in[:]) + 1 ds_sds[:] = np.logical_and(ds_in[:] > lower_threshold, ds_in[:] <= upper_threshold) ds_sds[:], _ = label(ds_sds[:]) ds_sds[:] = remove_small_objects(ds_sds[:], min_size=seed_size) ds_sds[:] = relabel_sequential(ds_sds[:])[0] """ NOTE: numpy/scipy/skimage inplace is not written when using hdf5 Therefore, we cannot use: np.logical_and(ds_in[:] > lower_threshold, ds_in[:] <= upper_threshold, ds_sds[:]) num = label(ds_sds[:], output=ds_sds[:]) remove_small_objects(ds_sds[:], min_size=seed_size, in_place=True) """ # determine the mask mask = np.ones(ds_in.shape[:3], dtype='bool') mask = utils.string_masks(masks, mask) h5file_mask, ds_mask = utils.h5_write(None, ds_in.shape[:3], 'uint8', outpaths['mask'], element_size_um=elsize, axislabels=axlab) ds_mask[:] = mask # ds_mask[:].fill(1) # ds_mask[:] = utils.string_masks(masks, ds_mask[:]) # perform the watershed if invert: ds_out[:] = watershed(-ds_in[:], ds_sds[:], mask=ds_mask[:]) else: ds_out[:] = watershed(ds_in[:], ds_sds[:], mask=ds_mask[:]) # close and return h5file_in.close() try: h5file_out.close() h5file_sds.close() h5file_mask.close() except (ValueError, AttributeError): return ds_out, ds_sds, ds_mask
def filter_labels( h5path_in, h5path_mm='', min_labelsize=0, close=None, relabel_from=0, h5path_out='', save_steps=False, protective=False, ): """Map groups of labels to a single label.""" # check output path outpaths = {'out': h5path_out, 'closed': ''} root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) labels = ds_in[:] # if min_labelsize: # remove_small_objects(labels, min_size=min_labelsize, in_place=True) if close is not None: labels = close_labels(labels, close) if h5path_mm: print('removing voxels in mask') h5file_mm, ds_mm, _, _ = utils.h5_load(h5path_mm) labels[ds_mm[:].astype('bool')] = 0 h5file_mm.close() if save_steps: utils.save_step(outpaths, 'closed', labels, elsize, axlab) if min_labelsize: print('removing small labels') remove_small_objects(labels, min_size=min_labelsize, in_place=True) # if save_steps: # utils.save_step(outpaths, 'small', smalllabels, elsize, axlab) if relabel_from > 1: print('relabeling from {}'.format(relabel_from)) labels = relabel_sequential(labels, relabel_from)[0] # TODO: save mapping? ds_out[:, :, :] = labels # close and return try: h5file_in.close() h5file_out.close() except (ValueError, AttributeError): return ds_out
def series2stack( inputdir, regex='*.tif', element_size_um=[None, None, None], outlayout='zyx', datatype='', chunksize=[20, 20, 20], dataslices=None, usempi=False, outputformats=['.h5'], outputpath='', save_steps=False, protective=False, ): """"Convert a directory of tifs to an hdf5 stack.""" # Check if any output paths already exist. if '.h5' in outputformats: outpaths = {'out': outputpath} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # Get the list of input filepaths. files = sorted(glob.glob(os.path.join(inputdir, regex))) # Get some metadata from the inputfiles zyxdims, datatype, element_size_um = get_metadata(files, datatype, outlayout, element_size_um) # (plane, row, column) indexing to outlayout (where prc -> zyx). in2out = ['zyx'.index(o) for o in outlayout] # Get the properties of the output dataset. slices = utils.get_slice_objects_prc(dataslices, zyxdims) # prc-order files = files[slices[0]] datashape_out_prc = (len(files), len(range(*slices[1].indices(slices[1].stop))), len(range(*slices[2].indices(slices[2].stop)))) datashape_out = [datashape_out_prc[i] for i in in2out] # Reshape the file list into a list of blockwise file lists. scs = chunksize[outlayout.index('z')] # chunksize slice dimension files_blocks = zip(* [iter(files)] * scs) rem = len(files) % scs if rem: files_blocks += [tuple(files[-rem:])] # Get slice objects for every output block. slices_out_prc = [[slice(bnr * scs, bnr * scs + scs), slice(0, datashape_out_prc[1]), slice(0, datashape_out_prc[2])] for bnr in range(0, len(files_blocks))] slices_out = [[sliceset_prc[i] for i in in2out] for sliceset_prc in slices_out_prc] # Prepare for processing with MPI. mpi_info = utils.get_mpi_info(usempi) series = np.array(range(0, len(files_blocks)), dtype=int) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] # Open the outputfile for writing and create the dataset or output array. if '.h5' in outputformats: h5file_out, ds_out = utils.h5_write(None, datashape_out, datatype, outputpath, element_size_um=element_size_um, axislabels=outlayout, chunks=tuple(chunksize), comm=mpi_info['comm']) outdir = os.path.dirname(outputpath.split('.h5')[0]) else: ds_out = None outdir = outputpath # Write blocks of 2D images to the outputfile(s). for blocknr in series: if '.h5' in outputformats: ds_out = process_block(files_blocks[blocknr], ds_out, slices, slices_out[blocknr], in2out, outputformats, outdir) else: process_slices(files_blocks[blocknr], slices, slices_out[blocknr], outputformats, outdir, datatype) # Close the h5 files or return the output array. try: h5file_out.close() except (ValueError, AttributeError): return ds_out except UnboundLocalError: pass
def mergeblocks( h5paths_in, blockoffset=[0, 0, 0], blocksize=[], margin=[0, 0, 0], fullsize=[], is_labelimage=False, relabel=False, neighbourmerge=False, save_fwmap=False, blockreduce=[], func='np.amax', datatype='', usempi=False, h5path_out='', save_steps=False, protective=False, ): """Merge blocks of data into a single hdf5 file.""" # prepare mpi mpi_info = utils.get_mpi_info(usempi) series = np.array(range(0, len(h5paths_in)), dtype=int) if mpi_info['enabled']: series = utils.scatter_series(mpi_info, series)[0] # TODO: save_steps # check output paths outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5paths_in[0], comm=mpi_info['comm']) try: ndim = ds_in.ndim except AttributeError: ndim = len(ds_in.dims) # get the size of the outputfile # TODO: option to derive fullsize from dset_names? if blockreduce: datasize = np.subtract(fullsize, blockoffset) outsize = [ int(np.ceil(d / np.float(b))) for d, b in zip(datasize, blockreduce) ] elsize = [e * b for e, b in zip(elsize, blockreduce)] else: # FIXME: 'zyx(c)' stack assumed outsize = np.subtract(fullsize, blockoffset) if ndim == 4: outsize = list(outsize) + [ds_in.shape[3]] # TODO: flexible insert datatype = datatype or ds_in.dtype chunks = ds_in.chunks or None h5file_in.close() # open data for writing h5file_out, ds_out = utils.h5_write(data=None, shape=outsize, dtype=datatype, h5path_full=h5path_out, chunks=chunks, element_size_um=elsize, axislabels=axlab, comm=mpi_info['comm']) # merge the datasets maxlabel = 0 for i in series: h5path_in = h5paths_in[i] try: maxlabel = process_block(h5path_in, ndim, blockreduce, func, blockoffset, blocksize, margin, fullsize, ds_out, is_labelimage, relabel, neighbourmerge, save_fwmap, maxlabel, usempi, mpi_info) print('processed block {:03d}: {}'.format(i, h5path_in)) except Exception as e: print('failed block {:03d}: {}'.format(i, h5path_in)) print(e) # close and return try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def separate_sheaths( h5path_in, h5path_lmm='', h5path_wsmask='', h5path_mask='', h5path_mmm='', MAdilation=0, h5path_dist='', sigmoidweighting=0, margin=50, h5path_out='', save_steps=False, protective=False, ): """Separate the myelin compartment into individual myelin sheaths.""" # check output paths outpaths = { 'out': h5path_out, 'wsmask': '', 'madil{:02d}'.format(MAdilation): '', 'distance_simple': '', 'sheaths_simple': '', 'distance_sigmod': '', } root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) ds_mma = ds_in[:] != 0 # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # load/calculate a mask to constrain the watershed in mask_mmregion = get_wsmask(outpaths, ds_mma, h5path_wsmask, h5path_mask, h5path_mmm, MAdilation, elsize, axlab) elsize_abs = np.absolute(elsize) seeds = grey_dilation(ds_in, size=(3, 3, 3)) # load/calculate the distance transform if h5path_dist: h5file_dist, ds_dist, _, _ = utils.h5_load(h5path_dist) else: if sigmoidweighting: if h5path_lmm: ds_lmm = utils.h5_load(h5path_lmm, load_data=True)[0] else: ds_dist = distance_transform_edt(~ds_mma, sampling=elsize_abs) ds_dist = img_as_float(ds_dist) utils.save_step(outpaths, 'distance_simple', ds_dist, elsize, axlab) ds_lmm = watershed(ds_dist, seeds, mask=mask_mmregion) utils.save_step(outpaths, 'sheaths_simple', ds_lmm, elsize, axlab) ds_dist, _ = distance_transform_sw(ds_in, ds_lmm, elsize_abs, sigmoidweighting, margin) utils.save_step(outpaths, 'distance_sigmod', ds_dist, elsize, axlab) ds_out[:] = watershed(ds_dist, seeds, mask=mask_mmregion) # TODO: save median widths mw else: ds_dist = distance_transform_edt(~ds_mma, sampling=elsize_abs) utils.save_step(outpaths, 'distance_simple', ds_dist, elsize, axlab) ds_out[:] = watershed(ds_dist, seeds, mask=mask_mmregion) # close and return h5file_in.close() if h5path_dist: h5file_dist.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def remap_labels( inputfile, delete_labels=[], delete_files=[], except_files=[], merge_labels=[], merge_files=[], split_labels=[], split_files=[], aux_labelvolume='', min_labelsize=0, min_segmentsize=0, keep_only_largest=False, conncomp=False, nifti_output=False, nifti_transpose=False, h5path_out='', save_steps=False, protective=False, ): """Delete/split/merge/... labels in a labelvolume.""" # check output paths outpaths = {'out': h5path_out, 'deleted': '', 'split': '', 'merged': ''} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.load(inputfile) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) ds_out[:] = ds_in[:] # filter labels on size root = os.path.splitext(h5file_out.filename)[0] ls_small = utils.filter_on_size(ds_out[:], min_labelsize, False, save_steps, root, ds_out.name[1:], outpaths, elsize, axlab)[1] delete_labels = set(delete_labels) | ls_small # delete labels delete_labelsets(ds_out, delete_labels, delete_files, except_files) if save_steps: # FIXME! save_diff(ds_in[:], ds_out[:], 'deleted', outpaths, elsize, axlab) # split labels split_labelsets(ds_out, split_labels, split_files, aux_labelvolume, conncomp) if save_steps: save_diff(ds_in[:], ds_out[:], 'split', outpaths, elsize, axlab) # merge labels merge_labelsets(ds_out, merge_labels, merge_files) if save_steps: save_diff(ds_in[:], ds_out[:], 'merged', outpaths, elsize, axlab) # remove small, non-contiguous segments of labels if min_segmentsize or keep_only_largest: filter_segments(ds_out[:], min_segmentsize, keep_only_largest) if nifti_output: if nifti_transpose: ds_out[:] = np.transpose(ds_out[:]) elsize = elsize[::-1] axlab = axlab[::-1] fpath = '{}_{}.nii.gz'.format(root, ds_out.name[1:]) utils.write_to_nifti(fpath, ds_out[:], elsize) # close and return h5file_in.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def stack2stack( inputfile, outputfile, dset_name='', blockoffset=[], datatype='', uint8conv=False, inlayout='', outlayout='', elsize=[], chunksize=[], additional_outputs=[], nzfills=5, dataslices=None, save_steps=False, protective=False, ): """Convert/select/downscale/transpose/... an hdf5 dataset.""" # output root and exts root, ext = split_filepath(outputfile) outexts = list(set(additional_outputs + [ext])) # Check if any output paths already exist. outpaths = {'out': outputfile, 'addext': (root, additional_outputs)} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return h5file_in, ds_in, h5elsize, h5axlab = utils.h5_load(inputfile) try: ndim = ds_in.ndim except AttributeError: ndim = len(ds_in.dims) # data layout # FIXME: h5axlab not necessarily xyzct! # FIXME: this forces a possibly erroneous outlayout when inlayout is not found inlayout = inlayout or ''.join(h5axlab) or 'zyxct'[0:ndim] outlayout = outlayout or inlayout in2out = [inlayout.index(l) for l in outlayout] # element size elsize = elsize or (h5elsize[in2out] if h5elsize is not None else None) # chunksize if chunksize is not None: chunksize = tuple(chunksize) or ( True if not any(chunksize) else (tuple([ds_in.chunks[i] for i in in2out]) if ds_in.chunks else None ) ) # datatype datatype = datatype or ds_in.dtype if dset_name: _, x, X, y, Y, z, Z = utils.split_filename(dset_name, blockoffset) slices = {'x': [x, X, 1], 'y': [y, Y, 1], 'z': [z, Z, 1]} if ndim > 3: C = ds_in.shape[inlayout.index('c')] slices['c'] = [0, C, 1] sliceslist = [slices[dim] for dim in inlayout] dataslices = [item for sl in sliceslist for item in sl] # get the selected and transformed data # TODO: most memory-efficient solution data = utils.load_dataset(ds_in, elsize, inlayout, outlayout, datatype, dataslices, uint8conv)[0] h5file_in.close() # write the data for ext in outexts: if '.nii' in ext: if data.dtype == 'float16': data = data.astype('float') utils.write_to_nifti(root + '.nii.gz', data, elsize) if '.h5' in ext: utils.h5_write(data, data.shape, data.dtype, outputfile, element_size_um=elsize, axislabels=outlayout, chunks=chunksize) if (('.tif' in ext) | ('.png' in ext) | ('.jpg' in ext)) & (data.ndim < 4): if data.ndim == 2: data = data.atleast_3d() outlayout += 'z' utils.write_to_img(root, data, outlayout, nzfills, ext) return data
def nodes_of_ranvier( h5path_in, min_labelsize=0, remove_small_labels=False, h5path_boundarymask='', merge_methods=['neighbours'], overlap_threshold=20, h5path_data='', h5path_mmm='', searchradius=[100, 30, 30], h5path_out='', save_steps=False, protective=False, ): """Find labels that do not traverse through the volume.""" # check output paths outpaths = {'out': h5path_out, 'largelabels': '', 'smalllabelmask': '', 'boundarymask': '', 'labels_nt': '', 'labels_tv': '', 'filled': '', } root, ds_main = outpaths['out'].split('.h5') for dsname, outpath in outpaths.items(): grpname = ds_main + "_steps" outpaths[dsname] = os.path.join(root + '.h5' + grpname, dsname) status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) labels = ds_in[:] # FIXME: do we make a copy, or use ds_out? # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) # start with the set of all labels ulabels = np.unique(labels) maxlabel = np.amax(ulabels) labelset = set(ulabels) print("number of labels in labelvolume: {}".format(len(labelset))) # get the labelsets that touch the borders sidesmask = get_boundarymask(h5path_boundarymask, 'invdil') ls_bot = set(np.unique(labels[:4, :, :])) ls_top = set(np.unique(labels[-4:, :, :])) ls_sides = set(np.unique(labels[sidesmask])) ls_border = ls_bot | ls_top | ls_sides ls_centre = labelset - ls_border # get the labels that do not touch the border twice ls_bts = (ls_bot ^ ls_top) ^ ls_sides ls_tbs = (ls_top ^ ls_bot) ^ ls_sides ls_sbt = (ls_sides ^ ls_bot) ^ ls_top ls_nt = ls_centre | ls_bts | ls_tbs | ls_sbt # filter labels on size root = os.path.splitext(h5file_out.filename)[0] ls_small = utils.filter_on_size(labels, labelset, min_labelsize, remove_small_labels, save_steps, root, ds_out.name[1:], outpaths, elsize, axlab)[2] labelset -= ls_small ls_nt -= ls_small ls_short = filter_on_heigth(labels, 5) labelset -= ls_short ls_nt -= ls_short ls_tv = labelset - ls_nt print('number of large, long labels: {}'.format(len(labelset))) print('number of large, long in-volume labels: {}'.format(len(ls_nt))) print('number of large, long through-volume labels: {}'.format(len(ls_tv))) # map the large labels that don't traverse the volume fw_nt = np.zeros(maxlabel + 1, dtype='i') for l in ls_nt: fw_nt[l] = l labels_nt = fw_nt[labels] # automated label merge labelsets = {} min_labelsize = 10 labelsets, filled = merge_labels(labels_nt, labelsets, merge_methods, overlap_threshold, h5path_data, h5path_mmm, min_labelsize, searchradius) fw = np.zeros(maxlabel + 1, dtype='i') ds_out[:] = utils.forward_map(np.array(fw), labels, labelsets) if save_steps: utils.save_step(outpaths, 'boundarymask', sidesmask, elsize, axlab) utils.save_step(outpaths, 'labels_nt', labels_nt, elsize, axlab) fw_tv = np.zeros(maxlabel + 1, dtype='i') for l in ls_tv: fw_tv[l] = l labels_tv = fw_tv[labels] utils.save_step(outpaths, 'labels_tv', labels_tv, elsize, axlab) if filled is not None: fw = np.zeros(maxlabel + 1, dtype='i') filled = utils.forward_map(np.array(fw), filled, labelsets) utils.save_step(outpaths, 'filled', filled, elsize, axlab) filestem = '{}_{}_automerged'.format(root, ds_out.name[1:]) utils.write_labelsets(labelsets, filestem, filetypes=['txt', 'pickle']) # close and return h5file_in.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out
def agglo_from_labelmask( h5path_in, h5path_lv='', ratio_threshold=0, h5path_out='', save_steps=False, protective=False, ): """Apply mapping of labelsets to a labelvolume.""" # check output paths outpaths = {'out': h5path_out} status = utils.output_check(outpaths, save_steps, protective) if status == "CANCELLED": return # open data for reading h5file_in, ds_in, elsize, axlab = utils.h5_load(h5path_in) h5file_lv, ds_lv, _, _ = utils.h5_load(h5path_lv) # open data for writing h5file_out, ds_out = utils.h5_write(None, ds_in.shape, ds_in.dtype, h5path_out, element_size_um=elsize, axislabels=axlab) ulabels = np.unique(ds_in) maxlabel = np.amax(ulabels) print("number of labels in watershed: {:d}".format(maxlabel)) fwmap = np.zeros(maxlabel + 1, dtype='i') areas_ws = np.bincount(ds_in.ravel()) labelsets = {} rp_lw = regionprops(ds_lv, ds_in) for prop in rp_lw: maskedregion = prop.intensity_image[prop.image] counts = np.bincount(maskedregion) svoxs_in_label = [l for sl in np.argwhere(counts) for l in sl] ratios_svox_in_label = [float(counts[svox]) / float(areas_ws[svox]) for svox in svoxs_in_label] fwmask = np.greater(ratios_svox_in_label, ratio_threshold) labelset = np.array(svoxs_in_label)[fwmask] labelsets[prop.label] = set(labelset) - set([0]) basepath = h5path_in.split('.h5/')[0] utils.write_labelsets(labelsets, basepath + "_svoxsets", filetypes=['pickle']) ds_out[:] = utils.forward_map(np.array(fwmap), ds_in, labelsets) # close and return h5file_in.close() h5file_lv.close() try: h5file_out.close() except (ValueError, AttributeError): return ds_out