def warping_error( lbl_truth, lbl_proposed, doComps=True, simpleLUT=None, connectivity=1 ): warped, classified, nonSimpleTypes, diff, simpleLUT = binary_warping( lbl_truth, lbl_proposed, return_nonSimple=True, connectivity=connectivity, simpleLUT=simpleLUT ) wrp_err = float(diff) / lbl_truth.size if doComps: lbls, nSplits = nd.measurements.label(classified == nonSimpleTypes['dic']['RESULT_SPLIT'], structure=np.ones((3,3,3),dtype=np.bool)) nonSimpleTypesOut = np.zeros(lbls.shape, dtype=lbls.dtype) nonSimpleTypesOut[lbls > 0] = nonSimpleTypes['dic']['RESULT_SPLIT'] lbls, nMerges = nd.measurements.label(classified == nonSimpleTypes['dic']['RESULT_MERGE'], structure=np.ones((3,3,3),dtype=np.bool)) nonSimpleTypesOut[lbls > 0] = nonSimpleTypes['dic']['RESULT_MERGE'] else: nSplits = (classified == nonSimpleTypes['dic']['RESULT_SPLIT']).sum(dtype=np.int64) nMerges = (classified == nonSimpleTypes['dic']['RESULT_MERGE']).sum(dtype=np.int64) nonSimpleTypesOut = nonSimpleTypes return wrp_err, nSplits, nMerges, nonSimpleTypesOut, simpleLUT
def warping_error( lbl_truth, lbl_proposed, doComps=True, simpleLUT=None, connectivity=1 ): from pyCext import binary_warping warped, classified, nonSimpleTypes, diff, simpleLUT = binary_warping( lbl_truth, lbl_proposed, return_nonSimple=True, connectivity=connectivity, simpleLUT=simpleLUT ) wrp_err = float(diff) / lbl_truth.size if doComps: lbls, nSplits = nd.measurements.label(classified == nonSimpleTypes['dic']['RESULT_SPLIT'], structure=np.ones((3,3,3),dtype=bool)) nonSimpleTypesOut = np.zeros(lbls.shape, dtype=lbls.dtype) nonSimpleTypesOut[lbls > 0] = nonSimpleTypes['dic']['RESULT_SPLIT'] lbls, nMerges = nd.measurements.label(classified == nonSimpleTypes['dic']['RESULT_MERGE'], structure=np.ones((3,3,3),dtype=bool)) nonSimpleTypesOut[lbls > 0] = nonSimpleTypes['dic']['RESULT_MERGE'] else: nSplits = (classified == nonSimpleTypes['dic']['RESULT_SPLIT']).sum(dtype=np.int64) nMerges = (classified == nonSimpleTypes['dic']['RESULT_MERGE']).sum(dtype=np.int64) nonSimpleTypesOut = nonSimpleTypes return wrp_err, nSplits, nMerges, nonSimpleTypesOut, simpleLUT
def watershed_cube(self): writeVerbose = False; #writeVerbose = self.dpWatershedTypes_verbose readVerbose = False; #readVerbose = self.dpWatershedTypes_verbose # load the probability data, allocate as array of volumes instead of 4D ndarray to maintain C-order volumes probs = [None]*self.ntypes; bwseeds = [None]*self.nfg_types if self.srclabels: # this code path is typically not used in favor of the label checker for fully labeled 3d gt components. # but, some ground truth (for example, 2d ECS cases) was only labeled with voxel type, # so this is used to create ground truth components from the voxel types. loadh5 = emLabels.readLabels(srcfile=self.srclabels, chunk=self.chunk.tolist(), offset=self.offset.tolist(), size=self.size.tolist(), data_type='uint16', verbose=writeVerbose) self.datasize = loadh5.datasize; self.chunksize = loadh5.chunksize; self.attrs = loadh5.data_attrs # pre-allocate for srclabels method, labeled areas are set to prob of 1 below for i in range(self.ntypes): probs[i] = np.zeros(self.size, dtype=emProbabilities.PROBS_DTYPE, order='C') if self.TminSrc < 2: # simple method with no "cleaning" for i in range(self.ntypes): probs[i][loadh5.data_cube==i] = 1 else: # optionally "clean" labels by removing small bg and fg components for each foreground type fgbwlabels = np.zeros(self.size, dtype=np.bool) for i in range(self.nfg_types): # background connected components and threshold comps, nlbls = nd.measurements.label(loadh5.data_cube!=i+1) comps, sizes = emLabels.thresholdSizes(comps, minSize=self.TminSrc) # foreground connected components and threshold comps, nlbls = nd.measurements.label(comps==0) comps, sizes = emLabels.thresholdSizes(comps, minSize=self.TminSrc) # keep track of mask for all foreground types bwlabels = (comps > 0); fgbwlabels = np.logical_or(fgbwlabels, bwlabels) probs[i+1][bwlabels] = 1 # set background type as all areas that are not in foreground types after "cleaning" probs[0][np.logical_not(fgbwlabels)] = 1 else: # check if background is in the prob file hdf = h5py.File(self.probfile,'r'); has_bg = self.bg_type in hdf; hdf.close() for i in range(0 if has_bg else 1, self.ntypes): loadh5 = dpLoadh5.readData(srcfile=self.probfile, dataset=self.types[i], chunk=self.chunk.tolist(), offset=self.offset.tolist(), size=self.size.tolist(), data_type=emProbabilities.PROBS_STR_DTYPE, verbose=readVerbose) self.datasize = loadh5.datasize; self.chunksize = loadh5.chunksize; self.attrs = loadh5.data_attrs probs[i] = loadh5.data_cube; del loadh5 # if background was not in hdf5 then create it as 1-sum(fg type probs) if not has_bg: probs[0] = np.ones_like(probs[1]) for i in range(1,self.ntypes): probs[0] -= probs[i] #assert( (probs[0] >= 0).all() ) # comment for speed probs[0][probs[0] < 0] = 0 # rectify # save some of the parameters as attributes self.attrs['types'] = self.types; self.attrs['fg_types'] = self.fg_types self.attrs['fg_types_labels'] = self.fg_types_labels # save connnetivity structure and warping LUT because used on each iteration (for speed) self.bwconn = nd.morphology.generate_binary_structure(dpLoadh5.ND, self.connectivity) self.bwconn2d = self.bwconn[:,:,1]; self.simpleLUT = None # load the warpings if warping mode is enabled warps = None if self.warpfile: warps = [None]*self.nwarps for i in range(self.nwarps): loadh5 = dpLoadh5.readData(srcfile=self.warpfile, dataset=self.warp_datasets[i], chunk=self.chunk.tolist(), offset=self.offset.tolist(), size=self.size.tolist(), verbose=readVerbose) warps[i] = loadh5.data_cube; del loadh5 # xxx - may need to revisit cropping, only intended to be used with warping method. if self.docrop: c = self.cropborder; s = self.size # DO NOT use variables c or s below # optionally apply filters in attempt to fill small background (membrane) probability gaps. if self.close_bg > 0: # create structuring element n = 2*self.close_bg + 1; h = self.close_bg; strel = np.zeros((n,n,n),dtype=np.bool); strel[h,h,h]=1; strel = nd.binary_dilation(strel,iterations=self.close_bg) # xxx - this was the only thing tried here that helped some but didn't work well against the skeletons probs[0] = nd.grey_closing( probs[0], structure=strel ) for i in range(self.nfg_types): probs[i+1] = nd.grey_opening( probs[i+1], structure=strel ) # xxx - this gave worse results #probs[0] = nd.maximum_filter( probs[0], footprint=strel ) # xxx - this had almost no effect #probs[0] = nd.grey_closing( probs[0], structure=strel ) # argmax produces the winner-take-all assignment for each supervoxel. # background type was put first, so voxType of zero is background (membrane). voxType = np.concatenate([x.reshape(x.shape + (1,)) for x in probs], axis=3).argmax(axis=3) # write out the winning type for each voxel # save some params from this watershed run in the attributes d = self.attrs.copy(); d['thresholds'] = self.Ts; d['Tmins'] = self.Tmins data = voxType.astype(emVoxelType.VOXTYPE_DTYPE) if self.docrop: data = data[c[0]:s[0]-c[0],c[1]:s[1]-c[1],c[2]:s[2]-c[2]] emVoxelType.writeVoxType(outfile=self.outlabels, chunk=self.chunk.tolist(), offset=self.offset_crop.tolist(), size=self.size_crop.tolist(), datasize=self.datasize.tolist(), chunksize=self.chunksize.tolist(), verbose=writeVerbose, attrs=d, data=data) # only allow a voxel to be included in the type of component that had max prob for that voxel. # do this by setting the non-winning probabilities to zero. for i in range(self.ntypes): probs[i][voxType != i] = 0; # create a type mask for each foreground type to select only current voxel type (winner-take-all from network) voxTypeSel = [None] * self.nfg_types; voxTypeNotSel = [None] * self.nfg_types for i in range(self.nfg_types): voxTypeSel[i] = (voxType == i+1) # create an inverted version, only used for complete fill not for warping (which requires C-contiguous), # so apply crop here if cropping enabled voxTypeNotSel[i] = np.logical_not(voxTypeSel[i]) if self.docrop: voxTypeNotSel[i] = voxTypeNotSel[i][c[0]:s[0]-c[0],c[1]:s[1]-c[1],c[2]:s[2]-c[2]] # need C-contiguous probabilities for binary_warping. for i in range(self.nfg_types): if not probs[i+1].flags.contiguous or np.isfortran(probs[i+1]): probs[i+1] = np.ascontiguousarray(probs[i+1]) # iteratively apply thresholds, each time only keeping components that have fallen under size Tmin. # at last iteration keep all remaining components. # do this separately for foreground types. for k in range(self.nTmin): for i in range(self.nfg_types): bwseeds[i] = np.zeros(self.size, dtype=np.bool, order='C') for i in range(self.nthresh): if self.dpWatershedTypes_verbose: print('creating supervoxels at threshold = %.8f with Tmin = %d' % (self.Ts[i], self.Tmins[k])) t = time.time() types_labels = [None]*self.nfg_types; types_uclabels = [None]*self.nfg_types; if self.skeletonize: types_sklabels = [None]*self.nfg_types types_nlabels = np.zeros((self.nfg_types,),dtype=np.int64) types_ucnlabels = np.zeros((self.nfg_types,),dtype=np.int64) for j in range(self.nfg_types): # run connected components at this threshold on labels labels, nlabels = nd.measurements.label(probs[j+1] > self.Ts[i], self.bwconn) # merge the current thresholded components with the previous seeds to get current bwlabels bwlabels = np.logical_or(labels, bwseeds[j]) # take the current components under threshold and merge with the seeds for the next iteration if i < self.nthresh-1: labels, sizes = emLabels.thresholdSizes(labels, minSize=-self.Tmins[k]) bwseeds[j] = np.logical_or(labels, bwseeds[j]) # this if/elif switch determines the main method for creating the labels. # xxx - make cropping to be done in more efficient way, particular to avoid filling cropped areas if self.method == 'overlap': # definite advantage to this method over other methods, but cost is about 2-3 times slower. # labels are linked per zslice using precalculated slice to slice warpings based on the probs. labels, nlabels = self.label_overlap(bwlabels, voxTypeSel[j], warps) # xxx - add switches to only optionally export the unconnected labels #uclabels = labels; ucnlabels = nlabels; # crop right after the labels are created and stay uncropped from here. # xxx - labels will be wrong unless method implicitly handled the cropping during the labeling. # currently only the warping method is doing, don't need cropping for other methods anyways. if self.docrop: labels = labels[c[0]:s[0]-c[0],c[1]:s[1]-c[1],c[2]:s[2]-c[2]] # this method can not create true unconnected 3d labels, but should be unconnected in 2d. # NOTE: currently this only removes 6-connectivity, no matter what specified connecitity is # xxx - some method of removing adjacencies with arbitrary connectivity? uclabels, ucnlabels = emLabels.remove_adjacencies(labels) elif self.method == 'skim-ws': # xxx - still trying to evaluate if there is any advantage to this more traditional watershed. # it does not leave a non-adjacency boundary and is about 1.5 times slower than bwmorph # run connected components on the thresholded labels merged with previous seeds labels, nlabels = nd.measurements.label(bwlabels, self.bwconn) # run a true watershed based the current foreground probs using current components as markers labels = morph.watershed(probs[j+1], labels, connectivity=self.bwconn, mask=voxTypeSel[j]) # remove any adjacencies created during the watershed # NOTE: currently this only removes 6-connectivity, no matter what specified connecitity is # xxx - some method of removing adjacencies with arbitrary connectivity? uclabels, ucnlabels = emLabels.remove_adjacencies(labels) else: if self.method == 'comps-ws' and i>1: # this is an alternative to the traditional watershed that warps out only based on stepping # back through the thresholds in reverse order. has advantages of non-connectivity. # may help slightly for small supervoxels but did not show much improved metrics in # terms of large-scale connectivity (against skeletons) # about 4-5 times slower than regular warping method. # make an unconnected version of bwlabels by warping out but with mask only for this type # everything above current threshold is already labeled, so only need to use gray thresholds # starting below the current threshold level. bwlabels, diff, self.simpleLUT = binary_warping(bwlabels, np.ones(self.size,dtype=np.bool), mask=voxTypeSel[j], borderval=False, slow=True, simpleLUT=self.simpleLUT, connectivity=self.connectivity, gray=probs[j+1], grayThresholds=self.Ts[i-1::-1].astype(np.float32, order='C')) else: assert( self.method == 'comps' ) # bad method option # make an unconnected version of bwlabels by warping out but with mask only for this type bwlabels, diff, self.simpleLUT = binary_warping(bwlabels, np.ones(self.size,dtype=np.bool), mask=voxTypeSel[j], borderval=False, slow=True, simpleLUT=self.simpleLUT, connectivity=self.connectivity) # run connected components on the thresholded labels merged with previous seeds (warped out) uclabels, ucnlabels = nd.measurements.label(bwlabels, self.bwconn); # in this case the normal labels are the same as the unconnected labels because of warping labels = uclabels; nlabels = ucnlabels; # optionally make a skeletonized version of the unconnected labels # xxx - revisit this, currently not being used for anything, started as a method to skeletonize GT if self.skeletonize: # method to skeletonize using max range endpoints only sklabels, sknlabels = emLabels.ucskeletonize(uclabels, mask=voxTypeSel[j], sampling=self.attrs['scale'] if hasattr(self.attrs,'scale') else None) assert( sknlabels == ucnlabels ) # fill out these labels out so that they fill in remaining voxels based on voxType. # this uses bwdist method for finding nearest neighbors, so connectivity can be violoated. # this is mitigated by first filling out background using the warping transformation # (or watershed) above, then this step is only to fill in remaining voxels for the # current foreground voxType. labels = emLabels.nearest_neighbor_fill(labels, mask=voxTypeNotSel[j], sampling=self.attrs['scale'] if hasattr(self.attrs,'scale') else None) # save the components labels generated for this type types_labels[j] = labels.astype(emLabels.LBLS_DTYPE, copy=False); types_uclabels[j] = uclabels.astype(emLabels.LBLS_DTYPE, copy=False); types_nlabels[j] = nlabels if self.fg_types_labels[j] < 0 else 1 types_ucnlabels[j] = ucnlabels if self.fg_types_labels[j] < 0 else 1 if self.skeletonize: types_sklabels[j] = sklabels.astype(emLabels.LBLS_DTYPE, copy=False) # merge the fg components labels. they can not overlap because voxel type is winner-take-all. nlabels = 0; ucnlabels = 0; labels = np.zeros(self.size_crop, dtype=emLabels.LBLS_DTYPE); uclabels = np.zeros(self.size_crop, dtype=emLabels.LBLS_DTYPE); if self.skeletonize: sklabels = np.zeros(self.size, dtype=emLabels.LBLS_DTYPE); for j in range(self.nfg_types): sel = (types_labels[j] > 0); ucsel = (types_uclabels[j] > 0); if self.skeletonize: sksel = (types_sklabels[j] > 0); if self.fg_types_labels[j] < 0: labels[sel] += (types_labels[j][sel] + nlabels); uclabels[ucsel] += (types_uclabels[j][ucsel] + ucnlabels); if self.skeletonize: sklabels[sksel] += (types_sklabels[j][sksel] + ucnlabels); nlabels += types_nlabels[j]; ucnlabels += types_ucnlabels[j]; else: labels[sel] = self.fg_types_labels[j]; uclabels[ucsel] = self.fg_types_labels[j]; if self.skeletonize: sklabels[sksel] = self.fg_types_labels[j] nlabels += 1; ucnlabels += 1; if self.dpWatershedTypes_verbose: print('\tnlabels = %d' % (nlabels,)) #print('\tnlabels = %d %d' % (nlabels,labels.max())) # for debug only #assert(nlabels == labels.max()) # sanity check for non-overlapping voxTypeSel, comment for speed print('\tdone in %.4f s' % (time.time() - t,)) # make a fully-filled out version using bwdist nearest foreground neighbor wlabels = emLabels.nearest_neighbor_fill(labels, mask=None, sampling=self.attrs['scale'] if hasattr(self.attrs,'scale') else None) # write out the results if self.nTmin == 1: subgroups = ['%.8f' % (self.Ts[i],)] else: subgroups = ['%d' % (self.Tmins[k],), '%.8f' % (self.Ts[i],)] d = self.attrs.copy(); d['threshold'] = self.Ts[i]; d['types_nlabels'] = types_nlabels; d['Tmin'] = self.Tmins[k] emLabels.writeLabels(outfile=self.outlabels, chunk=self.chunk.tolist(), offset=self.offset_crop.tolist(), size=self.size_crop.tolist(), datasize=self.datasize.tolist(), chunksize=self.chunksize.tolist(), data=labels, verbose=writeVerbose, attrs=d, strbits=self.outlabelsbits, subgroups=['with_background']+subgroups ) emLabels.writeLabels(outfile=self.outlabels, chunk=self.chunk.tolist(), offset=self.offset_crop.tolist(), size=self.size_crop.tolist(), datasize=self.datasize.tolist(), chunksize=self.chunksize.tolist(), data=wlabels, verbose=writeVerbose, attrs=d, strbits=self.outlabelsbits, subgroups=['zero_background']+subgroups ) d['type_nlabels'] = types_ucnlabels; emLabels.writeLabels(outfile=self.outlabels, chunk=self.chunk.tolist(), offset=self.offset_crop.tolist(), size=self.size_crop.tolist(), datasize=self.datasize.tolist(), chunksize=self.chunksize.tolist(), data=uclabels, verbose=writeVerbose, attrs=d, strbits=self.outlabelsbits, subgroups=['no_adjacencies']+subgroups ) if self.skeletonize: emLabels.writeLabels(outfile=self.outlabels, chunk=self.chunk.tolist(), offset=self.offset_crop.tolist(), size=self.size_crop.tolist(), datasize=self.datasize.tolist(), chunksize=self.chunksize.tolist(), data=sklabels, verbose=writeVerbose, attrs=d, strbits=self.outlabelsbits, subgroups=['skeletonized']+subgroups )
def label_overlap(self, bwlabels, mask, warps=None): # this method operates slice by slice zlabels = np.zeros(self.size, dtype=np.int64) nzlabels = 0; prv_labels = None; connections = [None]*(self.size[2]-1) s = self.size; s2 = [s[0], s[1], 1]; c = self.cropborder if warps: x = np.arange(s[0], dtype=warps[0].dtype); y = np.arange(s[1], dtype=warps[0].dtype) X = np.meshgrid(x,y, indexing='ij') for z in range(self.size[2]): # get bwlabels and mask for the current zslice cur_bwlabels = bwlabels[:,:,z]; cur_mask = mask[:,:,z] # make an unconnected version of bwlabels by warping out but with mask only for this type bw = cur_bwlabels[:,:,None].copy(order='C'); msk = cur_mask[:,:,None].copy(order='C') bw, diff, self.simpleLUT = binary_warping(bw, np.ones(s2,dtype=np.bool), mask=msk, borderval=False, slow=True, simpleLUT=self.simpleLUT, connectivity=self.connectivity) cur_fill_bwlabels = bw[:,:,0] # run connected components on the thresholded labels merged with previous seeds (warped out) cur_fill_labels, cur_nlabels = nd.measurements.label(cur_fill_bwlabels, self.bwconn2d, output=np.int64) # make labels for this zslice unique and add to whole label cube sel = (cur_fill_labels > 0); cur_fill_labels[sel] += nzlabels; zlabels[:,:,z] = cur_fill_labels # get eroded labels by applying mask for original bwlabels cur_labels = cur_fill_labels.copy(); cur_labels[np.logical_not(cur_bwlabels)] = 0 # warp the previous slice to this slice and connect them if z > 0: if warps: # apply warping from previous label slice to current slice using nearest neighbor interpolation. cur_warpsx = warps[0][:,:,z-1]; cur_warpsy = warps[1][:,:,z-1] xi = X[0] + cur_warpsx; yi = X[1] + cur_warpsy # remove warps that are out of the size bounds #xi[xi < 0] = 0; xi[xi > s[0]-1] = s[0]-1; yi[yi < 0] = 0; yi[yi > s[1]-1] = s[1]-1 f = interpolate.RegularGridInterpolator((x,y), prv_labels, method='nearest', bounds_error=False, fill_value=0) prv_labels = f( np.vstack((xi.ravel(),yi.ravel())).T ).reshape(prv_labels.shape) # map the previous warped labels to the current labels based on pixel-by-pixel overlap of eroded labels. # only used the xy cropped area to do the linkage. prv_labels_crop = prv_labels; cur_labels_crop = cur_labels if self.docrop: prv_labels_crop = prv_labels_crop[c[0]:s[0]-c[0],c[1]:s[1]-c[1]] cur_labels_crop = cur_labels_crop[c[0]:s[0]-c[0],c[1]:s[1]-c[1]] tmp = dpWatershedTypes.unique_rows(\ np.ascontiguousarray( np.vstack((prv_labels_crop.ravel(),cur_labels_crop.ravel())).T )) # remove background connections (any rows with zeros) connections[z-1] = tmp[(tmp>0).all(axis=1),:] # loop updates for linking current slice to next prv_labels = cur_labels; nzlabels += cur_nlabels # run graph connected components on graph created from pairwise connections. # this graph represents labels that have been linked by the warping between zslices. G = nx.Graph(); G.add_edges_from(np.vstack(connections)) compsG = nx.connected_components(G); nlabels = 0; mapping = np.zeros((nzlabels+1,), dtype=np.int64) for nodes in compsG: # create mapping from current per-zslice labels to linked labels across zslices nlabels += 1; mapping[np.array(tuple(nodes),dtype=np.int64)] = nlabels # create the final labels using the mapping built from the graph connected components return mapping[zlabels], nlabels
def clean(self): # read voxel types first, allows for cavity_fill and get_svox_type to be both specified if self.get_svox_type or self.write_voxel_type or self.apply_bg_mask: if self.dpCleanLabels_verbose: print('Reading supervoxel types') t = time.time() voxType = emVoxelType.readVoxType(srcfile=self.srcfile, chunk=self.chunk.tolist(), offset=self.offset.tolist(), size=self.size.tolist()) voxel_type = voxType.data_cube.copy(order='C') ntypes = len(voxType.data_attrs['types']) if self.dpCleanLabels_verbose: print('\tdone in %.4f s' % (time.time() - t)) # minpath overlay creation is intended to improve proofreading speed by highlighting connected paths if self.minpath > 0: import skfmm if self.minpath_skel: from pyCext import binary_warping # xxx - allow for multiple minpaths with different labels? selmin = (self.data_cube == self.minpath) pts, npts = nd.measurements.label(selmin, self.fgbwconn) if self.dpCleanLabels_verbose: print( 'Finding shortest paths for all pairwise combinations of %d points' % (npts, )) t = time.time() labels = self.data_cube sel_ECS, ECS_label = self.getECS(labels) labels[sel_ECS] = 0 selbg = (labels == 0) assert (self.minpath != ECS_label ) # can't have minpath and ECS label defined the same # create paths for all pair-wise combinations of points paths = np.zeros(self.size, dtype=np.uint8) for i in range(npts): for j in range(i + 1, npts): s1 = (pts == i + 1) m = np.ones(self.size, dtype=np.double) m[s1] = 0 d1 = skfmm.distance( np.ma.masked_array( nd.distance_transform_edt( m, return_indices=False, return_distances=True, sampling=self.data_attrs['scale']), selbg)) s2 = (pts == j + 1) m = np.ones(self.size, dtype=np.double) m[s2] = 0 d2 = skfmm.distance( np.ma.masked_array( nd.distance_transform_edt( m, return_indices=False, return_distances=True, sampling=self.data_attrs['scale']), selbg)) # xxx - need something like imregionalmin in 3d, could not quickly find an easy solution d = d1 + d2 bwlabels = ((d.data < self.minpath_perc * d.min()) & ~d.mask) bwlabels[s1 | s2] = 1 if self.minpath_skel: # optionally skeletonize keeping original minpath points as anchors bwlabels, diff, simpleLUT = binary_warping( bwlabels.copy(order='C'), np.zeros(self.size, dtype=bool), mask=(~selmin).copy(order='C'), borderval=False, slow=True, connectivity=self.fg_connectivity) # fill back out slightly so more easily viewed in itksnap bwlabels, diff, simpleLUT = binary_warping( bwlabels.copy(order='C'), np.ones(self.size, dtype=bool), borderval=False, slow=True, simpleLUT=simpleLUT, connectivity=self.fg_connectivity, numiters=1) paths[bwlabels] = 1 self.data_cube = paths if self.dpCleanLabels_verbose: print('\tdone in %.4f s' % (time.time() - t)) # smoothing operates on each label one at a time if self.smooth: if self.dpCleanLabels_verbose: print('Smoothing labels object by object') t = time.time() # threshold sizes to remove empty labels self.data_cube, sizes = emLabels.thresholdSizes(self.data_cube, minSize=1) # exposed smoothing kernel size and contour level as parameters smooth_size = self.smooth_size #contour_level = self.contour_lvl # calculate padding based on smoothing kernel size rad = int(1.5 * smooth_size.max()) sizes = np.array(self.data_cube.shape) sz = sizes + 2 * rad image_with_zeros = np.zeros(sz, dtype=self.data_cube.dtype) # create zeros 3 dimensional array image_with_zeros[ rad:-rad, rad:-rad, rad: -rad] = self.data_cube # embed label array into zeros array image_with_brd = np.lib.pad(self.data_cube, ((rad, rad), (rad, rad), (rad, rad)), 'edge') # do not smooth ECS labels sel_ECS, ECS_label = self.getECS(image_with_brd) if self.dpCleanLabels_verbose and ECS_label: print('\tignoring ECS label %d' % (ECS_label, )) # get bounding boxes for each supervoxel in zero padded label volume svox_bnd = nd.measurements.find_objects(image_with_zeros) # iterate over labels nSeeds = len(svox_bnd) lbls = np.zeros(sz, dtype=self.data_cube.dtype) for j in range(nSeeds): if ECS_label and j + 1 == ECS_label: continue #if self.dpCleanLabels_verbose: # print('Smoothing label %d / %d' % (j+1,nSeeds)); t = time.time() pbnd = tuple( [slice(x.start - rad, x.stop + rad) for x in svox_bnd[j]]) Lcrpsel = (image_with_brd[pbnd] == j + 1) Lcrp = Lcrpsel.astype(np.double) # smoothing operation just box filter on cropped binarized label Lfilt = nd.filters.uniform_filter(Lcrp, size=smooth_size, mode='constant') # this feature allows for variable contour level depending on if object splits apart if len(self.contour_lvl) > 1: # get the original number of components for the object nlabels_orig = nd.measurements.label( Lcrpsel, self.fgbwconn)[1] for c in np.arange( self.contour_lvl[1], self.contour_lvl[0] - self.contour_lvl[2] / 10, -self.contour_lvl[2]): # incase smoothing below contour level, use without smoothing if not (Lfilt > c).any(): Lfilt = Lcrp # check new number of labels, stop if it same (or less?) than original number of components csel = (Lfilt > c) nlabels = nd.measurements.label(csel, self.fgbwconn)[1] if nlabels <= nlabels_orig: break else: contour_level = self.contour_lvl[0] # incase smoothing below contour level, use without smoothing if not (Lfilt > contour_level).any(): Lfilt = Lcrp csel = (Lfilt > contour_level) # assign smoothed output for current label lbls[pbnd][csel] = j + 1 # put ECS labels back if ECS_label: lbls[sel_ECS] = ECS_label if self.dpCleanLabels_verbose: print('\tdone in %.4f s' % (time.time() - t)) self.data_cube = lbls[rad:-rad, rad:-rad, rad:-rad] if self.remove_adjacencies: r = self.remove_adjacencies assert (r > 1 and r % 2 == 1) # bad nbhd size labels = self.data_cube.astype(np.uint32, copy=True, order='C') sel_ECS, ECS_label = self.getECS(labels) labels[sel_ECS] = 0 if self.dpCleanLabels_verbose: print('Removing adjacencies with nbhd %d%s' % (r, ', ignoring ECS label %d' % (ECS_label, ) if ECS_label else '')) t = time.time() self.data_cube = emLabels.remove_adjacencies_nconn(labels, bwconn=np.ones( (r, r, r), dtype=bool)) if ECS_label: self.data_cube[sel_ECS] = ECS_label if self.dpCleanLabels_verbose: print('\tdone in %.4f s' % (time.time() - t)) if self.minsize > 0: labels = self.data_cube labels, nlabels = self.minsize_scrub(labels, self.minsize, self.minsize_fill) self.data_cube = labels # allow this to work before self.get_svox_type or self.write_voxel_type self.data_attrs['types_nlabels'] = [nlabels] # NOTE: cavity_fill not intended to work with ECS labeled with single value (ECS components are fine) if self.cavity_fill: if self.labelwise: if self.dpCleanLabels_verbose: print('Removing cavities labelwise using conn %d' % (self.bg_connectivity, )) print('\tGetting bounding boxes') t = time.time() # iterate over labels, fill each label within bounding box svox_bnd = nd.measurements.find_objects(self.data_cube) if self.dpCleanLabels_verbose: print('\t\tdone in %.4f s' % (time.time() - t)) t = time.time() verbose = self.dpCleanLabels_verbose self.dpCleanLabels_verbose = False nSeeds = len(svox_bnd) lbls = np.zeros(self.size, dtype=self.data_cube.dtype) for j in range(nSeeds): csel, selbg, msk = self.cavity_fill_voxels( self.data_cube[svox_bnd[j]] == j + 1, tab=True) lbls[svox_bnd[j]][csel] = j + 1 del self.data_cube self.data_cube = lbls if verbose: print('\tdone in %.4f s' % (time.time() - t)) self.dpCleanLabels_verbose = True else: self.data_cube, selbg, msk = self.cavity_fill_voxels( self.data_cube) # this prevents any supervoxels as being classified as "membrane". # many scripts assume that membrane is labeled as background (label 0). if self.get_svox_type or self.write_voxel_type: if self.dpCleanLabels_verbose: print('\tRemoving cavities from voxel type') t = time.time() if ntypes - 1 > 1: voxel_type = emLabels.nearest_neighbor_fill( voxel_type, mask=selbg, sampling=self.data_attrs['scale']) else: voxel_type[msk] = 1 print('\t\tdone in %.4f s' % (time.time() - t)) del msk, selbg # NOTE: cavity_fill not intended to work with ECS labeled with single value (ECS components are fine) # This method is not foolproof, in one shot it removes all labels below a specified size and then reruns # cavity fill. If a label was not filled then it puts it back. This will not remove any labels that are # in a cavity but connected to background via a bunch of other labels smaller than specified size. if self.cavity_fill_minsize > 1: if self.dpCleanLabels_verbose: print('Removing labels < %d in cavities' % (self.cavity_fill_minsize, )) labels_orig = self.data_cube data, nlabels = self.minsize_scrub(labels_orig, self.cavity_fill_minsize, False, tab=True, no_remap=True) # do a normal cavity fill after labels smaller than cavity_fill_minsize are removed labels, selbg, msk = self.cavity_fill_voxels(data, tab=True) del msk, selbg if self.dpCleanLabels_verbose: print('\tReplacing non-cavity labels') sel_not_fill = np.logical_and(labels_orig > 0, labels == 0) labels[sel_not_fill] = labels_orig[sel_not_fill] del labels_orig, sel_not_fill # remove any zero labels (that were removed as cavities) self.data_cube, nlabels = self.minsize_scrub(labels, 1, False, tab=True) del labels # allow this to work before self.get_svox_type or self.write_voxel_type self.data_attrs['types_nlabels'] = [nlabels] if self.relabel: labels = self.data_cube sel_ECS, ECS_label = self.getECS(labels) labels[sel_ECS] = 0 if self.dpCleanLabels_verbose: print('Relabeling fg components with conn %d%s' % (self.fg_connectivity, ', ignoring ECS label %d' % (ECS_label, ) if ECS_label else '')) print('\tnlabels = %d, max = %d, before re-label' % (len(np.unique(labels)), labels.max())) t = time.time() labels, nlabels = nd.measurements.label(labels, self.fgbwconn) labels, nlabels = self.setECS(labels, sel_ECS, ECS_label, nlabels) self.data_cube = labels # allow this to work before self.get_svox_type or self.write_voxel_type self.data_attrs['types_nlabels'] = [nlabels] if self.dpCleanLabels_verbose: print('\tnlabels = %d after re-label' % (nlabels, )) print('\tdone in %.4f s' % (time.time() - t)) # this step re-writes the original background (membrane) mask back to the supervoxels. # this is useful if agglomeration was done using the completely watershedded supervoxels. if self.apply_bg_mask: if self.dpCleanLabels_verbose: print('Applying background (membrane) mask to supervoxels') t = time.time() sel = (voxel_type == 0) self.data_cube[sel] = 0 if self.dpCleanLabels_verbose: print('\tdone in %.4f s' % (time.time() - t)) # this step is always last, as writes new voxel_type depending on the cleaning that was done if self.get_svox_type or self.write_voxel_type: if self.dpCleanLabels_verbose: print('Recomputing supervoxel types and re-ordering labels') t = time.time() # moved this as first step to allow other steps to modify voxel_type #voxType = emVoxelType.readVoxType(srcfile=self.srcfile, chunk=self.chunk.tolist(), # offset=self.offset.tolist(), size=self.size.tolist()) #voxel_type = voxType.data_cube.copy(order='C') #ntypes = len(voxType.data_attrs['types']) labels = self.data_cube.copy(order='C') #nlabels = labels.max(); assert(nlabels == self.data_attrs['types_nlabels'][0]) nlabels = sum(self.data_attrs['types_nlabels']) supervoxel_type, voxel_type = emLabels.type_components( labels, voxel_type, nlabels, ntypes) assert (supervoxel_type.size == nlabels) # reorder labels so that supervoxels are grouped by / in order of supervoxel type remap = np.zeros((nlabels + 1, ), dtype=self.data_cube.dtype) remap[np.argsort(supervoxel_type) + 1] = np.arange( 1, nlabels + 1, dtype=self.data_cube.dtype) self.data_cube = remap[self.data_cube] types_nlabels = [(supervoxel_type == x).sum(dtype=np.int64) for x in range(1, ntypes)] assert (sum(types_nlabels) == nlabels ) # indicates voxel type does not match supervoxels self.data_attrs['types_nlabels'] = types_nlabels if self.write_voxel_type: if self.dpCleanLabels_verbose: print( 'Rewriting voxel type pixel data based on supervoxel types' ) d = voxType.data_attrs.copy() #d['types_nlabels'] = emVoxelType.writeVoxType(outfile=self.outfile, chunk=self.chunk.tolist(), offset=self.offset.tolist(), size=self.size.tolist(), datasize=voxType.datasize.tolist(), chunksize=voxType.chunksize.tolist(), data=voxel_type.astype( emVoxelType.VOXTYPE_DTYPE), attrs=d) if self.dpCleanLabels_verbose: print('\tdone in %.4f s' % (time.time() - t)) # this step should not be mixed with other steps if self.replace_ECS: assert (len(self.data_attrs['types_nlabels']) == 2) sel_ECS = (self.data_cube > self.data_attrs['types_nlabels'][0]) sel_ICS = np.logical_and( self.data_cube > 0, self.data_cube <= self.data_attrs['types_nlabels'][0]) self.data_cube[sel_ICS] += self.min_label self.data_cube[sel_ECS] = self.ECS_label # xxx - probably shouldn't be using this anyways? self.data_attrs['types_nlabels'] = self.data_attrs[ 'types_nlabels'][0]
from dpLoadh5 import dpLoadh5 from dpWriteh5 import dpWriteh5 #from emdrp.utils.typesh5 import emLabels, emProbabilities, emVoxelType #from emdrp.utils.typesh5 import emLabels from pyCext import binary_warping overlay_in = '/home/watkinspv/Downloads/K0057_soma_annotation/affine_2Pstack/K0057_D31_soma_seg_overlays_v3.gipl' data, hdr, info = dpWriteh5.gipl_read_volume(overlay_in) data = data.reshape(hdr['sizes'][:3][::-1]).transpose(((2, 1, 0))) print(data.shape, data.dtype) # warp all the way down to points for each object. xxx - could be prohibitive for larger volumes bwlabels, diff, simpleLUT = binary_warping((data > 0).copy(order='C'), np.zeros(data.shape, dtype=np.bool), borderval=False, slow=True, connectivity=3) subs = np.nonzero(bwlabels) #np.set_printoptions(threshold=np.nan); print(np.transpose(subs)) # get soma information back from original overlay types = data[subs] # convert subscripts into appropriate downsampling space offset = np.array( [1, 1, 0] ) # try to deal with warping bias, has to do with shape cells were labeled with 9x9x8 rect subs_out = (np.transpose(subs) + offset) * np.array([16, 16, 16]) // np.array( [12, 12, 4])