def getBoundariesOfimage(image): """ find edges by using erosion """ if np.ndim(image) == 2: sElement = ndimage.generate_binary_structure(2, 1) else: sElement = ndimage.generate_binary_structure(3, 1) erode_im = scipy.ndimage.morphology.binary_erosion(image, sElement) b = image - erode_im return b
def adjust_spot_positions(image, label_image, hp, debug=None): """Re-evaluate the spot positions based on the segmentation. Parameters: image: The original image (can be masked) that was sent to findspot3d label_image: the label image containing two labels hp: the original hotpoints debug: set to true to write out an image debugimg.nii.gz with the stuff """ struct2 = generate_binary_structure(3, 2) struct1 = generate_binary_structure(3, 1) peak_points =[] if debug is None: temp_path = os.getenv("PYSBR_TEMP") if temp_path is not None: debug = os.path.join(temp_path, "debug-labels.nii.gz") if debug is not None: debimg = image.copy() nlabels = label_image.max() if nlabels!=len(hp): raise RuntimeError( 'number of labels and hotspots should be the same' ) tins = [] for n in range(nlabels): label = n+1 area = binary_closing(label_image == label, struct2) thiniter = np.sum(area.reshape(-1)) / 1500 + 1 csbr.thinning3d(area, thiniter) tins.append(area) for n in range(nlabels): label = n+1 #avoid that a single pixel breaks the evaluation by running a closing area = label_image == label #evaluate the boundary dmask = binary_dilation(area, struct1) border = np.bitwise_xor(dmask, area) p = adjust_spot_position(image, border, image[tuple(hp[n])], tins[n], tins[(n + 1) % 2]) peak_points.append(p) if debug is not None: debimg[border>0] = 196 debimg[p] = 0 nib.save(nib.Nifti1Image(debimg, global_affine), debug) peak_points = np.array( peak_points ) return peak_points
def remove_small_objects(arr, min_size, connectivity=8): ''' Remove objects less than the given size. Function is based on skimage.morphology.remove_small_objects Parameters ---------- arr : numpy.ndarray Binary array containing the mask. min_size : int Smallest allowed size. connectivity : int, optional Connectivity of the neighborhood. ''' struct = nd.generate_binary_structure(arr.ndim, connectivity) labels, num = nd.label(arr, struct) sizes = nd.sum(arr, labels, range(1, num + 1)) for i, size in enumerate(sizes): if size >= min_size: continue posns = np.where(labels == i + 1) arr[posns] = 0 return arr
def moment_masking(cube, kernel_size, clip=5, dilations=1): ''' ''' if not signal_id_flag: raise ImportError("signal-id is not installed." " This function is not available.") smooth_data = convolve(cube.filled_data[:], gauss_kern(kernel_size)) fake_mask = LazyMask(np.isfinite, cube=cube) smooth_cube = SpectralCube(data=smooth_data, wcs=cube.wcs, mask=fake_mask) smooth_scale = Noise(smooth_cube).scale mask = (smooth_cube > (clip * smooth_scale)).include() # Now dilate the mask once dilate_struct = nd.generate_binary_structure(3, 3) mask = nd.binary_dilation(mask, structure=dilate_struct, iterations=dilations) return mask
def getPatchNamesFromMask(mask, x, y, root='mask'): """ Returns an array of patch names for each (x, y) pair """ import math import scipy.ndimage as nd import numpy as np act_pixels = mask rank = len(act_pixels.shape) connectivity = nd.generate_binary_structure(rank, rank) mask_labels, count = nd.label(act_pixels, connectivity) patchNums = [] patchNames = [] for xs, ys in zip(x, y): try: patchNums.append(mask_labels[xs, ys]) except: patchNums.append(0) # Check if there is a patch with id = 0. If so, this means there were # some Gaussians that fell outside of the regions in the patch # mask file. n = 0 for p in patchNums: if p != 0: in_patch = np.where(patchNums == p) patchNames.append('{0}_patch_'.format(root)+str(p)) else: patchNames.append('patch_'+str(n)) n += 1 return np.array(patchNames)
def art_qi2(img, airmask, ncoils=12, erodemask=True): """ Calculates **qi2**, the distance between the distribution of noise voxel (non-artifact background voxels) intensities, and a centered Chi distribution. :param numpy.ndarray img: input data :param numpy.ndarray airmask: input air mask without artifacts """ from matplotlib import rc import seaborn as sn import matplotlib.pyplot as plt rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']}) # rc('text', usetex=True) if erodemask: struc = nd.generate_binary_structure(3, 2) # Perform an opening operation on the background data. airmask = nd.binary_erosion(airmask, structure=struc).astype(np.uint8) # Artifact-free air region data = img[airmask > 0] data = data[data < np.percentile(data, 99.5)] maxvalue = int(data.max()) nbins = maxvalue if maxvalue < 100 else 100 # Estimate data pdf hist, bin_edges = np.histogram(data, density=True, bins=nbins) bin_centers = [np.mean(bin_edges[i:i+1]) for i in range(len(bin_edges)-1)] max_pos = np.argmax(hist) # Fit central chi distribution param = chi.fit(data, 2*ncoils, loc=bin_centers[max_pos]) pdf_fitted = chi.pdf(bin_centers, *param[:-2], loc=param[-2], scale=param[-1]) # Write out figure of the fitting out_file = op.abspath('background_fit.png') fig = plt.figure() ax1 = fig.add_subplot(111) sn.distplot(data, bins=nbins, norm_hist=True, kde=False, ax=ax1) #_, bins, _ = ax1.hist(data, nbins, normed=True, color='gray', linewidth=0) ax1.plot(bin_centers, pdf_fitted, 'k--', linewidth=1.2) fig.suptitle('Noise distribution on the air mask, and fitted chi distribution') ax1.set_xlabel('Intensity') ax1.set_ylabel('Frequency') fig.savefig(out_file, format='png', dpi=300) plt.close() # Find t2 (intensity at half width, right side) ihw = 0.5 * hist[max_pos] t2idx = 0 for i in range(max_pos + 1, len(bin_centers)): if hist[i] < ihw: t2idx = i break # Compute goodness-of-fit (gof) return (float(np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() / len(pdf_fitted[t2idx:])), out_file)
def compute_mask(aparc, labels=[0, 5000]): import nibabel as nb import numpy as np import os.path as op import scipy.ndimage as nd segnii = nb.load(aparc) seg = segnii.get_data() mask = np.ones_like(seg, dtype=np.uint8) for l in labels: mask[seg == l] = 0 struct = nd.iterate_structure(nd.generate_binary_structure(3, 1), 4) mask = nd.binary_dilation(mask, structure=struct).astype(np.uint8) mask = nd.binary_closing(mask, structure=struct) mask = nd.binary_fill_holes(mask, structure=struct).astype(np.uint8) mask[mask > 0] = 1 mask[mask <= 0] = 0 hdr = segnii.get_header().copy() hdr.set_data_dtype(np.uint8) hdr.set_xyzt_units("mm", "sec") out_file = op.abspath("nobstem_mask.nii.gz") nii = nb.Nifti1Image(mask, segnii.get_affine(), hdr).to_filename(out_file) return out_file
def gen_data(xsize, ysize, nstars=3, starradius=10, brightness=2000): # 1) lots of stars, big # 2) lots of tiny stars # 3) few stars, big # 4) few stars, tiny footprint = ndimage.generate_binary_structure(2,1) ret = numpy.zeros((xsize, ysize)) for star in xrange(nstars): xcenter = random.randint(0, xsize-1) ycenter = random.randint(0, ysize-1) for x in xrange(xcenter-1, xcenter+2): for y in xrange(ycenter-1, ycenter+2): if x >= 0 and y >= 0 and x < xsize and y < ysize: ret[x,y] = brightness / 3 ret[xcenter, ycenter] = brightness for i in xrange(starradius): ret = ndimage.grey_dilation(ret, footprint=footprint) # add some cosmic rays (single points) for i in xrange(30): xcenter = random.randint(0, xsize-1) ycenter = random.randint(0, ysize-1) ret[xcenter, ycenter] = brightness return ret
def ClusterizeImage(image,thresh=None,connectivity=3): if thresh is None: thresh = 0 image[np.where(image<=thresh)] = 0 s = generate_binary_structure(3,connectivity) larray, nf = label(image,s) return larray
def edge_matrix(labels, connectivity=1): """Generate a COO matrix containing the coordinates of edge pixels. Parameters ---------- labels : array of int An array of labeled pixels (or voxels). connectivity : int in {1, ..., labels.ndim} The square connectivity for considering neighborhood. Returns ------- edges : sparse.coo_matrix A COO matrix where (i, j) indicate neighboring labels and the corresponding data element is the linear index of the edge pixel in the labels array. """ conn = ndi.generate_binary_structure(labels.ndim, connectivity) eroded = ndi.grey_erosion(labels, footprint=conn).ravel() dilated = ndi.grey_dilation(labels, footprint=conn).ravel() labels = labels.ravel() boundaries0 = np.flatnonzero(eroded != labels) boundaries1 = np.flatnonzero(dilated != labels) labels_small = np.concatenate((eroded[boundaries0], labels[boundaries1])) labels_large = np.concatenate((labels[boundaries0], dilated[boundaries1])) n = np.max(labels_large) + 1 data = np.concatenate((boundaries0, boundaries1)) sparse_graph = sparse.coo_matrix((data, (labels_small, labels_large)), dtype=np.int_, shape=(n, n)) return sparse_graph
def objextract(Fg): s = nd.generate_binary_structure(2,2) labeled_array, num_features = nd.measurements.label(Fg, structure=s) coor = [] cnt = [] if num_features == 0: idx = [] else: lth = 200 # label pixel number less than lth will be removed Lth = 6500 for i in range(1,num_features+1): coor.append(np.where(labeled_array==i)) cnt.append(len(np.where(labeled_array==i)[1])) cnt = array(cnt) idx = arange(num_features) idx = idx[(cnt<Lth)&(cnt>lth)] if len(idx)==0: idx = [] elif len(idx)>1: #idx = [idx[cnt[idx].argmax()]] idx = sorted(range(len(cnt)),key=lambda x:cnt[x])[::-1][0:2] return idx,labeled_array,coor,cnt
def find_local_max(img, d_rad, threshold=1e-15): """ This is effectively a replacement for pkfnd in the matlab/IDL code. The output of this function is meant to be feed into :py:func:`~subpixel_centroid` The magic of numpy means this should work for any dimension data. :param img: an ndarray representing the data to find the local maxes :param d_rad: the radius of the dilation, the smallest possible spacing between local maximum :param threshold: optional, voxels < threshold are ignored. :rtype: (d,N) array of the local maximums. """ d_rad = int(d_rad) img = np.array(np.squeeze(img)) # knock out singleton dimensions img[img < threshold] = -np.inf # mask out pixels below threshold dim = img.ndim # get the dimension of data # make structuring element s = ndimage.generate_binary_structure(dim, 1) # scale it up to the desired size d_struct = ndimage.iterate_structure(s, int(d_rad)) dilated_img = ndimage.grey_dilation(img, footprint=d_struct, cval=0, mode='constant') # do the dilation # find the locations that are the local maximum # TODO clean this up local_max = np.where(np.exp(img - dilated_img) > (1 - 1e-15)) # the extra [::-1] is because matplotlib and ndimage disagree an xy vs yx return np.vstack(local_max[::-1])
def thresholding(img, thresh, size=9): """ Segment using a thresholding algorithm Input: - img ndarray : Image array (ndim=2) - thresh float : Threshold value for pixels selectino - size int : Minimum size a group of pixels must have Output: - regions : Binary array for each segmented region --- """ logging.debug("Threshold: %.2f", thresh) logging.debug("Objects min size: %d", size) # Take the binary image thresholded img_bin = img > thresh # And use (MO) binary opening (erosion + dilation) for cleaning spurious Trues strct = ndi.generate_binary_structure(2, 2) img_bin = ndi.binary_opening(img_bin, strct) # Label each group/region (value==True) of pixels regions, nlbl = ndi.label(img_bin) for i in xrange(1, nlbl + 1): inds = np.where(regions == i) if inds[0].size < size: regions[inds] = 0 logging.debug("Threshold labels: %s", np.unique(regions)) return regions.astype(np.bool)
def remove_small_objects(ar, min_size=64, connectivity=1, in_place=False): # Should use `issubdtype` for bool below, but there's a bug in numpy 1.7 if not (ar.dtype == bool or np.issubdtype(ar.dtype, np.integer)): raise TypeError("Only bool or integer image types are supported. " "Got %s." % ar.dtype) if in_place: out = ar else: out = ar.copy() if min_size == 0: # shortcut for efficiency return out if out.dtype == bool: selem = nd.generate_binary_structure(ar.ndim, connectivity) ccs = np.zeros_like(ar, dtype=np.int32) nd.label(ar, selem, output=ccs) else: ccs = out try: component_sizes = np.bincount(ccs.ravel()) except ValueError: raise ValueError("Negative value labels are not supported. Try " "relabeling the input with `scipy.ndimage.label` or " "`skimage.morphology.label`.") too_small = component_sizes < min_size too_small_mask = too_small[ccs] out[too_small_mask] = 0 return out
def artifact_mask(imdata, airdata, distance): """Computes a mask of artifacts found in the air region""" import nibabel as nb if not np.issubdtype(airdata.dtype, np.integer): airdata[airdata < .95] = 0 airdata[airdata > 0.] = 1 bg_img = imdata * airdata # Find the background threshold (the most frequently occurring value # excluding 0) # CHANGED - to the 75 percentile bg_threshold = np.percentile(bg_img[airdata > 0], 75) # Apply this threshold to the background voxels to identify voxels # contributing artifacts. qi1_img = np.zeros_like(bg_img) qi1_img[bg_img > bg_threshold] = 1 qi1_img[distance < .10] = 0 # Create a structural element to be used in an opening operation. struc = nd.generate_binary_structure(3, 1) qi1_img = nd.binary_opening(qi1_img, struc).astype(np.uint8) qi1_img[airdata <= 0] = 0 return qi1_img
def test_3d(self): """tests the detection of maxima in 3D.""" img = np.zeros((8, 8, 8), dtype=np.uint8) local_maxima = np.zeros((8, 8, 8), dtype=np.uint8) # first maximum: only one pixel img[1, 1:3, 1:3] = 100 img[2, 2, 2] = 200 img[3, 1:3, 1:3] = 100 local_maxima[2, 2, 2] = 1 # second maximum: three pixels in z-direction img[5:8, 1, 1] = 200 local_maxima[5:8, 1, 1] = 1 # third: two maxima in 0 and 3. img[0, 5:8, 5:8] = 200 img[1, 6, 6] = 100 img[2, 5:7, 5:7] = 200 img[0:3, 5:8, 5:8] += 50 local_maxima[0, 5:8, 5:8] = 1 local_maxima[2, 5:7, 5:7] = 1 # four : one maximum in the corner of the square img[6:8, 6:8, 6:8] = 200 img[7, 7, 7] = 255 local_maxima[7, 7, 7] = 1 se = ndi.generate_binary_structure(3, 1) out = extrema.local_maxima(img, se) error = diff(local_maxima, out) assert error < eps
def artifact_mask(imdata, airdata): """Computes a mask of artifacts found in the air region""" if not np.issubdtype(airdata.dtype, np.integer): airdata[airdata < .95] = 0 airdata[airdata > 0.] = 1 bg_img = imdata * airdata # Find the background threshold (the most frequently occurring value # excluding 0) hist, bin_edges = np.histogram(bg_img[bg_img > 0], bins=128) bg_threshold = np.mean(bin_edges[np.argmax(hist)]) # Apply this threshold to the background voxels to identify voxels # contributing artifacts. qi1_img = np.zeros_like(bg_img) qi1_img[bg_img > bg_threshold] = bg_img[bg_img > bg_threshold] # Create a structural element to be used in an opening operation. struc = nd.generate_binary_structure(3, 2) # Perform an a grayscale erosion operation. qi1_img = nd.grey_erosion(qi1_img, structure=struc).astype(np.float32) # Binarize and binary dilation qi1_img[qi1_img > 0.] = 1 qi1_img[qi1_img < 1.] = 0 qi1_img = nd.binary_dilation(qi1_img, structure=struc).astype(np.uint8) return qi1_img
def test_mask(): vol = np.zeros((30, 30, 30)) vol[15, 15, 15] = 1 struct = generate_binary_structure(3, 1) voln = binary_dilation(vol, structure=struct, iterations=4).astype('f4') initial = np.sum(voln > 0) mask = voln.copy() thresh = otsu(mask) mask = mask > thresh initial_otsu = np.sum(mask > 0) assert_equal(initial_otsu, initial) mins, maxs = bounding_box(mask) voln_crop = crop(mask, mins, maxs) initial_crop = np.sum(voln_crop > 0) assert_equal(initial_crop, initial) applymask(voln, mask) final = np.sum(voln > 0) assert_equal(final, initial) # Test multi_median. median_test = np.arange(25).reshape(5, 5) median_control = median_test.copy() medianradius = 3 median_test = multi_median(median_test, medianradius, 3) medarr = np.ones_like(median_control.shape) * ((medianradius * 2) + 1) median_filter(median_control, medarr, output=median_control) median_filter(median_control, medarr, output=median_control) median_filter(median_control, medarr, output=median_control) assert_equal(median_test, median_control)
def multi_label_edge_detection(data): f = nd.generate_binary_structure(len(data.shape), 1) bound = (nd.grey_erosion(data,footprint=f) != nd.grey_dilation(data,footprint=f)) - \ (nd.binary_dilation(data.astype(np.bool)) - data.astype(np.bool)) # the unwanted thick bounds data=bound.astype(data.dtype) return data
def test_labeling(): "Test cluster labeling" shape = flat_shape = (4, 20) pmap = np.empty(shape, np.float_) struct = ndimage.generate_binary_structure(2, 1) struct[::2] = False conn = np.array([(0, 1), (0, 3), (1, 2), (2, 3)], np.uint32) criteria = None # some clusters pmap[:] = [[ 3, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0], [ 0, 1, 0, 0, 0, 0, 8, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 0], [ 0, 3, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4], [ 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0]] cmap, cids = label_clusters(pmap, 2, 0, conn, criteria) assert_equal(len(cids), 6) assert_array_equal(cmap > 0, np.abs(pmap) > 2) # some other clusters pmap[:] = [[ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0], [ 0, 4, 0, 0, 0, 0, 0, 4, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0], [ 0, 0, 4, 4, 0, 4, 4, 0, 4, 0, 0, 0, 4, 4, 1, 0, 4, 4, 0, 0], [ 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0]] cmap, cids = label_clusters(pmap, 2, 0, conn, criteria) assert_equal(len(cids), 6) assert_array_equal(cmap > 0, np.abs(pmap) > 2)
def _run_interface(self, runtime): in_file = nb.load(self.inputs.in_file) wm_mask = nb.load(self.inputs.wm_mask).get_data() wm_mask[wm_mask < 0.9] = 0 wm_mask[wm_mask > 0] = 1 wm_mask = wm_mask.astype(np.uint8) if self.inputs.erodemsk: # Create a structural element to be used in an opening operation. struc = nd.generate_binary_structure(3, 2) # Perform an opening operation on the background data. wm_mask = nd.binary_erosion(wm_mask, structure=struc).astype(np.uint8) data = in_file.get_data() data *= 1000.0 / np.median(data[wm_mask > 0]) out_file = fname_presuffix(self.inputs.in_file, suffix='_harmonized', newpath='.') in_file.__class__(data, in_file.affine, in_file.header).to_filename( out_file) self._results['out_file'] = out_file return runtime
def artifact_mask(imdata, airdata, distance, zscore=10.): """Computes a mask of artifacts found in the air region""" from statsmodels.robust.scale import mad if not np.issubdtype(airdata.dtype, np.integer): airdata[airdata < .95] = 0 airdata[airdata > 0.] = 1 bg_img = imdata * airdata if np.sum((bg_img > 0).astype(np.uint8)) < 100: return np.zeros_like(airdata) # Find the background threshold (the most frequently occurring value # excluding 0) bg_location = np.median(bg_img[bg_img > 0]) bg_spread = mad(bg_img[bg_img > 0]) bg_img[bg_img > 0] -= bg_location bg_img[bg_img > 0] /= bg_spread # Apply this threshold to the background voxels to identify voxels # contributing artifacts. qi1_img = np.zeros_like(bg_img) qi1_img[bg_img > zscore] = 1 qi1_img[distance < .10] = 0 # Create a structural element to be used in an opening operation. struc = nd.generate_binary_structure(3, 1) qi1_img = nd.binary_opening(qi1_img, struc).astype(np.uint8) qi1_img[airdata <= 0] = 0 return qi1_img
def split_exclusions(image, labels, exclusions, dilation=0, connectivity=1, standard_seeds=False): """Ensure that no segment in 'labels' overlaps more than one exclusion.""" labels = labels.copy() cur_label = labels.max() dilated_exclusions = exclusions.copy() foot = generate_binary_structure(exclusions.ndim, connectivity) for i in range(dilation): dilated_exclusions = grey_dilation(exclusions, footprint=foot) hashed = labels * (exclusions.max() + 1) + exclusions hashed[exclusions == 0] = 0 violations = bincount(hashed.ravel()) > 1 violations[0] = False if sum(violations) != 0: offending_labels = labels[violations[hashed]] mask = zeros(labels.shape, dtype=bool) for offlabel in offending_labels: mask += labels == offlabel if standard_seeds: seeds = label(mask * (image == 0))[0] else: seeds = label(mask * dilated_exclusions)[0] seeds[seeds > 0] += cur_label labels[mask] = watershed(image, seeds, connectivity, mask)[mask] return labels
def manual_split(probs, seg, body, seeds, connectivity=1, boundary_seeds=None): """Manually split a body from a segmentation using seeded watershed. Input: - probs: the probability of boundary in the volume given. - seg: the current segmentation. - body: the label to be split. - seeds: the seeds for the splitting (should be just two labels). [-connectivity: the connectivity to use for watershed.] [-boundary_seeds: if not None, these locations become inf in probs.] Value: - the segmentation with the selected body split. """ struct = generate_binary_structure(seg.ndim, connectivity) body_pixels = seg == body bbox = find_objects(body_pixels)[0] body_pixels = body_pixels[bbox] body_boundary = binary_dilation(body_pixels, struct) - body_pixels non_body_pixels = True - body_pixels - body_boundary probs = probs.copy()[bbox] probs[non_body_pixels] = probs.min()-1 if boundary_seeds is not None: probs[boundary_seeds[bbox]] = probs.max()+1 probs[body_boundary] = probs.max()+1 seeds = label(seeds.astype(bool)[bbox], struct)[0] outer_seed = seeds.max()+1 # should be 3 seeds[non_body_pixels] = outer_seed seg_new = watershed(probs, seeds, dams=(seg==0).any(), connectivity=connectivity, show_progress=True) seg = seg.copy() new_seeds = unique(seeds)[:-1] for new_seed, new_label in zip(new_seeds, [0, body, seg.max()+1]): seg[bbox][seg_new == new_seed] = new_label return seg
def get_largest_two_component(img, prt = False, threshold = None): s = ndimage.generate_binary_structure(3,2) # iterate structure labeled_array, numpatches = ndimage.label(img,s) # labeling sizes = ndimage.sum(img,labeled_array,range(1,numpatches+1)) sizes_list = [sizes[i] for i in range(len(sizes))] sizes_list.sort() #if(prt): # print('component size', sizes_list, flush = True) if(len(sizes) == 1): return img else: if(threshold): out_img = np.zeros_like(img) for temp_size in sizes_list: if(temp_size > threshold): temp_lab = np.where(sizes == temp_size)[0] + 1 temp_cmp = labeled_array == temp_lab out_img = (out_img + temp_cmp) > 0 return out_img else: max_size1 = sizes_list[-1] max_size2 = sizes_list[-2] max_label1 = np.where(sizes == max_size1)[0] + 1 max_label2 = np.where(sizes == max_size2)[0] + 1 component1 = labeled_array == max_label1 component2 = labeled_array == max_label2 #if(prt): # print(max_size2, max_size1, max_size2/max_size1, flush = True) if(max_size2*10 > max_size1): component1 = (component1 + component2) > 0 return component1
def __init__(self, label_image=None, connectivity=1, data=None, **attr): super(RAG, self).__init__(data, **attr) if self.number_of_nodes() == 0: self.max_id = 0 else: self.max_id = max(self.nodes_iter()) if label_image is not None: fp = ndi.generate_binary_structure(label_image.ndim, connectivity) # In the next ``ndi.generic_filter`` function, the kwarg # ``output`` is used to provide a strided array with a single # 64-bit floating point number, to which the function repeatedly # writes. This is done because even if we don't care about the # output, without this, a float array of the same shape as the # input image will be created and that could be expensive in # memory consumption. ndi.generic_filter( label_image, function=_add_edge_filter, footprint=fp, mode='nearest', output=as_strided(np.empty((1,), dtype=np.float_), shape=label_image.shape, strides=((0,) * label_image.ndim)), extra_arguments=(self,))
def test_labeling(): "Test cluster labeling" shape = flat_shape = (4, 20) pmap = np.empty(shape, np.float_) out = np.empty(shape, np.uint32) bin_buff = np.empty(shape, np.bool_) int_buff = np.empty(shape, np.uint32) struct = ndimage.generate_binary_structure(2, 1) struct[::2] = False conn = np.array([(0, 1), (0, 3), (1, 2), (2, 3)], np.uint32) criteria = None # some clusters pmap[:] = [[ 3, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0], [ 0, 1, 0, 0, 0, 0, 8, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 0], [ 0, 3, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4], [ 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0]] cids = _label_clusters(pmap, out, bin_buff, int_buff, 2, 0, struct, False, flat_shape, conn, criteria) assert_equal(len(cids), 6) # some other clusters pmap[:] = [[ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0], [ 0, 4, 0, 0, 0, 0, 0, 4, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0], [ 0, 0, 4, 4, 0, 4, 4, 0, 4, 0, 0, 0, 4, 4, 1, 0, 4, 4, 0, 0], [ 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0]] cids = _label_clusters(pmap, out, bin_buff, int_buff, 2, 0, struct, False, flat_shape, conn, criteria) assert_equal(len(cids), 6)
def f_returnInternalEdge(self,cl_array): # Internal edge: Count of neighboring non-zero cell kernel = ndimage.generate_binary_structure(2, 1) # Make a kernel kernel[1, 1] = 0 b = ndimage.convolve(cl_array, kernel, mode="constant") n_interior = b[cl_array != 0].sum() # Number of interiror edges return n_interior
def InDecPatch(self,which,amount): s = ndimage.generate_binary_structure(2,1) # taxi-cab struct if which == 0: ras = ndimage.binary_dilation(self.cl_array,s,iterations=amount,border_value=0) else: ras = ndimage.binary_erosion(self.cl_array,s,iterations=amount,border_value=0) return(ras)
def compute_sparsity(im): l_x = len(im) X, Y = np.ogrid[:l_x, :l_x] mask = ((X - l_x/2)**2 + (Y - l_x/2)**2 <= (l_x/2)**2) grad1 = ndimage.morphological_gradient(im, footprint=np.ones((3, 3))) grad2 = ndimage.morphological_gradient(im, footprint=ndimage.generate_binary_structure(2, 1)) return (grad1[mask] > 0).mean(), (grad2[mask] > 0).mean()
def noise_cube(data, mask=None, nThresh=30, iterations=1, do_map=True, do_spec=True, box=None, spec_box=None, bandpass_smooth_window=None, bandpass_smooth_order=3, oversample_boundary=False): """ Makes an empirical estimate of the noise in a cube assuming that it is normally distributed about zero. Treats the spatial and spectral dimensions as separable. Parameters: ----------- data : np.array Array of data (floats) Keywords: --------- mask : np.bool Boolean array with False indicating where data can be used in the noise estimate. (i.e., True is signal) do_map : np.bool Estimate spatial variations in the noise. Default is True. If set to False, all locations in a plane have the same noise estimate. do_spec : np.bool Estimate spectral variations in the noise. Default is True. If set to False, all channels in a spectrum have the same noise estimate. box : int Spatial size of the box over which noise is calculated in pixels. Default: no box, every pixel gets its own noise estimte. spec_box : int Spectral size of the box overwhich the noise is calculated. Default: no box, each channel gets its own noise estimate. nThresh : int Minimum number of data to be used in an individual noise estimate. iterations : int Number of times to iterate the noise solution to force Gaussian statistics. Default: no iterations. bandpass_smooth_window : int Number of channels used in bandpass smoothing kernel. Defaults to nChan / 4 where nChan number of channels. Set to zero to suppress smoothing. Uses Savitzky-Golay smoothing bandpass_smooth_order : int Polynomial order used in smoothing kernel. Defaults to 3. """ # TBD: add error checking # Create a mask that identifies voxels to be fitting the noise noisemask = np.isfinite(data) if mask is not None: noisemask[mask] = False # Default the spatial step size to individual pixels step = 1 halfbox = step // 2 # If the user has supplied a spatial box size, recast this into a # step size that critically samples the box and a halfbox size # used for convenience. if box is not None: step = np.floor(box / 2.5).astype(np.int) halfbox = int(box // 2) # Include all pixels adjacent to the spatial # boundary of the data as set by NaNs boundary = np.all(np.isnan(data), axis=0) if oversample_boundary: struct = nd.generate_binary_structure(2, 1) struct = nd.iterate_structure(struct, halfbox) rind = np.logical_xor(nd.binary_dilation(boundary, struct), boundary) extray, extrax = np.where(rind) else: extray, extrax = None, None # If the user has supplied a spectral box size, use this to # calculate a spectral step size. if spec_box is not None: spec_step = np.floor(spec_box / 2).astype(np.int) boxv = int(spec_box // 2) else: boxv = 0 # Default the bandpass smoothing window if bandpass_smooth_window is None: bandpass_smooth_window = 2 * (data.shape[0] // 8) + 1 # Initialize output to be used in the case of iterative # estimation. noise_cube_out = np.ones_like(data) # Iterate for ii in np.arange(iterations): if not do_map: # If spatial variations are turned off then estimate a # single value and fill the noise map with this value. noise_value = mad_zero_centered(data, mask=noisemask) noise_map = np.zeros(data.shape[1:]) + noise_value else: # Initialize map to be full of not-a-numbers noise_map = np.zeros(data.shape[1:]) + np.nan # Make a noise map xx = np.arange(data.shape[2]) yy = np.arange(data.shape[1]) # Sample starting at halfbox and stepping by step. In the # individual pixel limit this just samples every spectrum. xsamps = xx[halfbox::step] ysamps = yy[halfbox::step] xsampsf = (xsamps[np.newaxis, :] * (np.ones_like(ysamps))[:, np.newaxis]).flatten() ysampsf = (ysamps[:, np.newaxis] * np.ones_like(xsamps)).flatten() for x, y in zip(xsampsf, ysampsf): # Extract a minicube and associated mask from the cube minicube = data[:, (y - halfbox):(y + halfbox + 1), (x - halfbox):(x + halfbox + 1)] minicube_mask = noisemask[:, (y - halfbox):(y + halfbox + 1), (x - halfbox):(x + halfbox + 1)] # If we have enough data, fit a noise value for this entry if np.sum(minicube_mask) > nThresh: noise_map[y, x] = mad_zero_centered(minicube, mask=minicube_mask) if extrax is not None and extray is not None: for x, y in zip(extrax, extray): minicube = data[:, (y - halfbox):(y + halfbox + 1), (x - halfbox):(x + halfbox + 1)] minicube_mask = noisemask[:, (y - halfbox):(y + halfbox + 1), (x - halfbox):(x + halfbox + 1)] if np.sum(minicube_mask) > nThresh: noise_map[y, x] = mad_zero_centered(minicube, mask=minicube_mask) noise_map[boundary] = np.nan # If we are using a box size greater than an individual pixel # interpolate to fill in the noise map. if halfbox > 0: # Note the location of data, this is the location # where we want to fill in noise values. data_footprint = np.any(np.isfinite(data), axis=0) # Generate a smoothing kernel based on the box size. kernel = Gaussian2DKernel(box / np.sqrt(8 * np.log(2))) # Make a weight map to be used in the convolution, in # this weight map locations with measured values have # unity weight. This without measured values have zero # weight. # wt_map = np.isfinite(noise_map).astype(np.float) # wt_map[boundary] = np.nan # Take an average weighted by the kernel at each # location. # noise_map[np.isnan(noise_map)] = 0.0 # y, x = np.where(np.isfinite(noise_map)) # import scipy.interpolate as interp # func = interp.interp2d(x, y, noise_map[y, x], kind='cubic') noise_map = convolve(noise_map, kernel, boundary='extend') # yy, xx = np.indices(noise_map.shape) # noise_map_beta = interp.griddata((y, x), noise_map[y,x], # (yy, xx), method='cubic') # noise_map_beta = func(yy, xx) # noise_map_beta[boundary] = np.nan # noise_map = noise_map_beta # wt_map = convolve(wt_map, kernel, boundary='extend') # noise_map /= wt_map # Set the noise map to not-a-number outside the data # footprint. noise_map[~data_footprint] = np.nan # Initialize spectrum noise_spec = np.zeros(data.shape[0]) + np.nan if not do_spec: # If spectral variations are turned off then assume that # the noise_map describes all channels of the cube. pass else: # Loop over channels zz = np.arange(data.shape[0]) for z in zz: # Idententify the range of channels to be considered # in this estimate. lowz = np.clip(z - boxv, 0, data.shape[0]) hiz = np.clip(z + boxv + 1, 0, data.shape[0]) # Extract a slab from the cube and normalize it by the # noise map. Now any measured noise variations are # relative to those in the noise map. slab = data[lowz:hiz, :, :] / noise_map[np.newaxis, :, :] slab_mask = noisemask[lowz:hiz, :, :] noise_spec[z] = mad_zero_centered(slab, mask=slab_mask) # Smooth the spectral variations in the noise. if bandpass_smooth_window > 0: # Initialize a Savitzky-Golay filter then run it over # the noise spectrum. kernel = savgol_coeffs(int(bandpass_smooth_window), int(bandpass_smooth_order)) baddata = np.isnan(noise_spec) noise_spec = convolve(noise_spec, kernel, nan_treatment='interpolate', boundary='extend') noise_spec[baddata] = np.nan # Make sure that the noise spectrum is normalized by # setting the median to one. noise_spec /= np.nanmedian(noise_spec) # Combine the spatial and spectral variations into a # three-dimensional noise estimate. noise_cube = np.ones_like(data) noise_cube *= (noise_map[np.newaxis, :] * noise_spec[:, np.newaxis, np.newaxis]) if iterations == 1: return (noise_cube) else: # If iterating, normalize the data by the current noise # estimate and scale the current noise cube by the new # estimate. data = data / noise_cube noise_cube_out *= noise_cube # If iterating return the iterated noise cube. return (noise_cube_out)
def recipe_phangs_noise(incube=None, outfile=None, mask=None, noise_kwargs=None, return_spectral_cube=False, overwrite=False): """ Wrap noise_cube with a set of preferred parameters for the PHANGS-ALMA CO work. Parameters: ----------- cube : np.array Array of data (floats) Keywords: --------- mask : np.bool Boolean array with False indicating where data can be used in the noise estimate. (i.e., True is signal). """ # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&% # Error checking and work out inputs # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&% if type(incube) is SpectralCube: cube = incube elif type(incube) == str: cube = SpectralCube.read(incube) else: logger.error("Input must be a SpectralCube object or a filename.") # Initialize an empty kwargs dictionary if noise_kwargs is None: noise_kwargs = {} # If no box is specified, default to one about two beams across if 'box' not in noise_kwargs: pixels_per_beam = cube.pixels_per_beam box = np.ceil(2.5 * pixels_per_beam**0.5) noise_kwargs['box'] = box # Default to an odd bandpass smothing window if 'bandpass_smooth_window' not in noise_kwargs: spectral_smooth = np.ceil(cube.shape[0] / 5) // 2 * 2 + 1 noise_kwargs['bandpass_smooth_window'] = spectral_smooth if 'spec_box' not in noise_kwargs: noise_kwargs['spec_box'] = 5 if 'iterations' not in noise_kwargs: noise_kwargs['iterations'] = 4 # Require a valid cube input as a if mask is not None: if type(mask) is SpectralCube: noise_kwargs['mask'] = mask elif type(mask) == type("hello"): noise_kwargs['mask'] = SpectralCube.read(mask) else: logger.error( "Mask must be a SpectralCube object or a filename or None.") # Fill in the mask if it hasn't already been filled in. if 'mask' not in noise_kwargs: # Check if a non-trivial signal mask is attached to the cube if (np.sum(cube.mask.include()) < np.sum( np.isfinite(cube.filled_data[:].value))): noise_kwargs['mask'] = cube.mask.include() else: noise_kwargs['mask'] = None # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&% # Run the noise estimate # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&% data = cube.filled_data[:].value badmask = np.isnan(data) badmask = nd.binary_dilation(badmask, structure=nd.generate_binary_structure(3, 2)) data[badmask] = np.nan rms = noise_cube(data, **noise_kwargs) # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&% # Write or return as requested # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&% # In this case can avoid a recast if not return_spectral_cube and (outfile is None): return (rms) # Recast from numpy array to spectral cube header = cube.header header['DATAMIN'] = np.nanmin(rms) header['DATAMAX'] = np.nanmax(rms) header['COMMENT'] = 'Produced with PHANGS-ALMA pipeline version ' + version if tableversion: header[ 'COMMENT'] = 'Galaxy properties from PHANGS sample table version ' + tableversion rms = SpectralCube(rms, wcs=cube.wcs, header=header, meta={'BUNIT': cube.header['BUNIT']}) # Write to disk, if desired if outfile is not None: rms.write(outfile, overwrite=overwrite) if return_spectral_cube: return (rms) else: return (rms.filled_data[:].value)
def process(self, input=None, output_folder=None, progress_callback=None, filter=None, correction_factor=2, cutoff_cell_fusion=None, restore_safe_cells=False, _DEBUG=False, _VISUAL_DEBUG=False, **kwargs): start = timer() # filename0 = path # filename0_without_path = os.path.basename(filename0) # filename0_without_ext = os.path.splitext(filename0_without_path)[0] # parent_dir_of_filename0 = os.path.dirname(filename0) # TA_output_filename = os.path.join(parent_dir_of_filename0, filename0_without_ext, # 'handCorrection.tif') # TODO allow custom names here to allow ensemble methods # non_TA_final_output_name = os.path.join(output_folder, filename0_without_ext + '.tif') # # filename_to_use_to_save = non_TA_final_output_name # if TA_mode: # filename_to_use_to_save = TA_output_filename # # if TA_mode: # # try also to change path input name # if os.path.exists( # os.path.join(parent_dir_of_filename0, filename0_without_ext, 'raw_epyseg_output.tif')): # path = os.path.join(parent_dir_of_filename0, filename0_without_ext, 'raw_epyseg_output.tif') # img_orig = Img(path) # print('analyzing', path, self.stop_now) # try: # if self.progress_callback is not None: # self.progress_callback.emit((iii / len(list_of_files)) * 100) # else: # logger.info(str((iii / len(list_of_files)) * 100) + '%') # except: # traceback.print_exc() # pass # DO A DILATION OF SEEDS THEN AN EROSION TO JOIN CLOSE BY SEEDS img_orig = input img_has_seeds = True # mask with several channels if img_orig.has_c(): if restore_safe_cells: img_seg = img_orig[..., 0].copy() seeds_1 = img_orig[..., img_orig.shape[-1] - 1] seeds_1 = Img.invert(seeds_1) # seeds_1[seeds_1 >= 0.5] = 255 # seeds_1[seeds_1 < 0.5] = 0 seeds_1[seeds_1 >= 0.2] = 255 # TODO maybe be more stringent here seeds_1[seeds_1 < 0.2] = 0 s = ndimage.generate_binary_structure(2, 1) seeds_1 = ndimage.grey_dilation(seeds_1, footprint=s) seeds_1 = ndimage.grey_dilation(seeds_1, footprint=s) seeds_1 = ndimage.grey_dilation(seeds_1, footprint=s) seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s) seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s) # seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s) # seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s) # for debug if _DEBUG: Img(seeds_1, dimensions='hw').save( os.path.join(output_folder, 'extras', 'wshed_seeds.tif')) # not bad lab_seeds = label(seeds_1.astype(np.uint8), connectivity=2, background=0) # for region in regionprops(lab_seeds): if region.area < 10: for coordinates in region.coords: lab_seeds[coordinates[0], coordinates[1]] = 0 if _DEBUG: Img(seeds_1, dimensions='hw').save( os.path.join(output_folder, 'extras', 'wshed_seeds_deblobed.tif')) img_orig[..., 3] = Img.invert(img_orig[..., 3]) img_orig[..., 4] = Img.invert(img_orig[..., 4]) # seems to work --> now need to do the projection for c in range(1, img_orig.shape[-1] - 2): img_orig[..., 0] += img_orig[..., 1] img_orig[..., 0] /= img_orig.shape[-1] - 2 img_orig = img_orig[..., 0] else: # mask with single channel img_has_seeds = False if restore_safe_cells: img_seg = img_orig.copy() if restore_safe_cells: if _DEBUG: print(os.path.join(output_folder, 'extras', 'img_seg.tif')) Img(img_seg, dimensions='hw').save( os.path.join(output_folder, 'extras', 'img_seg.tif')) # for debug if _DEBUG: Img(img_orig, dimensions='hw').save(os.path.join(output_folder, 'extras', 'avg.tif')) img_saturated = img_orig.copy() if img_has_seeds: img_saturated[img_saturated >= 0.5] = 255 img_saturated[img_saturated < 0.5] = 0 if restore_safe_cells: # TODO maybe do a safe image img_seg[img_seg >= 0.3] = 255 img_seg[img_seg < 0.3] = 0 secure_mask = img_seg else: img_saturated[img_saturated >= 0.3] = 255 img_saturated[img_saturated < 0.3] = 0 if restore_safe_cells: img_seg[img_seg >= 0.95] = 255 img_seg[img_seg < 0.95] = 0 secure_mask = img_seg # convert it to seeds and make sure they are all present in there # if pixel is not labeled then read it if restore_safe_cells: labels_n_area_rescue_seeds = {} rescue_seeds = label(Img.invert(secure_mask), connectivity=1, background=0) for region in regionprops(rescue_seeds): labels_n_area_rescue_seeds[region.label] = region.area if _DEBUG: Img(secure_mask, dimensions='hw').save(os.path.join(output_folder, 'extras', 'secure_mask.tif')) # loop over those seeds to rescue # for debug if _DEBUG: Img(img_saturated, dimensions='hw').save( os.path.join(output_folder, 'extras', 'handCorrection.tif')) deblob = True if deblob: image_thresh = label(img_saturated, connectivity=2, background=0) # for debug if _DEBUG: Img(image_thresh, dimensions='hw').save( os.path.join(output_folder, 'extras', 'before_deblobed.tif')) # deblob min_size = 200 for region in regionprops(image_thresh): # take regions with large enough areas if region.area < min_size: for coordinates in region.coords: image_thresh[coordinates[0], coordinates[1]] = 0 image_thresh[image_thresh > 0] = 255 img_saturated = image_thresh # for debug if _DEBUG: Img(img_saturated, dimensions='hw').save( os.path.join(output_folder, 'extras', 'deblobed.tif')) del image_thresh # for debug if _DEBUG: Img(img_saturated, dimensions='hw').save( os.path.join(output_folder, 'extras', 'deblobed_out.tif')) extra_dilations = True if extra_dilations: # do a dilation of 2 to close bonds s = ndimage.generate_binary_structure(2, 1) dilated = ndimage.grey_dilation(img_saturated, footprint=s) dilated = ndimage.grey_dilation(dilated, footprint=s) # Img(dilated, dimensions='hw').save(os.path.join(os.path.splitext(path)[0], 'filled_one_px_holes.tif')) # other_seeds = label(invert(np.grey_dilation(dilated, footprint=s).astype(np.uint8)), connectivity=1, background=0) labs = label(Img.invert(img_saturated.astype(np.uint8)), connectivity=1, background=0) for region in regionprops(labs): seeds = [] # exclude tiny cells form dilation because they may end up completely closed if region.area >= 10 and region.area < 350: for coordinates in region.coords: dilated[coordinates[0], coordinates[1]] = 0 continue else: # pb when big cells around cause connections are not done # preserve cells at edges because they have to e naturally smaller because they are cut # put a size criterion too if region.area < 100 and ( region.bbox[0] <= 1 or region.bbox[1] <= 1 or region.bbox[2] >= labs.shape[-2] - 2 or region.bbox[ 3] >= \ labs.shape[-1] - 2): # edge cell detected --> removing dilation for coordinates in region.coords: dilated[coordinates[0], coordinates[1]] = 0 continue img_saturated = dilated # for debug if _DEBUG: Img(img_saturated, dimensions='hw').save( os.path.join(output_folder, 'extras', 'dilated_further.tif')) del dilated list_of_cells_to_dilate = [] labs = label(Img.invert(img_saturated.astype(np.uint8)), connectivity=1, background=0) # c'est cette correction qui fixe bcp de choses mais recree aussi des choses qui n'existent pas... --> voir à quoi sont dus ces lignes blobs # faudrait redeblober if img_has_seeds: for region in regionprops(labs, intensity_image=img_orig): seeds = [] if not extra_dilations and region.area < 10: continue # if small and no associated seeds --> remove it ??? maybe or not for coordinates in region.coords: id = lab_seeds[coordinates[0], coordinates[1]] if id != 0: seeds.append(id) seeds = set(seeds) if len(seeds) >= 2: # we may have found an undersegmented cell --> try segment it better list_of_cells_to_dilate.append(region.label) if len(list_of_cells_to_dilate) != 0: props = regionprops(labs, intensity_image=img_orig) for run in range(10): something_changed = False # early stop for region in props: if region.label not in list_of_cells_to_dilate: continue # TODO recheck those values and wether it makes sense threshold_values = [80 / 255, 60 / 255, 40 / 255, 30 / 255, 20 / 255, 10 / 255] # 160 / 255, 140 / 255, 120 / 255, 100 / 255, 1 / 255 , 2 / 255, , 5 / 255 try: for threshold in threshold_values: mask = region.image.copy() image = region.image.copy() image[region.intensity_image > threshold] = True image[region.intensity_image <= threshold] = False final = Img.invert(image.astype(np.uint8)) final[final < 255] = 0 final[mask == False] = 0 new_seeds = label(final, connectivity=1, background=0) props2 = regionprops(new_seeds) if len(props2) > 1: # cell was resplitted into smaller for r in props2: if r.area < 20: raise Exception region.image[mask == False] = False region.image[mask == True] = True region.image[new_seeds > 0] = False something_changed = True for coordinates in region.coords: img_saturated[coordinates[0], coordinates[1]] = 255 region.image[mask == False] = False region.image[mask == True] = True del final del new_seeds except: traceback.print_exc() pass if not something_changed: # print('no more changes anymore --> quitting') break # for debug if _DEBUG: Img(img_saturated, dimensions='hw').save( os.path.join(output_folder, 'extras', 'saturated_mask4.tif')) final_seeds = label(Img.invert(img_saturated), connectivity=1, background=0) # keep like that otherwise creates tiny cells with erroneous wshed # for debug if _DEBUG: Img(final_seeds, dimensions='hw').save( os.path.join(output_folder, 'extras', 'final_seeds_before.tif')) final_seeds = label(Img.invert(img_saturated), connectivity=2, background=0) # is that needed ??? # for debug if _DEBUG: Img(final_seeds, dimensions='hw').save( os.path.join(output_folder, 'extras', 'final_seeds_before2.tif')) final_seeds[img_saturated == 255] = 0 final_wshed = watershed(img_orig, markers=final_seeds, watershed_line=True) final_wshed[final_wshed != 0] = 1 # remove all seeds final_wshed[final_wshed == 0] = 255 # set wshed values to 255 final_wshed[final_wshed == 1] = 0 # set all other cell content to # filename0 = os.path.basename(path) # parent_path = os.path.dirname(os.path.dirname(path)) if filter is None or filter == 0: # TODO maybe offer the choice between saving wshed on predict or on orig # Img(final_wshed, dimensions='hw').save(os.path.join(output_folder, os.path.splitext(filename0)[ # 0]) + '.tif') # need put original name here TODO put image default name here # print('saving', filename_to_use_to_save) # Img(final_wshed.astype(np.uint8), dimensions='hw').save(filename_to_use_to_save) return final_wshed.astype(np.uint8) else: if isinstance(filter, int): filter_by_size = filter else: filter_by_size = None avg_area = 0 count = 0 if _DEBUG: Img(final_wshed, dimensions='hw').save(os.path.join(output_folder, 'extras', 'test_size_cells.tif')) final_seeds = Img.invert(final_wshed) final_seeds = label(final_seeds, connectivity=1, background=0) if _VISUAL_DEBUG: plt.imshow(final_seeds) plt.show() removed_seeds = [] keep_seeds = [] labels_n_bbox = {} labels_n_area = {} border_cells = [] ids_n_local_median = {} correspondance_between_cur_seeds_and_safe_ones = {} if isinstance(filter, str) and 'local' in filter: rps = regionprops(final_seeds) for region in rps: labels_n_bbox[region.label] = region.bbox labels_n_area[region.label] = region.area if (region.bbox[0] <= 3 or region.bbox[1] <= 3 or region.bbox[2] >= final_seeds.shape[-2] - 5 or region.bbox[ 3] >= \ final_seeds.shape[-1] - 5): border_cells.append(region.label) if restore_safe_cells: for coordinates in region.coords: if rescue_seeds[coordinates[0], coordinates[1]] != 0: # do r correspondance_between_cur_seeds_and_safe_ones[region.label] = rescue_seeds[ coordinates[0], coordinates[1]] break break _, tiles = Img.get_2D_tiles_with_overlap(final_seeds, overlap=64, dimension_h=-2, dimension_w=-1) for r in tiles: for tile in r: rps2 = regionprops(tile) for region in rps2: if self.stop_now: return if region.label in border_cells: continue if (region.bbox[0] <= 3 or region.bbox[1] <= 3 or region.bbox[2] >= final_seeds.shape[ -2] - 5 or region.bbox[ 3] >= \ final_seeds.shape[-1] - 5): continue area_of_neighboring_cells = [] for region2 in rps2: if region2.label == region.label: continue # find all cells with if self.rect_distance(region.bbox, region2.bbox) <= 1: area_of_neighboring_cells.append(labels_n_area[region2.label]) if area_of_neighboring_cells: median = statistics.median_low(area_of_neighboring_cells) ids_n_local_median[ region.label] = median / correction_factor if region.area <= median / correction_factor: removed_seeds.append(region.label) else: keep_seeds.append(region.label) removed_seeds = [x for x in removed_seeds if x not in keep_seeds] # TODO offer the things below as an option --> prevent removal of sure seeds or something like that if restore_safe_cells: removed_seeds_to_restore = [] for region in regionprops(final_seeds): if region.label in removed_seeds: first = True for coordinates in region.coords: if first and rescue_seeds[coordinates[0], coordinates[1]] != 0: percent_diff = min(labels_n_area[region.label], labels_n_area_rescue_seeds[ rescue_seeds[coordinates[0], coordinates[1]]]) / max( labels_n_area[region.label], labels_n_area_rescue_seeds[ rescue_seeds[coordinates[0], coordinates[1]]]) if (percent_diff >= 0.7 and percent_diff < 1.0) or ( labels_n_area[region.label] <= 200 and ( percent_diff >= 0.3 and percent_diff < 1.0)): if _DEBUG: print('0 finally not removing seed, safe seed', region.label, percent_diff, labels_n_area[region.label], labels_n_area_rescue_seeds[ rescue_seeds[coordinates[0], coordinates[1]]], labels_n_area[region.label] / labels_n_area_rescue_seeds[ rescue_seeds[coordinates[0], coordinates[1]]], region.centroid) removed_seeds_to_restore.append(region.label) break break removed_seeds = [x for x in removed_seeds if x not in removed_seeds_to_restore] else: areas = [] for region in regionprops(final_seeds): if (region.bbox[0] <= 3 or region.bbox[1] <= 3 or region.bbox[2] >= final_seeds.shape[-2] - 5 or region.bbox[3] >= final_seeds.shape[-1] - 5): continue avg_area += region.area count += 1 areas.append(region.area) avg_area /= count median = statistics.median_low(areas) if isinstance(filter, int): filter_by_size = filter elif 'avg' in filter: filter_by_size = avg_area / correction_factor elif 'median' in filter: filter_by_size = median / correction_factor # TODO maybe use stdev or alike to see if cell should really be removed if _DEBUG: print('filter cells below=', filter_by_size, 'avg cell area=', avg_area, 'median=', median) # , 'median', median if filter_by_size is not None and filter_by_size != 0: if _VISUAL_DEBUG: plt.imshow(final_seeds) plt.show() for region in regionprops(final_seeds): labels_n_bbox[region.label] = region.bbox labels_n_area[region.label] = region.area if region.area < filter_by_size: if (region.bbox[0] <= 2 or region.bbox[1] <= 2 or region.bbox[2] >= labs.shape[ -2] - 3 or region.bbox[ 3] >= \ labs.shape[ -1] - 3): continue removed_seeds.append(region.label) if cutoff_cell_fusion is not None and cutoff_cell_fusion > 1: cells_to_fuse = [] for idx, removed_seed in enumerate(removed_seeds): current_cells_to_fuse = set() closest_pair = None smallest_distance = None for idx2 in range(idx + 1, len(removed_seeds)): removed_seed2 = removed_seeds[idx2] if closest_pair is None: if self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) <= 1: closest_pair = removed_seed2 smallest_distance = self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) elif self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) <= smallest_distance: closest_pair = removed_seed2 smallest_distance = self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) if self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) <= 1: current_cells_to_fuse.add(removed_seed) current_cells_to_fuse.add(removed_seed2) if current_cells_to_fuse: cells_to_fuse.append(current_cells_to_fuse) cells_to_fuse = [frozenset(i) for i in cells_to_fuse] cells_to_fuse = list(dict.fromkeys(cells_to_fuse)) cells_to_keep = [] if cutoff_cell_fusion is not None and cutoff_cell_fusion > 0: superfuse = [] copy_of_cells_to_fuse = cells_to_fuse.copy() for idx, fuse in enumerate(copy_of_cells_to_fuse): current_fusion = set(fuse.copy()) changed = True while changed: changed = False for idx2 in range(len(copy_of_cells_to_fuse) - 1, idx, -1): fuse2 = copy_of_cells_to_fuse[idx2] if idx2 == idx: continue if fuse2.intersection(current_fusion): current_fusion.update(fuse2) del copy_of_cells_to_fuse[idx2] changed = True superfuse.append(current_fusion) for sf in superfuse: if len(sf) > cutoff_cell_fusion: for val in sf: cells_to_keep.append(val) seeds_to_fuse = [] cells_to_fuse = sorted(cells_to_fuse, key=len) for fuse in cells_to_fuse: cumulative_area = 0 for _id in fuse: if _id in cells_to_keep: if _id in removed_seeds: removed_seeds.remove(_id) continue cumulative_area += labels_n_area[_id] if filter_by_size is not None: if cumulative_area >= filter_by_size: #: #1200: #filter_by_size: # need hack this to get local area seeds_to_fuse.append(fuse) for _id in fuse: if _id in removed_seeds: removed_seeds.remove(_id) else: if cumulative_area >= ids_n_local_median[_id]: seeds_to_fuse.append(fuse) for _id in fuse: if _id in removed_seeds: removed_seeds.remove(_id) # need recolor all the seeds in there with the new seed stuff for fuse in seeds_to_fuse: for _id in fuse: break for region in regionprops(final_seeds): if region.label in fuse: for coordinates in region.coords: final_seeds[coordinates[0], coordinates[1]] = _id if _VISUAL_DEBUG: plt.imshow(final_seeds) plt.show() for region in regionprops(final_seeds): if region.label in removed_seeds: for coordinates in region.coords: final_seeds[coordinates[0], coordinates[1]] = 0 if _VISUAL_DEBUG: plt.imshow(final_seeds) plt.show() if _VISUAL_DEBUG: plt.imshow(final_seeds) plt.show() final_wshed = watershed(img_orig, markers=final_seeds, watershed_line=True) final_wshed[final_wshed != 0] = 1 # remove all seeds final_wshed[final_wshed == 0] = 255 # set wshed values to 255 final_wshed[final_wshed == 1] = 0 # set all other cell content to if _VISUAL_DEBUG: plt.imshow(final_wshed) plt.show() # print('saving', filename_to_use_to_save) # Img(final_wshed.astype(np.uint8), dimensions='hw').save(filename_to_use_to_save) duration = timer() - start if _DEBUG: print('final duration wshed in secs', duration) return final_wshed.astype(np.uint8) # is indeed a 2D image
def _deblend_source(data, segment_img, npixels, nlevels=32, contrast=0.001, mode='exponential', connectivity=8): """ Deblend a single labeled source. Parameters ---------- data : array_like The 2D array of the image. The should be a cutout for a single source. ``data`` should already be smoothed by the same filter used in :func:`~photutils.detect_sources`, if applicable. segment_img : `~photutils.segmentation.SegmentationImage` A cutout `~photutils.segmentation.SegmentationImage` object with the same shape as ``data``. ``segment_img`` should contain only *one* source label. npixels : int The number of connected pixels, each greater than ``threshold``, that an object must have to be detected. ``npixels`` must be a positive integer. nlevels : int, optional The number of multi-thresholding levels to use. Each source will be re-thresholded at ``nlevels``, spaced exponentially or linearly (see the ``mode`` keyword), between its minimum and maximum values within the source segment. contrast : float, optional The fraction of the total (blended) source flux that a local peak must have to be considered as a separate object. ``contrast`` must be between 0 and 1, inclusive. If ``contrast = 0`` then every local peak will be made a separate object (maximum deblending). If ``contrast = 1`` then no deblending will occur. The default is 0.001, which will deblend sources with a magnitude differences of about 7.5. mode : {'exponential', 'linear'}, optional The mode used in defining the spacing between the multi-thresholding levels (see the ``nlevels`` keyword). connectivity : {4, 8}, optional The type of pixel connectivity used in determining how pixels are grouped into a detected source. The options are 4 or 8 (default). 4-connected pixels touch along their edges. 8-connected pixels touch along their edges or corners. For reference, SExtractor uses 8-connected pixels. Returns ------- segment_image : `~photutils.segmentation.SegmentationImage` A 2D segmentation image, with the same shape as ``data``, where sources are marked by different positive integer values. A value of zero is reserved for the background. """ from scipy import ndimage from skimage.morphology import watershed if nlevels < 1: raise ValueError('nlevels must be >= 1, got "{0}"'.format(nlevels)) if contrast < 0 or contrast > 1: raise ValueError('contrast must be >= 0 or <= 1, got ' '"{0}"'.format(contrast)) if connectivity == 4: selem = ndimage.generate_binary_structure(2, 1) elif connectivity == 8: selem = ndimage.generate_binary_structure(2, 2) else: raise ValueError('Invalid connectivity={0}. ' 'Options are 4 or 8'.format(connectivity)) segm_mask = (segment_img.data > 0) source_values = data[segm_mask] source_min = np.min(source_values) source_max = np.max(source_values) if source_min == source_max: return segment_img # no deblending if source_min < 0: warnings.warn( 'Source "{0}" contains negative values, setting ' 'deblending mode to "linear"'.format(segment_img.labels[0]), AstropyUserWarning) mode = 'linear' source_sum = float(np.sum(source_values)) steps = np.arange(1., nlevels + 1) if mode == 'exponential': if source_min == 0: source_min = source_max * 0.01 thresholds = source_min * ((source_max / source_min)**(steps / (nlevels + 1))) elif mode == 'linear': thresholds = source_min + ((source_max - source_min) / (nlevels + 1)) * steps else: raise ValueError('"{0}" is an invalid mode; mode must be ' '"exponential" or "linear"') # create top-down tree of local peaks segm_tree = [] for level in thresholds[::-1]: segm_tmp = detect_sources(data, level, npixels=npixels, connectivity=connectivity) if segm_tmp.nlabels >= 2: fluxes = [] for i in segm_tmp.labels: fluxes.append(np.sum(data[segm_tmp == i])) idx = np.where((np.array(fluxes) / source_sum) >= contrast)[0] if len(idx >= 2): segm_tree.append(segm_tmp) nbranch = len(segm_tree) if nbranch == 0: return segment_img else: for j in np.arange(nbranch - 1, 0, -1): intersect_mask = (segm_tree[j].data * segm_tree[j - 1].data).astype(bool) intersect_labels = np.unique(segm_tree[j].data[intersect_mask]) if segm_tree[j - 1].nlabels <= len(intersect_labels): segm_tree[j - 1] = segm_tree[j] else: # If a higher tree level has more peaks than in the # intersected label(s) with the level below, then remove # the intersected label(s) in the lower level, add the # higher level, and relabel. segm_tree[j].remove_labels(intersect_labels) new_segments = segm_tree[j].data + segm_tree[j - 1].data new_segm, nsegm = ndimage.label(new_segments) segm_tree[j - 1] = SegmentationImage(new_segm) return SegmentationImage( watershed(-data, segm_tree[0].data, mask=segment_img.data, connectivity=selem))
def count_cells(self, rawim, probim, maskim): raw = rawim.copy() # binarize probability # THIS ASSUMES THAT probim HAS VALUE RANGE OF [0,255]! # This normalization is handled by Reader process binary = (probim > self.prob_threshold * 255) # apply mask binary[maskim] = 0 # define connectivity struct = ndi.generate_binary_structure(3, 1) # label objects lbl, _ = ndi.label(binary, struct) # unique ids and counts unique, counts = np.unique(lbl, return_counts=True) # if no object was found, continue if unique.max() == 0: return [] # filter objects by its volume small, medium, large = [], [], [] for uq, ct in zip(unique, counts): if uq == 0: continue # skip zero! if ct <= self.min_volume: small.append([ uq, ct ]) # if object is smaller than mimimum size, it gets excluded elif self.min_volume < ct <= self.max_volume: medium.append([uq, ct]) else: large.append([uq, ct]) # list to store detected objects detected_obj = [] # take care of medium objects obj_ids = [e[0] for e in medium] volumes = [e[1] for e in medium] if obj_ids: # skip if empty coms = ndi.center_of_mass(binary, lbl, obj_ids) coms = np.array(coms).round().astype(np.int) # convert to integer for i, com in enumerate(coms): this_idx = obj_ids[i] deltaI, bg = self._get_intensity(raw, lbl, this_idx, com, self.intensity_mode, self.local_max_rad, self.local_min_rad, self.local_min_percentile) vol = volumes[i] obj = [com[2], com[1], com[0], deltaI, bg, vol] # X, Y, Z, intensity, volume detected_obj.append(obj) # take care of large objects obj_ids = [e[0] for e in large] if obj_ids: # skip if empty coms = self._com_large_obj(raw, lbl, obj_ids) # centers of mass and its ID for com in coms: this_idx = com[3] xyz = com[0:3] deltaI, bg = self._get_intensity( raw, lbl, this_idx, xyz, mode="local_mean", max_rad=2, min_rad=self.local_min_rad, min_percentile=self.local_min_percentile) vol = self.max_volume obj = [com[2], com[1], com[0], deltaI, bg, vol] # X, Y, Z, intensity, volume detected_obj.append(obj) return detected_obj
dice = [] for f in files: gt_file = 'gt/' + f img_gt = nib.load(gt_file).get_data() > 0 pred_file_1 = 'pred_whole_tumor_axial/' + f[:-12] + '_niftynet_out.nii.gz' pred_file_2 = 'pred_whole_tumor_coronal/' + f[:-12] + '_niftynet_out.nii.gz' pred_file_3 = 'pred_whole_tumor_sagittal/' + f[:-12] + '_niftynet_out.nii.gz' img_pred_1 = nib.load(pred_file_1).get_data()[..., 0, 1] img_pred_2 = nib.load(pred_file_2).get_data()[..., 0, 1] img_pred_3 = nib.load(pred_file_3).get_data()[..., 0, 1] img_pred = (img_pred_1 + img_pred_2 + img_pred_3) / 3.0 #img_pred = img_pred_1 #img_pred = img_pred_2 #img_pred = img_pred_3 img_pred = img_pred > 0.5 struct = ndimage.generate_binary_structure(3, 2) img_pred = ndimage.morphology.binary_closing(img_pred, structure=struct) img_pred = get_largest_two_component(img_pred, False, 2000) true_pos = np.float(np.sum(img_gt * img_pred)) union = np.float(np.sum(img_gt) + np.sum(img_pred)) d = true_pos * 2.0 / union print('%s: %s' % (f[:-12], d)) dice.append(d) print('%s images mean %s std %s' % (len(dice), np.mean(dice), np.std(dice)))
def _binary_significant_features(radar, binary_field, size_bins=75, size_limits=(0, 300), structure=None, debug=False, verbose=False): """ Objectively determine the minimum echo feature size (in radar gates) and remove features smaller than this from the specified radar field. This function can be used to objectively remove salt and pepper noise from binary (mask) radar fields. Unexpected results may occur if the specified radar field is not a binary field. Parameters ---------- radar : Radar Radar object containing the specified binary field. binary_field : str The binary radar field that will have insignificant echo features removed. size_bins : int, optional Number of bins used to bin echo feature sizes and thus define its distribution. size_limits : list or tuple, optional Limits of the echo feature size distribution. The upper limit needs to be large enough to include the minimum feature size. The size bin width is defined by both the size_bins and size_limits parameters. structure : array_like, optional Binary structuring element used to define connected features. The default structuring element has a squared connectivity equal to one. debug, verbose : bool, optional True to print debugging and progress information, respectively, False to suppress. """ # Parse binary structuring element if structure is None: structure = ndimage.generate_binary_structure(2, 1) # Parse feature size arrays size_data = np.zeros_like(radar.fields[binary_field]['data'], dtype=np.int32) feature_sizes = [] for i, sweep in enumerate(radar.iter_slice()): # Parse radar sweep data # Non-zero elements of the array form the subset to be dilated data = radar.fields[binary_field]['data'][sweep] is_valid_gate = np.ma.filled(data, 0) # Label the connected features in the sweep data and create index # array which defines each unique labeled feature labels, nlabels = ndimage.label(is_valid_gate, structure=structure, output=None) index = np.arange(1, nlabels + 1, 1) if debug: print 'Unique features in sweep {}: {}'.format(i, nlabels) # Compute the size in radar gates of each labeled feature sweep_sizes = ndimage.labeled_comprehension(is_valid_gate, labels, index, np.count_nonzero, np.int32, 0) feature_sizes.append(sweep_sizes) # Set each labeled feature to its total size in radar gates for label, size in zip(index, sweep_sizes): size_data[sweep][labels == label] = size feature_sizes = np.hstack(feature_sizes) # Bin and count occurrences of labeled feature sizes # Compute bin centers and bin width counts, bin_edges = np.histogram(feature_sizes, bins=size_bins, range=size_limits, normed=False, weights=None, density=False) bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0 bin_width = np.diff(bin_edges).mean() if debug: print 'Feature size bin width: {} gate(s)'.format(bin_width) # Compute the peak of the labeled feature size distribution # We expect the peak of this distribution to be close to 1 radar gate peak_size = bin_centers[counts.argmax()] - bin_width / 2.0 if debug: print 'Feature size at peak: {} gate(s)'.format(peak_size) # Determine the first instance when the count (sample size) of a feature # size bin reaches 0 in the right side of the feature size distribution # This will define the minimum feature size is_zero_size = np.logical_and(bin_centers > peak_size, np.isclose(counts, 0, atol=1.0e-5)) min_size = bin_centers[is_zero_size].min() - bin_width / 2.0 if debug: _range = [0.0, min_size] print 'Insignificant feature size range: {} gates'.format(_range) # Remove insignificant features from the binary radar field radar.fields[binary_field]['data'][size_data < min_size] = 0 return
def _raveled_offsets_and_distances( image_shape, *, footprint=None, connectivity=1, center=None, spacing=None, order='C', ): """Compute offsets to neighboring pixels in raveled coordinate space. This function also returns the corresponding distances from the center pixel given a spacing (assumed to be 1 along each axis by default). Parameters ---------- image_shape : tuple of int The shape of the image for which the offsets are being computed. footprint : array of bool The footprint of the neighborhood, expressed as an n-dimensional array of 1s and 0s. If provided, the connectivity argument is ignored. connectivity : {1, ..., ndim} The square connectivity of the neighborhood: the number of orthogonal steps allowed to consider a pixel a neighbor. See `scipy.ndimage.generate_binary_structure`. Ignored if footprint is provided. center : tuple of int Tuple of indices to the center of the footprint. If not provided, it is assumed to be the center of the footprint, either provided or generated by the connectivity argument. spacing : tuple of float The spacing between pixels/voxels along each axis. order : 'C' or 'F' The ordering of the array, either C or Fortran ordering. Returns ------- raveled_offsets : ndarray Linear offsets to a samples neighbors in the raveled image, sorted by their distance from the center. distances : ndarray The pixel distances correspoding to each offset. Notes ----- This function will return values even if `image_shape` contains a dimension length that is smaller than `footprint`. Examples -------- >>> off, d = _raveled_offsets_and_distances( ... (4, 5), footprint=np.ones((4, 3)), center=(1, 1) ... ) >>> off array([-5, -1, 1, 5, -6, -4, 4, 6, 10, 9, 11]) >>> d[0] 1.0 >>> d[-1] # distance from (1, 1) to (3, 2) 2.236... """ ndim = len(image_shape) if footprint is None: footprint = ndi.generate_binary_structure(rank=ndim, connectivity=connectivity) if center is None: center = tuple(s // 2 for s in footprint.shape) if not footprint.ndim == ndim == len(center): raise ValueError( "number of dimensions in image shape, footprint and its" "center index does not match") offsets = np.stack([(idx - c) for idx, c in zip(np.nonzero(footprint), center)], axis=-1) if order == 'F': offsets = offsets[:, ::-1] image_shape = image_shape[::-1] elif order != 'C': raise ValueError("order must be 'C' or 'F'") # Scale offsets in each dimension and sum ravel_factors = image_shape[1:] + (1, ) ravel_factors = np.cumprod(ravel_factors[::-1])[::-1] raveled_offsets = (offsets * ravel_factors).sum(axis=1) # Sort by distance if spacing is None: spacing = np.ones(ndim) weighted_offsets = offsets * spacing distances = np.sqrt(np.sum(weighted_offsets**2, axis=1)) sorted_raveled_offsets = raveled_offsets[np.argsort(distances)] sorted_distances = np.sort(distances) # If any dimension in image_shape is smaller than footprint.shape # duplicates might occur, remove them if any(x < y for x, y in zip(image_shape, footprint.shape)): # np.unique reorders, which we don't want _, indices = np.unique(sorted_raveled_offsets, return_index=True) sorted_raveled_offsets = sorted_raveled_offsets[np.sort(indices)] sorted_distances = sorted_distances[np.sort(indices)] # Remove "offset to center" sorted_raveled_offsets = sorted_raveled_offsets[1:] sorted_distances = sorted_distances[1:] return sorted_raveled_offsets, sorted_distances
lambda i, j: (i, -j), lambda i, j: (j, i) ] def rotate(piece_type, pattern): return np.array( [rotate_func[pattern](i, j) for (i, j) in pieces[piece_type]], dtype=np.int8) def get_blocks(piece_type, pattern, location): x, y = location return rotate(piece_type, pattern) + [x, y] struct4 = ndimage.generate_binary_structure(2, 1) struct8 = ndimage.generate_binary_structure(2, 2) class Blokus(BaseState): init_pieces_sets = [set(), set(), set(), set()] init_player = 0 num_players = 4 def __init__(self, pieces_sets=init_pieces_sets, current_player=init_player): self.pieces_sets = pieces_sets self.current_player = current_player self.legal = self.get_legal()
def _resolve_neighborhood(footprint, connectivity, ndim, enforce_adjacency=True): """Validate or create a footprint (structuring element). Depending on the values of `connectivity` and `footprint` this function either creates a new footprint (`footprint` is None) using `connectivity` or validates the given footprint (`footprint` is not None). Parameters ---------- footprint : ndarray The footprint (structuring) element used to determine the neighborhood of each evaluated pixel (``True`` denotes a connected pixel). It must be a boolean array and have the same number of dimensions as `image`. If neither `footprint` nor `connectivity` are given, all adjacent pixels are considered as part of the neighborhood. connectivity : int A number used to determine the neighborhood of each evaluated pixel. Adjacent pixels whose squared distance from the center is less than or equal to `connectivity` are considered neighbors. Ignored if `footprint` is not None. ndim : int Number of dimensions `footprint` ought to have. enforce_adjacency : bool A boolean that determines whether footprint must only specify direct neighbors. Returns ------- footprint : ndarray Validated or new footprint specifying the neighborhood. Examples -------- >>> _resolve_neighborhood(None, 1, 2) array([[False, True, False], [ True, True, True], [False, True, False]]) >>> _resolve_neighborhood(None, None, 3).shape (3, 3, 3) """ if footprint is None: if connectivity is None: connectivity = ndim footprint = ndi.generate_binary_structure(ndim, connectivity) else: # Validate custom structured element footprint = np.asarray(footprint, dtype=bool) # Must specify neighbors for all dimensions if footprint.ndim != ndim: raise ValueError( "number of dimensions in image and footprint do not" "match") # Must only specify direct neighbors if enforce_adjacency and any(s != 3 for s in footprint.shape): raise ValueError("dimension size in footprint is not 3") elif any((s % 2 != 1) for s in footprint.shape): raise ValueError("footprint size must be odd along all dimensions") return footprint
def test_morphology(): """ 测试图像的数学形态学。 :return: """ show_content = False show_images = False el = ndimage.generate_binary_structure(2, 1) el_int = el.astype(np.int) if show_content: print(el) print(el_int) # 腐蚀:最小化滤镜 a = np.zeros((7, 7), dtype=np.int) a[1:6, 2:5] = 1 a_erosion1 = ndimage.binary_erosion(a).astype(a.dtype) a_erosion2 = ndimage.binary_erosion(a, structure=np.ones( (5, 5))).astype(a.dtype) if show_content: print(a) print(a_erosion1) print(a_erosion2) if show_images: image_arrays = [a, a_erosion1, a_erosion2] image_arrays = bin2gray(image_arrays) images = ndarray_2_image(image_arrays) pil_image_demo.plt_images(images, 3) # 膨胀:最大化滤镜 a = np.zeros((5, 5)) a[2, 2] = 1 a_dilation = ndimage.binary_dilation(a).astype(a.dtype) if show_content: print(a) print(a_dilation) if show_images: image_arrays = [a, a_dilation] image_arrays = bin2gray(image_arrays) images = ndarray_2_image(image_arrays) pil_image_demo.plt_images(images, 3) # 开操作:腐蚀 + 膨胀 square = np.zeros((32, 32)) square[10:-10, 10:-10] = 1 np.random.seed(2) x, y = (32 * np.random.random((2, 20))).astype(np.int) square[x, y] = 1 # 图像开操作 open_square = ndimage.binary_opening(square) # 先腐蚀后膨胀 eroded_square = ndimage.binary_erosion(square) reconstruction = ndimage.binary_propagation(eroded_square, mask=square) if show_images: image_arrays = [square, open_square, eroded_square, reconstruction] image_arrays = bin2gray(image_arrays) images = ndarray_2_image(image_arrays) pil_image_demo.plt_images(images, 2)
def S2_PSF_optimization(self): self.h, self.v = mtile_cal(self.lat, self.lon) m = mgrs.MGRS() mg_coor = m.toMGRS(self.lat, self.lon, MGRSPrecision=4) self.place = mg_coor[:5] #self.Hfiles = glob.glob(directory +'l_data/LC8%03d%03d%d*LGN00_sr_band1.tif'%(self.path, self.row, self.year)) self.Hfile = os.getcwd() + '/s_data/%s/%s/%s/%d/%d/%d/0/' % ( mg_coor[:2], mg_coor[2], mg_coor[3:5], self.year, self.month, self.day) #Lfile = glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf'%(year,doy,h,v))[0] self.doy = datetime.datetime(self.year, self.month, self.day).timetuple().tm_yday self.Lfiles = [ glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf' % (self.year, i, self.h, self.v))[0] for i in range(self.doy - 8, self.doy + 9) ] if glob.glob(self.Hfile + 'cloud.tif') == []: cl = classification(fhead=self.Hfile, bands=(2, 3, 4, 8, 11, 12, 13), bounds=None) cl.Get_cm_p() self.cloud = cl.cm.copy() tifffile.imsave(self.Hfile + 'cloud.tif', self.cloud.astype(int)) self.H_data = np.repeat(np.repeat(cl.b12, 2, axis=1), 2, axis=0) del cl else: b12 = gdal.Open(self.Hfile + 'B12.jp2').ReadAsArray() * 0.0001 self.H_data = np.repeat(np.repeat(b12, 2, axis=1), 2, axis=0) self.cloud = tifffile.imread(self.Hfile + 'cloud.tif').astype(bool) cloud_cover = 1. * self.cloud.sum() / self.cloud.size cloud_cover = 1. * self.cloud.sum() / self.cloud.size if cloud_cover > 0.2: print 'Too much cloud, cloud proportion: %.03f !!' % cloud_cover else: mete = readxml('%smetadata.xml' % self.Hfile) self.sza = np.zeros(7) self.sza[:] = mete['mSz'] self.saa = self.sza.copy() self.saa[:] = mete['mSa'] self.vza = (mete['mVz'])[[1, 2, 3, 7, 8, 11, 12], ] self.vaa = (mete['mVa'])[[1, 2, 3, 7, 8, 11, 12], ] self.L_inds, self.H_inds = get_coords(self.lat, self.lon) self.Lx, self.Ly = self.L_inds self.Hx, self.Hy = self.H_inds angles = (self.sza[-1], self.vza[-1], (self.vaa - self.saa)[-1]) if glob.glob(self.Lfiles[8][:-3] + 'L8.16days.pkl') == []: self.BRDF_16_days = np.array([get_brdf_six(Lfile,angles,bands=(7,), \ flag=None, Linds= self.L_inds) for Lfile in self.Lfiles]).squeeze() valid_range = (self.BRDF_16_days[:, 0, :] >= 0) & (self.BRDF_16_days[:, 0, :] <= 1) magic = 0.618034 test = self.BRDF_16_days[:, 0, :].copy() test[~valid_range] = np.nan W = magic**self.BRDF_16_days[:, 1, :] W[self.BRDF_16_days[:, 1, :] > 1] = 0 #smothed = smoothn(test, axis=0, isrobust=1, W=W, s=1)[0] pkl.dump(self.BRDF_16_days, open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'w')) #pkl.dump(smothed, open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'w')) else: self.BRDF_16_days = pkl.load( open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'r')) #smothed = pkl.load(open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'r')) struct = ndimage.generate_binary_structure(2, 2) dia_cloud = ndimage.binary_dilation(self.cloud, structure=struct, iterations=60).astype( self.cloud.dtype) mask = ~(self.H_data <= 0).astype('bool') small_mask = ndimage.binary_erosion(mask, structure=struct, iterations=60).astype( mask.dtype) self.val_mask = (~dia_cloud) & small_mask self.L_data = np.zeros(self.BRDF_16_days[8, 0, :].shape[0]) self.L_data[:] = np.nan self.L_data[self.BRDF_16_days[8, 1, :] == 0] = self.BRDF_16_days[ 8, 0, :][self.BRDF_16_days[8, 1, :] == 0] #args = s, self.L_data, avker = np.ones((120, 120)) navker = avker / avker.sum() self.s = signal.fftconvolve(self.H_data, navker, mode='same') self.s[~self.val_mask] = np.nan min_val = [-100, -100] max_val = [100, 100] ps, distributions = create_training_set(['xs', 'ys'], min_val, max_val, n_train=50) solved = parmap(self.op1, ps, nprocs=10) paras, costs = np.array([i[0] for i in solved ]), np.array([i[1] for i in solved]) xs, ys = paras[costs == costs.min()][0] if costs.min() < 0.1: min_val = [5, 5, -15, xs - 5, ys - 5] max_val = [100, 100, 15, xs + 5, ys + 5] self.bounds = [5, 100], [5, 100], [-15, 15], [xs - 5, xs + 5], [ys - 5, ys + 5] ps, distributions = create_training_set( ['xstd', 'ystd', 'ang', 'xs', 'ys'], min_val, max_val, n_train=50) #ps = zip(xstd.ravel(), ystd.ravel()) print 'Start solving...' self.solved = parmap(self.op, ps, nprocs=10) #costs = np.array([i[1] for i in self.solved]) #paras = np.array([i[0] for i in self.solved]) #min_mask = costs==costs.min() #min_costs = costs.min() #para = paras[min_mask] print self.solved return self.solved, self.BRDF_16_days else: print 'Cost is too large, plese check!', xs, ys, costs.min() return [[xs, ys, costs.min()], self.BRDF_16_days] '''
# Granulometry of synthetic image # read binary simulated image, normalize it I = imageio.imread("simulation.png") / 255 I = I[:, :, 2] > .5 plt.figure() plt.imshow(I) plt.show() granulometry(I, 35) # Granulometry of real image I = imageio.imread("poudre.bmp") # segmentation BW = I > 74 BW = ndimage.morphology.binary_fill_holes(BW) # suppress small objects se = ndimage.generate_binary_structure(2, 1) m = ndimage.morphology.binary_opening(BW) # opening by reconstruction BW = ndimage.morphology.binary_propagation(m, mask=BW) imageio.imwrite("segmentation_granulo.python.png", BW.astype('int')) plt.imshow(BW) plt.show() granulometry(BW, 20, filename="poudre")
def _permuted_ols_on_chunk( scores_original_data, tested_vars, target_vars, thread_id, threshold=None, confounding_vars=None, masker=None, n_perm=10000, n_perm_chunk=10000, intercept_test=True, two_sided_test=True, random_state=None, verbose=0, ): """Perform massively univariate analysis with permuted OLS on a data chunk. To be used in a parallel computing context. Parameters ---------- scores_original_data : array-like, shape=(n_descriptors, n_regressors) t-scores obtained for the original (non-permuted) data. tested_vars : array-like, shape=(n_samples, n_regressors) Explanatory variates. target_vars : array-like, shape=(n_samples, n_targets) fMRI data. F-ordered for efficient computations. thread_id : int process id, used for display. threshold : :obj:`float` Cluster-forming threshold in t-scale. This is only used for cluster-level inference. If ``masker`` is set to None, it will be ignored. .. versionadded:: 0.9.2.dev confounding_vars : array-like, shape=(n_samples, n_covars), optional Clinical data (covariates). masker : None or :class:`~nilearn.maskers.NiftiMasker` or \ :class:`nilearn.maskers.MultiNiftiMasker`, optional A mask to be used on the data. This is only used for cluster-level inference. If None, cluster-level inference will not be performed. .. versionadded:: 0.9.2.dev n_perm : int, optional Total number of permutations to perform, only used for display in this function. Default=10000. n_perm_chunk : int, optional Number of permutations to be performed. Default=10000. intercept_test : boolean, optional Change the permutation scheme (swap signs for intercept, switch labels otherwise). See :footcite:`Fisher1935`. Default=True. two_sided_test : boolean, optional If True, performs an unsigned t-test. Both positive and negative effects are considered; the null hypothesis is that the effect is zero. If False, only positive effects are considered as relevant. The null hypothesis is that the effect is zero or negative. Default=True random_state : int or None, optional Seed for random number generator, to have the same permutations in each computing units. verbose : int, optional Defines the verbosity level. Default=0. Returns ------- scores_as_ranks_part : array-like, shape=(n_regressors, n_descriptors) The ranks of the original scores in ``h0_fmax_part``. When ``n_descriptors`` or ``n_perm`` are large, it can be quite long to find the rank of the original scores into the whole H0 distribution. Here, it is performed in parallel by the workers involved in the permutation computation. h0_fmax_part : array-like, shape=(n_perm_chunk, n_regressors) Distribution of the (max) t-statistic under the null hypothesis (limited to this permutation chunk). h0_csfwe_part, h0_cmfwe_part : array-like, \ shape=(n_perm_chunk, n_regressors) Distribution of max cluster sizes/masses under the null hypothesis. Only calculated if ``masker`` is not None. Otherwise, these will both be None. .. versionadded:: 0.9.2.dev References ---------- .. footbibliography:: """ # initialize the seed of the random generator rng = check_random_state(random_state) n_samples, n_regressors = tested_vars.shape n_descriptors = target_vars.shape[1] # run the permutations t0 = time.time() h0_vfwe_part = np.empty((n_regressors, n_perm_chunk)) vfwe_scores_as_ranks_part = np.zeros((n_regressors, n_descriptors)) if threshold is not None: h0_csfwe_part = np.empty((n_regressors, n_perm_chunk)) h0_cmfwe_part = np.empty((n_regressors, n_perm_chunk)) bin_struct = ndimage.generate_binary_structure(3, 1) else: h0_csfwe_part, h0_cmfwe_part = None, None for i_perm in range(n_perm_chunk): if intercept_test: # sign swap (random multiplication by 1 or -1) target_vars = ( target_vars * (rng.randint(2, size=(n_samples, 1)) * 2 - 1) ) else: # shuffle data # Regarding computation costs, we choose to shuffle testvars # and covars rather than fmri_signal. # Also, it is important to shuffle tested_vars and covars # jointly to simplify t-scores computation (null dot product). shuffle_idx = rng.permutation(n_samples) tested_vars = tested_vars[shuffle_idx] if confounding_vars is not None: confounding_vars = confounding_vars[shuffle_idx] # OLS regression on randomized data perm_scores = np.asfortranarray( _t_score_with_covars_and_normalized_design( tested_vars, target_vars, confounding_vars ) ) if threshold is not None: arr4d = masker.inverse_transform(perm_scores.T).get_fdata() ( h0_csfwe_part[:, i_perm], h0_cmfwe_part[:, i_perm], ) = _calculate_cluster_measures( arr4d, threshold, bin_struct, two_sided_test=two_sided_test, ) # find the rank of the original scores in h0_vfwe_part # (when n_descriptors or n_perm are large, it can be quite long to # find the rank of the original scores into the whole H0 distribution. # Here, it is performed in parallel by the workers involved in the # permutation computation) # NOTE: This is not done for the cluster-level methods. if two_sided_test: # Get maximum absolute value for voxel-level FWE h0_vfwe_part[:, i_perm] = np.nanmax(np.fabs(perm_scores), axis=0) vfwe_scores_as_ranks_part += ( h0_vfwe_part[:, i_perm].reshape((-1, 1)) < np.fabs(scores_original_data).T ) else: # Get maximum value for voxel-level FWE h0_vfwe_part[:, i_perm] = np.nanmax(perm_scores, axis=0) vfwe_scores_as_ranks_part += ( h0_vfwe_part[:, i_perm].reshape((-1, 1)) < scores_original_data.T ) if verbose > 0: step = 11 - min(verbose, 10) if i_perm % step == 0: # If there is only one job, progress information is fixed if n_perm == n_perm_chunk: crlf = '\r' else: crlf = '\n' percent = float(i_perm) / n_perm_chunk percent = round(percent * 100, 2) dt = time.time() - t0 remaining = (100. - percent) / max(0.01, percent) * dt sys.stderr.write( f'Job #{thread_id}, processed {i_perm}/{n_perm_chunk} ' f'permutations ({percent:0.2f}%, {remaining} seconds ' f'remaining){crlf}' ) return ( vfwe_scores_as_ranks_part, h0_vfwe_part, h0_csfwe_part, h0_cmfwe_part, )
def count_lesions(netseg, target, thresh): """ Comparing segmentations volumetrically Connected component analysis of between prediction `netseg` and ground truth `target` across lesion bin sizes. :param netseg: network output on range [0,1], shape=(NxMxO) :type netseg: float16, float32, float64 :param target: ground truth labels, shape=(NxMxO) :type target: int16 :param thresh: threshold to binarize prediction `h` :type thresh: float16, float32, float64 :return: dict **************** Courtesy of Tanya Nair ******************** """ netseg[netseg >= thresh] = 1 netseg[netseg < thresh] = 0 netseg = netseg[0] target = target[0] mask_target = np.zeros((target.shape[1], target.shape[2], target.shape[3])) for lesion in range(target.shape[0]): mask_target += target[lesion] # To Test netseg = gt_mask (should get ROC as tpr = 1 and fdr = 0 everywhere) target, _ = utils.remove_tiny_les(mask_target, nvox=2) netseg0 = netseg.copy() netseg = ndimage.binary_dilation(netseg, structure=ndimage.generate_binary_structure(3, 2)) labels = {} nles = {} labels['target'], nles['target'] = ndimage.label(target) labels['netseg'], nles['netseg'] = ndimage.label(netseg) found_h = np.ones(nles['netseg'], np.int16) ntp = {'all': 0, 'small': 0, 'med': 0, 'large': 0} nfp = {'all': 0, 'small': 0, 'med': 0, 'large': 0} nfn = {'all': 0, 'small': 0, 'med': 0, 'large': 0} nb_les = {'all': 0, 'small': 0, 'med': 0, 'large': 0} nles_gt = {'all': nles['target'], 'small': 0, 'med': 0, 'large': 0} # Go through ground truth segmentation masks and count true positives/false negatives for i in range(1, nles['target'] + 1): gt_lesion_size = np.sum(target[labels['target'] == i]) nles_gt[utils.get_lesion_bin(gt_lesion_size)] += 1 # List of detected lesions in this area h_lesions = np.unique(labels['netseg'][labels['target'] == i]) # All the voxels in this area contribute to detecting the lesion nb_overlap = netseg[labels['target'] == i].sum() if nb_overlap >= 3 or nb_overlap >= 0.5 * gt_lesion_size: nb_les[utils.get_lesion_bin(gt_lesion_size)] += 1 ntp[utils.get_lesion_bin(gt_lesion_size)] += 1 for h_lesion in h_lesions: if h_lesion != 0: found_h[h_lesion - 1] = 0 else: nfn[utils.get_lesion_bin(gt_lesion_size)] += 1 for i in range(1, nles['netseg'] + 1): nb_vox = np.sum(netseg0[labels['netseg'] == i]) if found_h[i - 1] == 1: nfp[utils.get_lesion_bin(nb_vox)] += 1 nb_les['all'] = nb_les['small'] + nb_les['med'] + nb_les['large'] ntp['all'] = ntp['small'] + ntp['med'] + ntp['large'] nfp['all'] = nfp['small'] + nfp['med'] + nfp['large'] nfn['all'] = nfn['small'] + nfn['med'] + nfn['large'] tpr = {} fdr = {} for s in ntp.keys(): # tpr (sensitivity) if nb_les[s] != 0: tpr[s] = ntp[s] / nb_les[s] elif nb_les[s] == 0 and ntp[s] == 0: tpr[s] = 1 else: tpr[s] = 0 # ppv (1-fdr) if ntp[s] + nfp[s] != 0: ppv = ntp[s] / (ntp[s] + nfp[s]) elif ntp[s] == 0: ppv = 1 else: ppv = 0 fdr[s] = 1 - ppv return {'ntp': ntp, 'nfp': nfp, 'nfn': nfn, 'fdr': fdr, 'tpr': tpr, 'nles': nb_les, 'nles_gt': nles_gt}
def permuted_ols( tested_vars, target_vars, confounding_vars=None, model_intercept=True, n_perm=10000, two_sided_test=True, random_state=None, n_jobs=1, verbose=0, masker=None, threshold=None, output_type='legacy', ): """Massively univariate group analysis with permuted OLS. Tested variates are independently fitted to target variates descriptors (e.g. brain imaging signal) according to a linear model solved with an Ordinary Least Squares criterion. Confounding variates may be included in the model. Permutation testing is used to assess the significance of the relationship between the tested variates and the target variates :footcite:`Anderson2001`, :footcite:`Winkler2014`. A max-type procedure is used to obtain family-wise corrected p-values based on t-statistics (voxel-level FWE), cluster sizes, and cluster masses. The specific permutation scheme implemented here is the one of :footcite:`Freedman1983`. Its has been demonstrated in :footcite:`Anderson2001` that this scheme conveys more sensitivity than alternative schemes. This holds for neuroimaging applications, as discussed in details in :footcite:`Winkler2014`. Permutations are performed on parallel computing units. Each of them performs a fraction of permutations on the whole dataset. Thus, the max t-score amongst data descriptors can be computed directly, which avoids storing all the computed t-scores. The variates should be given C-contiguous. ``target_vars`` are fortran-ordered automatically to speed-up computations. Parameters ---------- tested_vars : array-like, shape=(n_samples, n_regressors) Explanatory variates, fitted and tested independently from each others. target_vars : array-like, shape=(n_samples, n_descriptors) fMRI data to analyze according to the explanatory and confounding variates. In a group-level analysis, the samples will typically be voxels (for volumetric data) or vertices (for surface data), while the descriptors will generally be images, such as run-wise z-statistic maps. confounding_vars : array-like, shape=(n_samples, n_covars), optional Confounding variates (covariates), fitted but not tested. If None, no confounding variate is added to the model (except maybe a constant column according to the value of ``model_intercept``). model_intercept : :obj:`bool`, optional If True, a constant column is added to the confounding variates unless the tested variate is already the intercept. Default=True. n_perm : :obj:`int`, optional Number of permutations to perform. Permutations are costly but the more are performed, the more precision one gets in the p-values estimation. If ``n_perm`` is set to 0, then no p-values will be estimated. Default=10000. two_sided_test : :obj:`bool`, optional If True, performs an unsigned t-test. Both positive and negative effects are considered; the null hypothesis is that the effect is zero. If False, only positive effects are considered as relevant. The null hypothesis is that the effect is zero or negative. Default=True. random_state : :obj:`int` or None, optional Seed for random number generator, to have the same permutations in each computing units. n_jobs : :obj:`int`, optional Number of parallel workers. If -1 is provided, all CPUs are used. A negative number indicates that all the CPUs except (abs(n_jobs) - 1) ones will be used. Default=1. verbose : :obj:`int`, optional verbosity level (0 means no message). Default=0. masker : None or :class:`~nilearn.maskers.NiftiMasker` or \ :class:`~nilearn.maskers.MultiNiftiMasker`, optional A mask to be used on the data. This is required for cluster-level inference, so it must be provided if ``threshold`` is not None. .. versionadded:: 0.9.2.dev threshold : None or :obj:`float`, optional Cluster-forming threshold in p-scale. This is only used for cluster-level inference. If None, cluster-level inference will not be performed. Default=None. .. warning:: Performing cluster-level inference will increase the computation time of the permutation procedure. .. versionadded:: 0.9.2.dev output_type : {'legacy', 'dict'}, optional Determines how outputs should be returned. The two options are: - 'legacy': return a pvals, score_orig_data, and h0_fmax. This option is the default, but it is deprecated until 0.13, when the default will be changed to 'dict'. It will be removed in 0.15. - 'dict': return a dictionary containing output arrays. This option will be made the default in 0.13. .. deprecated:: 0.9.2.dev The default value for this parameter will change from 'legacy' to 'dict' in 0.13, and the parameter will be removed completely in 0.15. .. versionadded:: 0.9.2.dev Returns ------- pvals : array-like, shape=(n_regressors, n_descriptors) Negative log10 p-values associated with the significance test of the n_regressors explanatory variates against the n_descriptors target variates. Family-wise corrected p-values. .. note:: This is returned if ``output_type`` == 'legacy'. .. deprecated:: 0.9.2.dev The 'legacy' option for ``output_type`` is deprecated. The default value will change to 'dict' in 0.13, and the ``output_type`` parameter will be removed in 0.15. score_orig_data : numpy.ndarray, shape=(n_regressors, n_descriptors) t-statistic associated with the significance test of the n_regressors explanatory variates against the n_descriptors target variates. The ranks of the scores into the h0 distribution correspond to the p-values. .. note:: This is returned if ``output_type`` == 'legacy'. .. deprecated:: 0.9.2.dev The 'legacy' option for ``output_type`` is deprecated. The default value will change to 'dict' in 0.13, and the ``output_type`` parameter will be removed in 0.15. h0_fmax : array-like, shape=(n_regressors, n_perm) Distribution of the (max) t-statistic under the null hypothesis (obtained from the permutations). Array is sorted. .. note:: This is returned if ``output_type`` == 'legacy'. .. deprecated:: 0.9.2.dev The 'legacy' option for ``output_type`` is deprecated. The default value will change to 'dict' in 0.13, and the ``output_type`` parameter will be removed in 0.15. .. versionchanged:: 0.9.2.dev Return H0 for all regressors, instead of only the first one. outputs : :obj:`dict` Output arrays, organized in a dictionary. .. note:: This is returned if ``output_type`` == 'dict'. This will be the default output starting in version 0.13. .. versionadded:: 0.9.2.dev Here are the keys: ============= ============== ========================================== key shape description ============= ============== ========================================== t (n_regressors, t-statistic associated with the n_descriptors) significance test of the n_regressors explanatory variates against the n_descriptors target variates. The ranks of the scores into the h0 distribution correspond to the p-values. logp_max_t (n_regressors, Negative log10 p-values associated with n_descriptors) the significance test of the n_regressors explanatory variates against the n_descriptors target variates. Family-wise corrected p-values, based on ``h0_max_t``. h0_max_t (n_regressors, Distribution of the max t-statistic under n_perm) the null hypothesis (obtained from the permutations). Array is sorted. logp_max_size (n_regressors, Negative log10 p-values associated with n_descriptors) the cluster-level significance test of the n_regressors explanatory variates against the n_descriptors target variates. Family-wise corrected, cluster-level p-values, based on ``h0_max_size``. Returned only if ``masker`` is not None. h0_max_size (n_regressors, Distribution of the max cluster size value n_perm) under the null hypothesis (obtained from the permutations). Array is sorted. Returned only if ``masker`` is not None. logp_max_mass (n_regressors, Negative log10 p-values associated with n_descriptors) the cluster-level significance test of the n_regressors explanatory variates against the n_descriptors target variates. Family-wise corrected, cluster-level p-values, based on ``h0_max_mass``. Returned only if ``masker`` is not None. h0_max_mass (n_regressors, Distribution of the max cluster mass value n_perm) under the null hypothesis (obtained from the permutations). Array is sorted. Returned only if ``masker`` is not None. ============= ============== ========================================== References ---------- .. footbibliography:: """ # initialize the seed of the random generator rng = check_random_state(random_state) # check n_jobs (number of CPUs) if n_jobs == 0: # invalid according to joblib's conventions raise ValueError( "'n_jobs == 0' is not a valid choice. " "Please provide a positive number of CPUs, or -1 for all CPUs, " "or a negative number (-i) for 'all but (i-1)' CPUs " "(joblib conventions)." ) elif n_jobs < 0: n_jobs = max(1, joblib.cpu_count() - int(n_jobs) + 1) else: n_jobs = min(n_jobs, joblib.cpu_count()) if (threshold is not None) and (masker is None): raise ValueError( 'If "threshold" is not None, masker must be defined as well.' ) elif (threshold is None) and (masker is not None): warnings.warn( 'If "threshold" is not set, then masker will be ignored.' ) if (threshold is not None) and (output_type == 'legacy'): warnings.warn( 'If "threshold" is not None, "output_type" must be set to "dict". ' 'Overriding.' ) output_type = 'dict' elif output_type == 'legacy': warnings.warn( category=DeprecationWarning, message=( 'The "legacy" output structure for "permuted_ols" is ' 'deprecated. ' 'The default output structure will be changed to "dict" ' 'in version 0.13.' ), stacklevel=3, ) # make target_vars F-ordered to speed-up computation if target_vars.ndim != 2: raise ValueError( "'target_vars' should be a 2D array. " f"An array with {target_vars.ndim} dimension(s) was passed." ) target_vars = np.asfortranarray(target_vars) # efficient for chunking n_descriptors = target_vars.shape[1] if np.any(np.all(target_vars == 0, axis=0)): warnings.warn( "Some descriptors in 'target_vars' have zeros across all samples. " "These descriptors will be ignored during null distribution " "generation." ) # check explanatory variates' dimensions if tested_vars.ndim == 1: tested_vars = np.atleast_2d(tested_vars).T n_samples, n_regressors = tested_vars.shape # check if explanatory variates is intercept (constant) or not if (n_regressors == 1 and np.unique(tested_vars).size == 1): intercept_test = True else: intercept_test = False # optionally add intercept if model_intercept and not intercept_test: if confounding_vars is not None: confounding_vars = np.hstack( (confounding_vars, np.ones((n_samples, 1)))) else: confounding_vars = np.ones((n_samples, 1)) # OLS regression on original data if confounding_vars is not None: # step 1: extract effect of covars from target vars covars_orthonormalized = _orthonormalize_matrix(confounding_vars) if not covars_orthonormalized.flags['C_CONTIGUOUS']: # useful to developer warnings.warn('Confounding variates not C_CONTIGUOUS.') covars_orthonormalized = np.ascontiguousarray( covars_orthonormalized) targetvars_normalized = _normalize_matrix_on_axis( target_vars).T # faster with F-ordered target_vars_chunk if not targetvars_normalized.flags['C_CONTIGUOUS']: # useful to developer warnings.warn('Target variates not C_CONTIGUOUS.') targetvars_normalized = np.ascontiguousarray(targetvars_normalized) beta_targetvars_covars = np.dot( targetvars_normalized, covars_orthonormalized ) targetvars_resid_covars = targetvars_normalized - np.dot( beta_targetvars_covars, covars_orthonormalized.T) targetvars_resid_covars = _normalize_matrix_on_axis( targetvars_resid_covars, axis=1 ) # step 2: extract effect of covars from tested vars testedvars_normalized = _normalize_matrix_on_axis( tested_vars.T, axis=1 ) beta_testedvars_covars = np.dot( testedvars_normalized, covars_orthonormalized ) testedvars_resid_covars = testedvars_normalized - np.dot( beta_testedvars_covars, covars_orthonormalized.T) testedvars_resid_covars = _normalize_matrix_on_axis( testedvars_resid_covars, axis=1 ).T.copy() n_covars = confounding_vars.shape[1] else: targetvars_resid_covars = _normalize_matrix_on_axis(target_vars).T testedvars_resid_covars = _normalize_matrix_on_axis(tested_vars).copy() covars_orthonormalized = None n_covars = 0 # check arrays contiguousity (for the sake of code efficiency) if not targetvars_resid_covars.flags['C_CONTIGUOUS']: # useful to developer warnings.warn('Target variates not C_CONTIGUOUS.') targetvars_resid_covars = np.ascontiguousarray(targetvars_resid_covars) if not testedvars_resid_covars.flags['C_CONTIGUOUS']: # useful to developer warnings.warn('Tested variates not C_CONTIGUOUS.') testedvars_resid_covars = np.ascontiguousarray(testedvars_resid_covars) # step 3: original regression (= regression on residuals + adjust t-score) # compute t score map of each tested var for original data # scores_original_data is in samples-by-regressors shape scores_original_data = _t_score_with_covars_and_normalized_design( testedvars_resid_covars, targetvars_resid_covars.T, covars_orthonormalized, ) if threshold is not None: # determine t-statistic threshold dof = n_samples - (n_regressors + n_covars) if two_sided_test: threshold_t = stats.t.isf(threshold / 2, df=dof) else: threshold_t = stats.t.isf(threshold, df=dof) else: threshold_t = None # Permutations # parallel computing units perform a reduced number of permutations each if n_perm > n_jobs: n_perm_chunks = np.asarray([n_perm / n_jobs] * n_jobs, dtype=int) n_perm_chunks[-1] += n_perm % n_jobs elif n_perm > 0: warnings.warn( f'The specified number of permutations is {n_perm} and the number ' f'of jobs to be performed in parallel has set to {n_jobs}. ' f'This is incompatible so only {n_perm} jobs will be running. ' 'You may want to perform more permutations in order to take the ' 'most of the available computing resources.' ) n_perm_chunks = np.ones(n_perm, dtype=int) # 0 or negative number of permutations => original data scores only elif output_type == 'legacy': return np.asarray([]), scores_original_data.T, np.asarray([]) else: return {'t': scores_original_data.T} # actual permutations, seeded from a random integer between 0 and maximum # value represented by np.int32 (to have a large entropy). ret = joblib.Parallel(n_jobs=n_jobs, verbose=verbose)( joblib.delayed(_permuted_ols_on_chunk)( scores_original_data, testedvars_resid_covars, targetvars_resid_covars.T, thread_id=thread_id + 1, threshold=threshold_t, confounding_vars=covars_orthonormalized, masker=masker, n_perm=n_perm, n_perm_chunk=n_perm_chunk, intercept_test=intercept_test, two_sided_test=two_sided_test, random_state=rng.randint(1, np.iinfo(np.int32).max - 1), verbose=verbose, ) for thread_id, n_perm_chunk in enumerate(n_perm_chunks)) # reduce results ( vfwe_scores_as_ranks_parts, vfwe_h0_parts, # maximum t-value null distribution csfwe_h0_parts, # cluster size null distribution cmfwe_h0_parts, # cluster mass null distribution ) = zip(*ret) # Voxel-level FWE vfwe_h0 = np.hstack((vfwe_h0_parts)) vfwe_scores_as_ranks = np.zeros((n_regressors, n_descriptors)) for vfwe_scores_as_ranks_part in vfwe_scores_as_ranks_parts: vfwe_scores_as_ranks += vfwe_scores_as_ranks_part vfwe_pvals = (n_perm + 1 - vfwe_scores_as_ranks) / float(1 + n_perm) if threshold is not None: # Cluster-size and cluster-mass FWE cluster_dict = {} # a dictionary to collect mass/size measures cluster_dict['size_h0'] = np.hstack((csfwe_h0_parts)) cluster_dict['mass_h0'] = np.hstack((cmfwe_h0_parts)) cluster_dict['size'] = np.zeros_like(vfwe_pvals).astype(int) cluster_dict['mass'] = np.zeros_like(vfwe_pvals) cluster_dict['size_pvals'] = np.zeros_like(vfwe_pvals) cluster_dict['mass_pvals'] = np.zeros_like(vfwe_pvals) scores_original_data_4d = masker.inverse_transform( scores_original_data.T ).get_fdata() bin_struct = ndimage.generate_binary_structure(3, 1) for i_regressor in range(n_regressors): scores_original_data_3d = scores_original_data_4d[..., i_regressor] # Label the clusters for both cluster mass and size inference labeled_arr3d, _ = ndimage.measurements.label( scores_original_data_3d > threshold_t, bin_struct, ) if two_sided_test: # Label positive and negative clusters separately n_positive_clusters = np.max(labeled_arr3d) temp_labeled_arr3d, _ = ndimage.measurements.label( scores_original_data_3d < -threshold_t, bin_struct, ) temp_labeled_arr3d[ temp_labeled_arr3d > threshold_t ] += n_positive_clusters labeled_arr3d = labeled_arr3d + temp_labeled_arr3d del temp_labeled_arr3d cluster_labels, idx, cluster_dict['size_regressor'] = np.unique( labeled_arr3d, return_inverse=True, return_counts=True, ) assert cluster_labels[0] == 0 # the background # Replace background's "cluster size" w zeros cluster_dict['size_regressor'][0] = 0 # Calculate mass for each cluster cluster_dict['mass_regressor'] = np.zeros(cluster_labels.shape) for j_val in cluster_labels[1:]: # skip background cluster_mass = np.sum( np.fabs(scores_original_data_3d[labeled_arr3d == j_val]) - threshold_t ) cluster_dict['mass_regressor'][j_val] = cluster_mass # Calculate p-values from size/mass values and associated h0s for metric in ['mass', 'size']: p_vals = _null_to_p( cluster_dict[f'{metric}_regressor'], cluster_dict[f'{metric}_h0'][i_regressor, :], 'larger', ) p_map = p_vals[np.reshape(idx, labeled_arr3d.shape)] metric_map = cluster_dict[f'{metric}_regressor'][ np.reshape(idx, labeled_arr3d.shape) ] # Convert 3D to image, then to 1D # There is a problem if the masker performs preprocessing, # so we use apply_mask here. cluster_dict[f'{metric}_pvals'][i_regressor, :] = np.squeeze( apply_mask( nib.Nifti1Image( p_map, masker.mask_img_.affine, masker.mask_img_.header, ), masker.mask_img_, ) ) cluster_dict[metric][i_regressor, :] = np.squeeze( apply_mask( nib.Nifti1Image( metric_map, masker.mask_img_.affine, masker.mask_img_.header, ), masker.mask_img_, ) ) if output_type == 'legacy': outputs = (-np.log10(vfwe_pvals), scores_original_data.T, vfwe_h0) else: outputs = { 't': scores_original_data.T, 'logp_max_t': -np.log10(vfwe_pvals), 'h0_max_t': vfwe_h0, } if threshold is not None: outputs['size'] = cluster_dict['size'] outputs['logp_max_size'] = -np.log10(cluster_dict['size_pvals']) outputs['h0_max_size'] = cluster_dict['size_h0'] outputs['mass'] = cluster_dict['mass'] outputs['logp_max_mass'] = -np.log10(cluster_dict['mass_pvals']) outputs['h0_max_mass'] = cluster_dict['mass_h0'] return outputs
# Scipy Measurements Label with boundary correction import scipy.ndimage as sn import numpy as n default_struct = sn.generate_binary_structure(3, 3) def clean_tuples(tuples): return sorted(set([(min(pair), max(pair)) for pair in tuples])) def merge_tuples_unionfind(tuples): # use classic algorithms union find with path compression # https://en.wikipedia.org/wiki/Disjoint-set_data_structure parent_dict = {} def subfind(x): # update roots while visiting parents if parent_dict[x] != x: parent_dict[x] = subfind(parent_dict[x]) return parent_dict[x] def find(x): if x not in parent_dict: # x forms new set and becomes a root parent_dict[x] = x return x if parent_dict[x] != x: # follow chain of parents of parents to find root parent_dict[x] = subfind(parent_dict[x])
3)_struct_zeros: structure = np.zeros((3,3,3)),other default 4)_blocks_two: no_of_blocks=2,other default 5)_blocks_ten: no_of_blocks=10,other default 6)_fakeghost_one: fakeghost=1 ,other default : will turn to 2 because fakeghost need to be >=2 7)_fakeghost_four: fakeghost=4 ,other default ''' from scipy.sparse import random import time as t import numpy as np from scipy import ndimage import os, sys import unittest sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath("voxel.py")))) import voxel as vc structure = ndimage.generate_binary_structure(3, 1) #np.ones((3,3,3)) try: input_dvar = np.load("dense_array.npy", mmap_mode="r") except: print("creating dense input array will take time...") input_dvar = random(400, 160000, density=0.7, dtype="float64") input_dvar = input_dvar.todense() input_dvar = np.array(input_dvar) input_dvar = np.reshape(input_dvar, (400, 400, 400)) np.save("dense_array.npy", input_dvar) # creating sparse input array try: input_svar = np.load("sparse_array.npy", mmap_mode="r") except: print("creating sparse input array will take time...")
def _raw_to_displayed(self, raw): """Determine displayed image from a saved raw image and a saved seed. This function ensures that the 0 label gets mapped to the 0 displayed pixel. Parameters ---------- raw : array or int Raw integer input image. Returns ------- image : array Image mapped between 0 and 1 to be displayed. """ if self._color_lookup_func is None: max_val = np.max(raw) self._color_lookup_func = self._get_color_lookup_func(raw, max_val) if (not self.show_selected_label and self._color_mode == LabelColorMode.DIRECT): u, inv = np.unique(raw, return_inverse=True) image = np.array([ self._label_color_index[x] if x in self._label_color_index else self._label_color_index[None] for x in u ])[inv].reshape(raw.shape) elif (not self.show_selected_label and self._color_mode == LabelColorMode.AUTO): image = self._color_lookup_func(raw) elif (self.show_selected_label and self._color_mode == LabelColorMode.AUTO): image = self._color_lookup_func(raw, self._selected_label) elif (self.show_selected_label and self._color_mode == LabelColorMode.DIRECT): selected = self._selected_label if selected not in self._label_color_index: selected = None index = self._label_color_index image = np.where( raw == selected, index[selected], np.where( raw != self._background_label, index[None], index[self._background_label], ), ) else: raise ValueError("Unsupported Color Mode") if self.contour > 0 and raw.ndim == 2: image = np.zeros_like(raw) struct_elem = ndi.generate_binary_structure(raw.ndim, 1) thickness = self.contour thick_struct_elem = ndi.iterate_structure(struct_elem, thickness).astype(bool) boundaries = ndi.grey_dilation( raw, footprint=struct_elem) != ndi.grey_erosion( raw, footprint=thick_struct_elem) image[boundaries] = raw[boundaries] image = self._all_vals[image] elif self.contour > 0 and raw.ndim > 2: warnings.warn( trans._( "Contours are not displayed during 3D rendering", deferred=True, )) return image
def threshold_map(img, min_cluster_size, threshold=None, mask=None, binarize=True, sided='two'): """ Cluster-extent threshold and binarize image. Parameters ---------- img : img_like or array_like Image object or 3D array to be clustered min_cluster_size : int Minimum cluster size (in voxels) threshold : float or None, optional Cluster-defining threshold for img. If None (default), assume img is already thresholded. mask : (S,) array_like or None, optional Boolean array for masking resultant data array. Default is None. binarize : bool, optional Default is True. sided : {'two', 'one', 'bi'}, optional How to apply thresholding. One-sided thresholds on the positive side. Two-sided thresholds positive and negative values together. Bi-sided thresholds positive and negative values separately. Default is 'two'. """ if not isinstance(img, np.ndarray): arr = img.get_data() else: arr = img.copy() if mask is not None: mask = mask.astype(bool) arr *= mask.reshape(arr.shape) clust_thresholded = np.zeros(arr.shape, int) if sided == 'two': test_arr = np.abs(arr) else: test_arr = arr.copy() # Positive values (or absolute values) first if threshold is not None: thresh_arr = test_arr >= threshold else: thresh_arr = test_arr > 0 # 6 connectivity struc = ndimage.generate_binary_structure(3, 1) labeled, _ = ndimage.label(thresh_arr, struc) unique, counts = np.unique(labeled, return_counts=True) clust_sizes = dict(zip(unique, counts)) clust_sizes = { k: v for k, v in clust_sizes.items() if v >= min_cluster_size } for i_clust in clust_sizes.keys(): if np.all(thresh_arr[labeled == i_clust] == 1): if binarize: clust_thresholded[labeled == i_clust] = 1 else: clust_thresholded[labeled == i_clust] = arr[labeled == i_clust] # Now negative values *if bi-sided* if sided == 'bi': if threshold is not None: thresh_arr = test_arr <= (-1 * threshold) else: thresh_arr = test_arr < 0 labeled, _ = ndimage.label(thresh_arr, struc) unique, counts = np.unique(labeled, return_counts=True) clust_sizes = dict(zip(unique, counts)) clust_sizes = { k: v for k, v in clust_sizes.items() if v >= min_cluster_size } for i_clust in clust_sizes.keys(): if np.all(thresh_arr[labeled == i_clust] == 1): if binarize: clust_thresholded[labeled == i_clust] = 1 else: clust_thresholded[labeled == i_clust] = arr[labeled == i_clust] # reshape to (S,) clust_thresholded = clust_thresholded.ravel() # if mask provided, mask output if mask is not None: clust_thresholded = clust_thresholded[mask] return clust_thresholded
def run(self, ips, imgs, para=None): inten = ImageManager.get(para['inten']) if not para['slice']: imgs = [inten.img] msks = [ips.img] else: msks = ips.imgs imgs = inten.imgs if len(msks) == 1: msks *= len(imgs) buf = imgs[0].astype(np.uint16) strc = ndimage.generate_binary_structure( 2, 1 if para['con'] == '4-connect' else 2) idct = ['Max', 'Min', 'Mean', 'Variance', 'Standard', 'Sum'] key = { 'Max': 'max', 'Min': 'min', 'Mean': 'mean', 'Variance': 'var', 'Standard': 'std', 'Sum': 'sum' } idct = [i for i in idct if para[key[i]]] titles = ['Slice', 'ID'][0 if para['slice'] else 1:] if para['center']: titles.extend(['Center-X', 'Center-Y']) if para['extent']: titles.extend(['Min-Y', 'Min-X', 'Max-Y', 'Max-X']) titles.extend(idct) k = ips.unit[0] data, mark = [], {'type': 'layers', 'body': {}} # data,mark=[],[] for i in range(len(imgs)): n = ndimage.label(msks[i], strc, output=buf) index = range(1, n + 1) dt = [] if para['slice']: dt.append([i] * n) dt.append(range(n)) xy = ndimage.center_of_mass(imgs[i], buf, index) xy = np.array(xy).round(2).T if para['center']: dt.extend([xy[1] * k, xy[0] * k]) boxs = [None] * n if para['extent']: boxs = ndimage.find_objects(buf) boxs = [(i[1].start + (i[1].stop - i[1].start) / 2, i[0].start + (i[0].stop - i[0].start) / 2, i[1].stop - i[1].start, i[0].stop - i[0].start) for i in boxs] for j in (0, 1, 2, 3): dt.append([i[j] * k for i in boxs]) if para['max']: dt.append(ndimage.maximum(imgs[i], buf, index).round(2)) if para['min']: dt.append(ndimage.minimum(imgs[i], buf, index).round(2)) if para['mean']: dt.append(ndimage.mean(imgs[i], buf, index).round(2)) if para['var']: dt.append(ndimage.variance(imgs[i], buf, index).round(2)) if para['std']: dt.append( ndimage.standard_deviation(imgs[i], buf, index).round(2)) if para['sum']: dt.append(ndimage.sum(imgs[i], buf, index).round(2)) layer = {'type': 'layer', 'body': []} xy = np.int0(xy).T texts = [(i[1], i[0]) + ('id=%d' % n, ) for i, n in zip(xy, range(len(xy)))] layer['body'].append({'type': 'texts', 'body': texts}) if para['extent']: layer['body'].append({'type': 'rectangles', 'body': boxs}) mark['body'][i] = layer data.extend(list(zip(*dt))) IPy.show_table(pd.DataFrame(data, columns=titles), inten.title + '-region statistic') inten.mark = GeometryMark(mark) inten.update = True
def clusterization(images: torch.Tensor, size=256, padding=70): imgs = images.cpu().numpy().squeeze() pattern = generate_binary_structure(2, 2) coord_result, prob_result = [], [] # print("img sum:", images.sum(dim=[1,2,3]).max()) # t1 = time.time() # for sample in range(imgs.shape[0]): def compute(sample): x, y = np.where((imgs[sample] > 1e-6)) measure_mask = np.zeros((2, size, size)) measure_mask[0, x, y] = 1 measure_mask[1, x, y] = imgs[sample, x, y] labeled_array, num_features = label(measure_mask[0], structure=pattern) # if num_features > 75: # print(num_features) x_coords, y_coords, prob_value = [], [], [] sample_centroids_coords, sample_probs_value = [], [] for i in range(1, num_features + 1): x_clust, y_clust = np.where(labeled_array == i) x_coords.append(np.average(x_clust) / size) y_coords.append(np.average(y_clust) / size) prob_value.append(np.sum(measure_mask[1, x_clust, y_clust])) assert (measure_mask[1, x_clust, y_clust].all() != 0) # print("PROB_VALUE ", prob_value) [x_coords.append(0) for i in range(padding - len(x_coords))] [y_coords.append(0) for i in range(padding - len(y_coords))] [prob_value.append(0) for i in range(padding - len(prob_value))] sample_centroids_coords.append([x_coords, y_coords]) sample_probs_value.append(prob_value) sample_centroids_coords = np.transpose( np.array(sample_centroids_coords), axes=(0, 2, 1)) sample_probs_value = np.array(sample_probs_value) # coord_result.append(sample_centroids_coords) # assert(sample_probs_value.sum() != 0) # assert(sample_probs_value.all() / sample_probs_value.sum() >= 0) # prob_result.append(sample_probs_value / sample_probs_value.sum()) return x_coords, y_coords, sample_probs_value / ( sample_probs_value.sum() + 1e-8) #return sample_centroids_coords, sample_probs_value / (sample_probs_value.sum() + 1e-8) processed_list = Parallel(n_jobs=16)(delayed(compute)(i) for i in range(imgs.shape[0])) for x, y, p in processed_list: coord_result.append( torch.cat((torch.tensor(y)[:, None], torch.tensor(x)[:, None]), dim=1)[None, ...]) prob_result.append(p) # print(time.time() - t1) return ProbabilityMeasure( torch.tensor(np.concatenate(prob_result, axis=0)).type(torch.float32), torch.cat(coord_result).type(torch.float32)).cuda()
def getLength( self, segments, boundaries, contacts, ids=None, distance='b2b', structElConn=1, line='straight', position=False): """ Calculates lengts of segments specified by (args) segments and ids. The segments can contact exactly one or two boundaries. In the one boundary case, the length is calculated as the maximal distance between segment points and the contact region, that is using the 'straight' line mode. (The distance between a point and a contact region is the distance between the point and its closest contact point.) In the two boundary case, there are two possibilities. If the line mode is 'straight', the length is calculated as a smallest straight (Euclidean) distance between points on the two contact regions. Otherwise, in the 'mid' or 'mid-seg' line modes, the length is calculated as a smallest sum of distances between a 'central' and two contact points. A central point has to belong to the intersection of the segment and a central layer formed exactly in the middle between the two boundaries. In other words, the sum of distances is minimized over all contact and mid points. The difference between the 'mid' and the 'mid-seg' modes is that in the 'mid-seg' mode the layers between the boundaries are formed on the segment alone, while in the 'mid' mode they are formed on both the segment and the neighboring inter-boundary region. Consequently, the layers formed using the 'mid-seg' mode and the distance calculated, might be a bit more precise. If argument distance is 'b2b' (two boundaries) or 'b-max' (one boundary), contact points are elements of boundaries that contact a segment, so the length is calculated between the boundaries. If it is 'c2c' (two boundaries) or 'c-max' (one boundary), contact points are elements of segments that contact a boundary. Consequently, the lengths calculated in the 'b2b' or 'b-max' modes are expected to be up to two pixels longer than those calculated in the 'c2c' or 'c-max' modes. In the case of two boundaries, the length is calculated between the contact points on the boundary (first end) and the segment (second end) dor arg distance 'b2c' and the other way round for 'c2b'. Arguments line and distance are saved as attributes lengthLine and contactMode, respectivly. If arg position is True, the positions of contact points (contact and end point for one boundary) are also calculated, generally increasing the run-time. Segments and boundaries objects have to have the same positioning (attributes offset and inset). Arguments: - segments: (Segment) object containing segments whose langths are calculated - bondaries: (Segment) object defining boundaries - contacts: (Contact) object containing the info about the contacts between the segments and the boundaries - ids: segment ids - distance: for two boundaries: 'b2b' (or 'boundary') for distance between contact points on boundaries, 'c2c' (or 'contact') between contact points on segments, 'b2c' between boundary and segment contacts and 'c2b' between segment and boundary contacts. For one boundary: 'b-max' and 'c-max'. - structElConn: (square) connectivity of the structuring element used to detect contacts (can be 1 to ndim). - line: The type of the line used to calculate the distance in the 2-boundary cases: 'straight', 'mid' or 'mid-seg' - position: flag indicating if the positions of the contact points used for the length measurment are calculated and returned, used only Return: - length: if pos is False - length, position_contact, position_other: in the one boundary case - length, position_1, position_2: in the two boundary case ToDo: make something like an average of b2b and c2c """ # set segments and ids self.setSegments(segments=segments, ids=ids) ids = self.ids # set structuring element struct_el = ndimage.generate_binary_structure( rank=self.ndim, connectivity=structElConn) # figure out expected number of boundaries if (distance == 'b-max') or (distance == 'c-max'): n_bound = 1 elif ((distance == 'b2b') or (distance == 'boundary') or (distance == 'c2c') or (distance == 'contact') or (distance == 'b2c') or (distance == 'c2b')): n_bound = 2 else: n_bound = -1 raise ValueError( "Argument distance: " + str(distance) + " was not understood." + "Defined values are 'b2b', 'boundary', 'c2c', 'contact', " + "'b2c' and 'c2c', 'b-max' and 'c-max'.") # save original insets seg_data, seg_inset = segments.getDataInset() seg_inset = copy(seg_inset) bound_data, bound_inset = boundaries.getDataInset() bound_inset = copy(bound_inset) # find length for each segment length = numpy.zeros(self.maxId+1) - 1 if position: position_1 = numpy.zeros((self.maxId+1, self.ndim), dtype='int') - 1 position_2 = numpy.zeros((self.maxId+1, self.ndim), dtype='int') - 1 for seg_id in ids: # find boundaries b_ids = contacts.findBoundaries(segmentIds=seg_id, nSegment=1) # use appropriate method to get the length of this segment if len(b_ids) == n_bound: # use smaller arrays for calculations segments.makeInset(ids=[seg_id], extend=1, expand=True) boundaries.useInset( inset=segments.inset, mode='abs', expand=True) # calculate if n_bound == 1: res = self._getSingleLength1Bound( segments=segments, boundaries=boundaries, boundaryIds=b_ids, id_=seg_id, distance=distance, structEl=struct_el, line=line, position=position) elif n_bound == 2: res = self._getSingleLength2Bound( segments=segments, boundaries=boundaries, boundaryIds=b_ids, id_=seg_id, distance=distance, structEl=struct_el, line=line, position=position) # parse result if position: length[seg_id] = res[0] position_1[seg_id] = [pos + ins.start for pos, ins \ in zip(res[1], segments.inset)] position_2[seg_id] = [pos + ins.start for pos, ins \ in zip(res[2], segments.inset)] else: length[seg_id] = res # recover full data segments.setDataInset(data=seg_data, inset=seg_inset) boundaries.setDataInset(data=bound_data, inset=bound_inset) # assign attributes self.length = length self.lengthLine = line self.contactMode = distance if position: self.end1 = position_1 self.end2 = position_2 # return if position: return length, position_1, position_2 else: return length
def get_largest_component(img): s = ndimage.generate_binary_structure(3, 1) # iterate structure labeled_array, numpatches = ndimage.label(img, s) # labeling sizes = ndimage.sum(img, labeled_array, range(1, numpatches + 1)) max_label = np.where(sizes == sizes.max())[0] + 1 return labeled_array == max_label
def rag_boundary(labels, edge_map, connectivity=2): """ Comouter RAG based on region boundaries Given an image's initial segmentation and its edge map this method constructs the corresponding Region Adjacency Graph (RAG). Each node in the RAG represents a set of pixels within the image with the same label in `labels`. The weight between two adjacent regions is the average value in `edge_map` along their boundary. labels : ndarray The labelled image. edge_map : ndarray This should have the same shape as that of `labels`. For all pixels along the boundary between 2 adjacent regions, the average value of the corresponding pixels in `edge_map` is the edge weight between them. connectivity : int, optional Pixels with a squared distance less than `connectivity` from each other are considered adjacent. It can range from 1 to `labels.ndim`. Its behavior is the same as `connectivity` parameter in `scipy.ndimage.filters.generate_binary_structure`. Examples -------- >>> from skimage import data, segmentation, filters, color >>> from skimage.future import graph >>> img = data.chelsea() >>> labels = segmentation.slic(img) >>> edge_map = filters.sobel(color.rgb2gray(img)) >>> rag = graph.rag_boundary(labels, edge_map) """ conn = ndi.generate_binary_structure(labels.ndim, connectivity) eroded = ndi.grey_erosion(labels, footprint=conn) dilated = ndi.grey_dilation(labels, footprint=conn) boundaries0 = (eroded != labels) boundaries1 = (dilated != labels) labels_small = np.concatenate((eroded[boundaries0], labels[boundaries1])) labels_large = np.concatenate((labels[boundaries0], dilated[boundaries1])) n = np.max(labels_large) + 1 # use a dummy broadcast array as data for RAG ones = as_strided(np.ones((1,), dtype=np.float), shape=labels_small.shape, strides=(0,)) count_matrix = sparse.coo_matrix((ones, (labels_small, labels_large)), dtype=np.int_, shape=(n, n)).tocsr() data = np.concatenate((edge_map[boundaries0], edge_map[boundaries1])) data_coo = sparse.coo_matrix((data, (labels_small, labels_large))) graph_matrix = data_coo.tocsr() graph_matrix.data /= count_matrix.data rag = RAG() rag.add_weighted_edges_from(_edge_generator_from_csr(graph_matrix), weight='weight') rag.add_weighted_edges_from(_edge_generator_from_csr(count_matrix), weight='count') for n in rag.nodes(): rag.node[n].update({'labels': [n]}) return rag
codes = np.take(lut, neighbours) code_mask = (codes == 2) if np.any(code_mask): pixel_removed = True skeleton[code_mask] = 0 code_mask = (codes == 3) if np.any(code_mask): pixel_removed = True skeleton[code_mask] = 0 return skeleton.astype(bool) # --------- Skeletonization by medial axis transform -------- _eight_connect = ndi.generate_binary_structure(2, 2) def medial_axis(image, mask=None, return_distance=False): """ Compute the medial axis transform of a binary image Parameters ---------- image : binary ndarray, shape (M, N) The image of the shape to be skeletonized. mask : binary ndarray, shape (M, N), optional If a mask is given, only those elements in `image` with a true value in `mask` are used for computing the medial axis. return_distance : bool, optional If true, the distance transform is returned as well as the skeleton.
def run_paintGrid(pd, omeEta, seed_hkl_ids, threshold, fiber_ndiv, omeTol=None, etaTol=None, omeRange=None, etaRange=None, omePeriod=(-np.pi, np.pi), qTol=1e-7, doMultiProc=True, nCPUs=multiprocessing.cpu_count(), useGrid=None): """ wrapper for indexer.paintGrid """ del_ome = omeEta.omegas[1] - omeEta.omegas[0] del_eta = omeEta.etas[1] - omeEta.etas[0] # tolerances in degrees... I know, pathological if omeTol is None: omeTol = 360. / float(fiber_ndiv) if etaTol is None: etaTol = 360. / float(fiber_ndiv) # must be consistent pd_hkl_ids = omeEta.iHKLList[seed_hkl_ids] tTh = pd.getTTh() bMat = pd.latVecOps['B'] csym = pd.getLaueGroup() qsym = pd.getQSym() if useGrid is not None: try: print "loading quaternion grid file: %s" % (useGrid) qfib = np.loadtxt(useGrid).T except: raise RuntimeError, "unable to load quaternion grid file" else: structureNDI_label = ndimage.generate_binary_structure(2, 2) qfib = [] ii = 0 jj = fiber_ndiv print "labeling maps..." labels = [] numSpots = [] coms = [] for i in seed_hkl_ids: labels_t, numSpots_t = ndimage.label( omeEta.dataStore[i] > threshold, structureNDI_label) coms_t = np.atleast_2d( ndimage.center_of_mass(omeEta.dataStore[i], labels=labels_t, index=np.arange(1, np.amax(labels_t) + 1))) labels.append(labels_t) numSpots.append(numSpots_t) coms.append(coms_t) pass # second pass for generation print "generating quaternions..." qfib_tmp = np.empty((4, fiber_ndiv * sum(numSpots))) for i in range(len(pd_hkl_ids)): for ispot in range(numSpots[i]): if not np.isnan(coms[i][ispot][0]): ome_c = omeEta.omeEdges[0] + (0.5 + coms[i][ispot][0]) * del_ome eta_c = omeEta.etaEdges[0] + (0.5 + coms[i][ispot][1]) * del_eta gVec_s = xrdutil.makeMeasuredScatteringVectors( tTh[pd_hkl_ids[i]], eta_c, ome_c) qfib_tmp[:, ii:jj] = rot.discreteFiber( pd.hkls[:, pd_hkl_ids[i]].reshape(3, 1), gVec_s, B=bMat, ndiv=fiber_ndiv, invert=False, csym=csym)[0] ii = jj jj += fiber_ndiv pass pass qfib.append(mutil.uniqueVectors(qfib_tmp)) pass qfib = np.hstack(qfib) print "Running paintGrid on %d orientations" % (qfib.shape[1]) complPG = idx.paintGrid(qfib, omeEta, omegaRange=omeRange, etaRange=etaRange, omeTol=d2r * omeTol, etaTol=d2r * etaTol, omePeriod=omePeriod, threshold=threshold, doMultiProc=doMultiProc, nCPUs=nCPUs) return complPG, qfib
def run(config_file): # construct graph config = parse_config(config_file) config_data = config['data'] config_net1 = config.get('network1', None) config_net2 = config.get('network2', None) config_net3 = config.get('network3', None) config_test = config['testing'] batch_size = config_test.get('batch_size', 5) if(config_net1): net_type1 = config_net1['net_type'] net_name1 = config_net1['net_name'] data_shape1 = config_net1['data_shape'] label_shape1 = config_net1['label_shape'] data_channel1= config_net1['data_channel'] class_num1 = config_net1['class_num'] # construct graph for 1st network full_data_shape1 = [batch_size] + data_shape1 + [data_channel1] x1 = tf.placeholder(tf.float32, shape = full_data_shape1) net_class1 = NetFactory.create(net_type1) net1 = net_class1(num_classes = class_num1,w_regularizer = None, b_regularizer = None, name = net_name1) net1.set_params(config_net1) predicty1 = net1(x1, is_training = True) proby1 = tf.nn.softmax(predicty1) else: config_net1ax = config['network1ax'] config_net1sg = config['network1sg'] config_net1cr = config['network1cr'] # construct graph for 1st network axial net_type1ax = config_net1ax['net_type'] net_name1ax = config_net1ax['net_name'] data_shape1ax = config_net1ax['data_shape'] label_shape1ax = config_net1ax['label_shape'] data_channel1ax= config_net1ax['data_channel'] class_num1ax = config_net1ax['class_num'] full_data_shape1ax = [batch_size] + data_shape1ax + [data_channel1ax] x1ax = tf.placeholder(tf.float32, shape = full_data_shape1ax) net_class1ax = NetFactory.create(net_type1ax) net1ax = net_class1ax(num_classes = class_num1ax,w_regularizer = None, b_regularizer = None, name = net_name1ax) net1ax.set_params(config_net1ax) predicty1ax = net1ax(x1ax, is_training = True) proby1ax = tf.nn.softmax(predicty1ax) # construct graph for 1st network sagittal net_type1sg = config_net1sg['net_type'] net_name1sg = config_net1sg['net_name'] data_shape1sg = config_net1sg['data_shape'] label_shape1sg = config_net1sg['label_shape'] data_channel1sg= config_net1sg['data_channel'] class_num1sg = config_net1sg['class_num'] # construct graph for 1st network full_data_shape1sg = [batch_size] + data_shape1sg + [data_channel1sg] x1sg = tf.placeholder(tf.float32, shape = full_data_shape1sg) net_class1sg = NetFactory.create(net_type1sg) net1sg = net_class1sg(num_classes = class_num1sg,w_regularizer = None, b_regularizer = None, name = net_name1sg) net1sg.set_params(config_net1sg) predicty1sg = net1sg(x1sg, is_training = True) proby1sg = tf.nn.softmax(predicty1sg) # construct graph for 1st network corogal net_type1cr = config_net1cr['net_type'] net_name1cr = config_net1cr['net_name'] data_shape1cr = config_net1cr['data_shape'] label_shape1cr = config_net1cr['label_shape'] data_channel1cr= config_net1cr['data_channel'] class_num1cr = config_net1cr['class_num'] # construct graph for 1st network full_data_shape1cr = [batch_size] + data_shape1cr + [data_channel1cr] x1cr = tf.placeholder(tf.float32, shape = full_data_shape1cr) net_class1cr = NetFactory.create(net_type1cr) net1cr = net_class1cr(num_classes = class_num1cr,w_regularizer = None, b_regularizer = None, name = net_name1cr) net1cr.set_params(config_net1cr) predicty1cr = net1cr(x1cr, is_training = True) proby1cr = tf.nn.softmax(predicty1cr) # networks for tumor core if(config_net2): net_type2 = config_net2['net_type'] net_name2 = config_net2['net_name'] data_shape2 = config_net2['data_shape'] label_shape2 = config_net2['label_shape'] data_channel2= config_net2['data_channel'] class_num2 = config_net2['class_num'] # construct graph for 2st network full_data_shape2 = [batch_size] + data_shape2 + [data_channel2] x2 = tf.placeholder(tf.float32, shape = full_data_shape2) net_class2 = NetFactory.create(net_type2) net2 = net_class2(num_classes = class_num2,w_regularizer = None, b_regularizer = None, name = net_name2) net2.set_params(config_net2) predicty2 = net2(x2, is_training = True) proby2 = tf.nn.softmax(predicty2) else: config_net2ax = config['network2ax'] config_net2sg = config['network2sg'] config_net2cr = config['network2cr'] # construct graph for 2st network axial net_type2ax = config_net2ax['net_type'] net_name2ax = config_net2ax['net_name'] data_shape2ax = config_net2ax['data_shape'] label_shape2ax = config_net2ax['label_shape'] data_channel2ax= config_net2ax['data_channel'] class_num2ax = config_net2ax['class_num'] full_data_shape2ax = [batch_size] + data_shape2ax + [data_channel2ax] x2ax = tf.placeholder(tf.float32, shape = full_data_shape2ax) net_class2ax = NetFactory.create(net_type2ax) net2ax = net_class2ax(num_classes = class_num2ax,w_regularizer = None, b_regularizer = None, name = net_name2ax) net2ax.set_params(config_net2ax) predicty2ax = net2ax(x2ax, is_training = True) proby2ax = tf.nn.softmax(predicty2ax) # construct graph for 2st network sagittal net_type2sg = config_net2sg['net_type'] net_name2sg = config_net2sg['net_name'] data_shape2sg = config_net2sg['data_shape'] label_shape2sg = config_net2sg['label_shape'] data_channel2sg= config_net2sg['data_channel'] class_num2sg = config_net2sg['class_num'] # construct graph for 2st network full_data_shape2sg = [batch_size] + data_shape2sg + [data_channel2sg] x2sg = tf.placeholder(tf.float32, shape = full_data_shape2sg) net_class2sg = NetFactory.create(net_type2sg) net2sg = net_class2sg(num_classes = class_num2sg,w_regularizer = None, b_regularizer = None, name = net_name2sg) net2sg.set_params(config_net2sg) predicty2sg = net2sg(x2sg, is_training = True) proby2sg = tf.nn.softmax(predicty2sg) # construct graph for 2st network corogal net_type2cr = config_net2cr['net_type'] net_name2cr = config_net2cr['net_name'] data_shape2cr = config_net2cr['data_shape'] label_shape2cr = config_net2cr['label_shape'] data_channel2cr= config_net2cr['data_channel'] class_num2cr = config_net2cr['class_num'] # construct graph for 2st network full_data_shape2cr = [batch_size] + data_shape2cr + [data_channel2cr] x2cr = tf.placeholder(tf.float32, shape = full_data_shape2cr) net_class2cr = NetFactory.create(net_type2cr) net2cr = net_class2cr(num_classes = class_num2cr,w_regularizer = None, b_regularizer = None, name = net_name2cr) net2cr.set_params(config_net2cr) predicty2cr = net2cr(x2cr, is_training = True) proby2cr = tf.nn.softmax(predicty2cr) # for enhanced tumor if(config_net3): net_type3 = config_net3['net_type'] net_name3 = config_net3['net_name'] data_shape3 = config_net3['data_shape'] label_shape3 = config_net3['label_shape'] data_channel3= config_net3['data_channel'] class_num3 = config_net3['class_num'] # construct graph for 3st network full_data_shape3 = [batch_size] + data_shape3 + [data_channel3] x3 = tf.placeholder(tf.float32, shape = full_data_shape3) net_class3 = NetFactory.create(net_type3) net3 = net_class3(num_classes = class_num3,w_regularizer = None, b_regularizer = None, name = net_name3) net3.set_params(config_net3) predicty3 = net3(x3, is_training = True) proby3 = tf.nn.softmax(predicty3) else: config_net3ax = config['network3ax'] config_net3sg = config['network3sg'] config_net3cr = config['network3cr'] # construct graph for 3st network axial net_type3ax = config_net3ax['net_type'] net_name3ax = config_net3ax['net_name'] data_shape3ax = config_net3ax['data_shape'] label_shape3ax = config_net3ax['label_shape'] data_channel3ax= config_net3ax['data_channel'] class_num3ax = config_net3ax['class_num'] full_data_shape3ax = [batch_size] + data_shape3ax + [data_channel3ax] x3ax = tf.placeholder(tf.float32, shape = full_data_shape3ax) net_class3ax = NetFactory.create(net_type3ax) net3ax = net_class3ax(num_classes = class_num3ax,w_regularizer = None, b_regularizer = None, name = net_name3ax) net3ax.set_params(config_net3ax) predicty3ax = net3ax(x3ax, is_training = True) proby3ax = tf.nn.softmax(predicty3ax) # construct graph for 3st network sagittal net_type3sg = config_net3sg['net_type'] net_name3sg = config_net3sg['net_name'] data_shape3sg = config_net3sg['data_shape'] label_shape3sg = config_net3sg['label_shape'] data_channel3sg= config_net3sg['data_channel'] class_num3sg = config_net3sg['class_num'] # construct graph for 3st network full_data_shape3sg = [batch_size] + data_shape3sg + [data_channel3sg] x3sg = tf.placeholder(tf.float32, shape = full_data_shape3sg) net_class3sg = NetFactory.create(net_type3sg) net3sg = net_class3sg(num_classes = class_num3sg,w_regularizer = None, b_regularizer = None, name = net_name3sg) net3sg.set_params(config_net3sg) predicty3sg = net3sg(x3sg, is_training = True) proby3sg = tf.nn.softmax(predicty3sg) # construct graph for 3st network corogal net_type3cr = config_net3cr['net_type'] net_name3cr = config_net3cr['net_name'] data_shape3cr = config_net3cr['data_shape'] label_shape3cr = config_net3cr['label_shape'] data_channel3cr= config_net3cr['data_channel'] class_num3cr = config_net3cr['class_num'] # construct graph for 3st network full_data_shape3cr = [batch_size] + data_shape3cr + [data_channel3cr] x3cr = tf.placeholder(tf.float32, shape = full_data_shape3cr) net_class3cr = NetFactory.create(net_type3cr) net3cr = net_class3cr(num_classes = class_num3cr,w_regularizer = None, b_regularizer = None, name = net_name3cr) net3cr.set_params(config_net3cr) predicty3cr = net3cr(x3cr, is_training = True) proby3cr = tf.nn.softmax(predicty3cr) all_vars = tf.global_variables() print('all vars', len(all_vars)) sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) if(config_net1): net1_vars = [x for x in all_vars if x.name[0:len(net_name1) + 1]==net_name1 + '/'] saver1 = tf.train.Saver(net1_vars) saver1.restore(sess, config_net1['model_file']) else: # net1ax_vars = [x for x in all_vars if x.name[0:len(net_name1ax)+1]==net_name1ax + '/'] # saver1ax = tf.train.Saver(net1ax_vars) # # saver1ax.restore(sess, config_net1sg['model_file']) # net1sg_vars = [x for x in all_vars if x.name[0:len(net_name1sg)+1]==net_name1sg + '/'] # for i in range(len(net1sg_vars)): # copy_value = tf.assign(net1sg_vars[i], net1ax_vars[i]) # copy_value.eval() # print('net1sg loaded') # saver1sg = tf.train.Saver(net1sg_vars) # saver1sg.save(sess, "model/msnet_wt32sg_20000cp.ckpt") # print('netsg saved') # # saver1ax.restore(sess, config_net1cr['model_file']) # net1cr_vars = [x for x in all_vars if x.name[0:len(net_name1cr)+1]==net_name1cr + '/'] # for i in range(len(net1cr_vars)): # copy_value = tf.assign(net1cr_vars[i], net1ax_vars[i]) # copy_value.eval() # saver1cr = tf.train.Saver(net1cr_vars) # saver1cr.save(sess, "model/msnet_wt32cr_20000cp.ckpt") # print('net1cr saved') # # saver1ax.restore(sess, config_net1ax['model_file']) net1ax_vars = [x for x in all_vars if x.name[0:len(net_name1ax) + 1]==net_name1ax + '/'] saver1ax = tf.train.Saver(net1ax_vars) print('net1ax', len(net1ax_vars)) saver1ax.restore(sess, config_net1ax['model_file']) net1sg_vars = [x for x in all_vars if x.name[0:len(net_name1sg) + 1]==net_name1sg + '/'] print('net1sg', len(net1sg_vars)) saver1sg = tf.train.Saver(net1sg_vars) saver1sg.restore(sess, config_net1sg['model_file']) net1cr_vars = [x for x in all_vars if x.name[0:len(net_name1cr) + 1]==net_name1cr + '/'] saver1cr = tf.train.Saver(net1cr_vars) print('net1cr', len(net1cr_vars)) saver1cr.restore(sess, config_net1cr['model_file']) if(config_net2): net2_vars = [x for x in all_vars if x.name[0:len(net_name2) + 1]==net_name2 + '/'] saver2 = tf.train.Saver(net2_vars) saver2.restore(sess, config_net2['model_file']) else: # net2ax_vars = [x for x in all_vars if x.name[0:len(net_name2ax)+1]==net_name2ax + '/'] # saver2ax = tf.train.Saver(net2ax_vars) # saver2ax.restore(sess, config_net2sg['model_file']) # net2sg_vars = [x for x in all_vars if x.name[0:len(net_name2sg)+1]==net_name2sg + '/'] # for i in range(len(net2sg_vars)): # copy_value = tf.assign(net2sg_vars[i], net2ax_vars[i]) # copy_value.eval() # print('net2sg loaded') # saver2sg = tf.train.Saver(net2sg_vars) # saver2sg.save(sess, "model/msnet_tc32sg_15000cp.ckpt") # print('net2sg saved') # # saver2ax.restore(sess, config_net2cr['model_file']) # net2cr_vars = [x for x in all_vars if x.name[0:len(net_name2cr)+1]==net_name2cr + '/'] # for i in range(len(net2cr_vars)): # copy_value = tf.assign(net2cr_vars[i], net2ax_vars[i]) # copy_value.eval() # saver2cr = tf.train.Saver(net2cr_vars) # saver2cr.save(sess, "model/msnet_tc32cr_10000cp.ckpt") # print('net2cr saved') # return # saver2ax.restore(sess, config_net2ax['model_file']) net2ax_vars = [x for x in all_vars if x.name[0:len(net_name2ax)+1]==net_name2ax + '/'] saver2ax = tf.train.Saver(net2ax_vars) saver2ax.restore(sess, config_net2ax['model_file']) net2sg_vars = [x for x in all_vars if x.name[0:len(net_name2sg)+1]==net_name2sg + '/'] saver2sg = tf.train.Saver(net2sg_vars) saver2sg.restore(sess, config_net2sg['model_file']) net2cr_vars = [x for x in all_vars if x.name[0:len(net_name2cr)+1]==net_name2cr + '/'] saver2cr = tf.train.Saver(net2cr_vars) saver2cr.restore(sess, config_net2cr['model_file']) if(config_net3): net3_vars = [x for x in all_vars if x.name[0:len(net_name3) + 1]==net_name3 + '/'] saver3 = tf.train.Saver(net3_vars) saver3.restore(sess, config_net3['model_file']) else: # net3ax_vars = [x for x in all_vars if x.name[0:len(net_name3ax)+1]==net_name3ax + '/'] # saver3ax = tf.train.Saver(net3ax_vars) # saver3ax.restore(sess, config_net3sg['model_file']) # net3sg_vars = [x for x in all_vars if x.name[0:len(net_name3sg)+1]==net_name3sg + '/'] # for i in range(len(net3sg_vars)): # copy_value = tf.assign(net3sg_vars[i], net3ax_vars[i]) # copy_value.eval() # print('net3sg loaded') # saver3sg = tf.train.Saver(net3sg_vars) # saver3sg.save(sess, "model/msnet_en32sg_20000cp.ckpt") # print('net3sg saved') # # saver3ax.restore(sess, config_net3cr['model_file']) # net3cr_vars = [x for x in all_vars if x.name[0:len(net_name3cr)+1]==net_name3cr + '/'] # for i in range(len(net3cr_vars)): # copy_value = tf.assign(net3cr_vars[i], net3ax_vars[i]) # copy_value.eval() # saver3cr = tf.train.Saver(net3cr_vars) # saver3cr.save(sess, "model/msnet_en32cr_20000cp.ckpt") # print('net3cr saved') # # saver3ax.restore(sess, config_net3ax['model_file']) net3ax_vars = [x for x in all_vars if x.name[0:len(net_name3ax) + 1]==net_name3ax+ '/'] saver3ax = tf.train.Saver(net3ax_vars) saver3ax.restore(sess, config_net3ax['model_file']) net3sg_vars = [x for x in all_vars if x.name[0:len(net_name3sg) + 1]==net_name3sg+ '/'] saver3sg = tf.train.Saver(net3sg_vars) saver3sg.restore(sess, config_net3sg['model_file']) net3cr_vars = [x for x in all_vars if x.name[0:len(net_name3cr) + 1]==net_name3cr+ '/'] saver3cr = tf.train.Saver(net3cr_vars) saver3cr.restore(sess, config_net3cr['model_file']) loader = DataLoader() loader.set_params(config_data) loader.load_data() # start to test image_num = loader.get_total_image_number() test_slice_direction = config_test.get('test_slice_direction', 'all') save_folder = config_test['save_folder'] test_time = [] struct = ndimage.generate_binary_structure(3, 2) margin = config_test.get('roi_patch_margin', 5) for i in range(image_num): # test of 1st network t0 = time.time() [imgs, weight, temp_name] = loader.get_image_data_with_name(i) groi = get_roi(weight > 0, margin) temp_imgs = [x[np.ix_(range(groi[0], groi[1]), range(groi[2], groi[3]), range(groi[4], groi[5]))] \ for x in imgs] temp_weight = weight[np.ix_(range(groi[0], groi[1]), range(groi[2], groi[3]), range(groi[4], groi[5]))] if(config_net1): data_shapes = [data_shape1, data_shape1, data_shape1] label_shapes = [label_shape1, label_shape1, label_shape1] nets = [net1, net1, net1] outputs = [proby1, proby1, proby1] inputs = [x1, x1, x1] data_channel = data_channel1 class_num = class_num1 else: data_shapes = [data_shape1ax, data_shape1sg, data_shape1cr] label_shapes = [label_shape1ax, label_shape1sg, label_shape1cr] nets = [net1ax, net1sg, net1cr] outputs = [proby1ax, proby1sg, proby1cr] inputs = [x1ax, x1sg, x1cr] data_channel = data_channel1ax class_num = class_num1ax prob1 = test_one_image_three_nets_adaptive_shape(temp_imgs, data_shapes, label_shapes, data_channel, class_num, batch_size, sess, nets, outputs, inputs, shape_mode = 0) pred1 = np.asarray(np.argmax(prob1, axis = 3), np.uint16) pred1 = pred1 * temp_weight # out_label = pred1 * temp_weight # label_convert_source = config_test.get('label_convert_source', None) # label_convert_target = config_test.get('label_convert_target', None) # if(label_convert_source and label_convert_target): # assert(len(label_convert_source) == len(label_convert_target)) # out_label = convert_label(out_label, label_convert_source, label_convert_target) # test of 2nd network wt_threshold = 2000 if(pred1.sum() == 0): print('net1 output is null', temp_name) roi2 = get_roi(temp_imgs[0] > 0, margin) else: pred1_lc = ndimage.morphology.binary_closing(pred1, structure = struct) pred1_lc = get_largest_two_component(pred1_lc, True, wt_threshold) roi2 = get_roi(pred1_lc, margin) sub_imgs = [x[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] \ for x in temp_imgs] sub_weight = temp_weight[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] if(config_net2): data_shapes = [data_shape2, data_shape2, data_shape2] label_shapes = [label_shape2, label_shape2, label_shape2] nets = [net2, net2, net2] outputs = [proby2, proby2, proby2] inputs = [x2, x2, x2] data_channel = data_channel2 class_num = class_num2 else: data_shapes = [data_shape2ax, data_shape2sg, data_shape2cr] label_shapes = [label_shape2ax, label_shape2sg, label_shape2cr] nets = [net2ax, net2sg, net2cr] outputs = [proby2ax, proby2sg, proby2cr] inputs = [x2ax, x2sg, x2cr] data_channel = data_channel2ax class_num = class_num2ax prob2 = test_one_image_three_nets_adaptive_3dshape(sub_imgs, data_shapes, label_shapes, data_channel, class_num, sess, nets, outputs, inputs) # prob2 = test_one_image_three_nets_adaptive_shape(sub_imgs, data_shapes, label_shapes, data_channel, class_num, batch_size, sess, nets, outputs, inputs, shape_mode = 1) pred2 = np.asarray(np.argmax(prob2, axis = 3), np.uint16) pred2 = pred2 * sub_weight # test of 3rd network if(pred2.sum() == 0): [roid, roih, roiw] = sub_imgs[0].shape roi3 = [0, roid, 0, roih, 0, roiw] subsub_imgs = sub_imgs subsub_weight = sub_weight else: pred2_lc = ndimage.morphology.binary_closing(pred2, structure = struct) pred2_lc = get_largest_two_component(pred2_lc) roi3 = get_roi(pred2_lc, margin) subsub_imgs = [x[np.ix_(range(roi3[0], roi3[1]), range(roi3[2], roi3[3]), range(roi3[4], roi3[5]))] \ for x in sub_imgs] subsub_weight = sub_weight[np.ix_(range(roi3[0], roi3[1]), range(roi3[2], roi3[3]), range(roi3[4], roi3[5]))] if(config_net3): data_shapes = [data_shape3, data_shape3, data_shape3] label_shapes = [label_shape3, label_shape3, label_shape3] nets = [net3, net3, net3] outputs = [proby3, proby3, proby3] inputs = [x3, x3, x3] data_channel = data_channel3 class_num = class_num3 else: data_shapes = [data_shape3ax, data_shape3sg, data_shape3cr] label_shapes = [label_shape3ax, label_shape3sg, label_shape3cr] nets = [net3ax, net3sg, net3cr] outputs = [proby3ax, proby3sg, proby3cr] inputs = [x3ax, x3sg, x3cr] data_channel = data_channel3ax class_num = class_num3ax prob3 = test_one_image_three_nets_adaptive_shape(subsub_imgs, data_shapes, label_shapes, data_channel, class_num, batch_size, sess, nets, outputs, inputs, shape_mode = 1) pred3 = np.asarray(np.argmax(prob3, axis = 3), np.uint16) pred3 = pred3 * subsub_weight # fuse results at 3 levels # convert subsub_label to full size (non-enhanced) label3_roi = np.zeros_like(pred2) label3_roi[np.ix_(range(roi3[0], roi3[1]), range(roi3[2], roi3[3]), range(roi3[4], roi3[5]))] = pred3 label3 = np.zeros_like(pred1) label3[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] = label3_roi # convert sub_label to full size (tumor core) label2 = np.zeros_like(pred1) label2[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] = pred2 # fuse the results label1_mask = (pred1 + label2 + label3) > 0 label1_mask = ndimage.morphology.binary_closing(label1_mask, structure = struct) label1_mask = get_largest_two_component(label1_mask, False, wt_threshold) label1 = pred1 * label1_mask label2_3_mask = (label2 + label3) > 0 label2_3_mask = label2_3_mask * label1_mask label2_3_mask = ndimage.morphology.binary_closing(label2_3_mask, structure = struct) label2_3_mask = remove_external_core(label1, label2_3_mask) if(label2_3_mask.sum() > 0): label2_3_mask = get_largest_two_component(label2_3_mask) label1 = (label1 + label2_3_mask) > 0 label2 = label2_3_mask label3 = label2 * label3 vox_3 = label3.sum() if(0 < vox_3 and vox_3 < 30): print('ignored voxel number ', vox_3, flush = True) label3 = np.zeros_like(label2) out_label = label1 * 2 out_label[label2>0] = 1 out_label[label3>0] = 4 out_label = np.asarray(out_label, np.int16) test_time.append(time.time() - t0) final_label = np.zeros_like(weight, np.int16) final_label[np.ix_(range(groi[0], groi[1]), range(groi[2], groi[3]), range(groi[4], groi[5]))] = out_label save_array_as_nifty_volume(final_label, save_folder+"/{0:}.nii.gz".format(temp_name)) print(temp_name, flush = True) test_time = np.asarray(test_time) print('test time', test_time.mean(), flush= True) np.savetxt(save_folder + '/test_time.txt', test_time) sess.close()
def L8_PSF_optimization(self): self.h, self.v = mtile_cal(self.lat, self.lon) pr = get_wrs(self.lat, self.lon) self.path, self.row = pr[0]['path'], pr[0]['row'] #self.Hfiles = glob.glob(directory +'l_data/LC8%03d%03d%d*LGN00_sr_band1.tif'%(self.path, self.row, self.year)) self.Hfile = directory + 'l_data/LC8%03d%03d%d%03dLGN00_toa_' % ( self.path, self.row, self.year, self.doy) #Lfile = glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf'%(year,doy,h,v))[0] self.Lfiles = [ glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf' % (self.year, i, self.h, self.v))[0] for i in range(self.doy - 8, self.doy + 9) ] if self.read_meta(self.Hfile, self.path, self.row) == None: print 'Too much cloud!!' else: self.sza, self.saa, self.vza, self.vaa, self.dic, self.corners = self.read_meta( self.Hfile, self.path, self.row) self.L_inds, self.H_inds = ML_geo_trans(self.lat, self.lon, self.dic, self.corners) self.Lx, self.Ly = self.L_inds self.Hx, self.Hy = self.H_inds tems = np.zeros((3, 6)) tems[0, :] = self.sza tems[1, :] = self.vza tems[2, :] = self.vaa - self.saa angles = (tems[0][-1], tems[1][-1], tems[2][-1]) #self.BRDF_16_days, self.composite_brdf, self.base_mask = self.compositing(angles, thre=0.005) if glob.glob(self.Lfiles[8][:-3] + 'L8.16days.pkl') == []: self.BRDF_16_days = np.array([get_brdf_six(Lfile,angles,bands=(7,), \ flag=None, Linds= self.L_inds) for Lfile in self.Lfiles]).squeeze() valid_range = (self.BRDF_16_days[:, 0, :] >= 0) & (self.BRDF_16_days[:, 0, :] <= 1) magic = 0.618034 test = self.BRDF_16_days[:, 0, :].copy() test[~valid_range] = np.nan W = magic**self.BRDF_16_days[:, 1, :] W[self.BRDF_16_days[:, 1, :] > 1] = 0 #smothed = smoothn(test, axis=0, isrobust=1, W =W, s=1)[0] pkl.dump(self.BRDF_16_days, open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'w')) #pkl.dump(smothed, open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'w')) else: self.BRDF_16_days = pkl.load( open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'r')) #smothed = pkl.load(open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'r')) cloud = gdal.Open(self.Hfile[:-5] + '_cfmask.tif').ReadAsArray() cl_mask = cloud == 4 # cloud pixels; strictest way is to set the clear pixels with cloud==0 struct = ndimage.generate_binary_structure(2, 2) dia_cloud = ndimage.binary_dilation(cl_mask, structure=struct, iterations=20).astype( cl_mask.dtype) self.H_data = gdal.Open(self.Hfile + 'band%d.tif' % 7).ReadAsArray() * 0.0001 mask = ~(self.H_data < 0).astype('bool') small_mask = ndimage.binary_erosion(mask, structure=struct, iterations=20).astype( mask.dtype) self.val_mask = (~dia_cloud) & small_mask self.L_data = np.zeros(self.BRDF_16_days[8, 0, :].shape[0]) self.L_data[:] = np.nan self.L_data[self.BRDF_16_days[8, 1, :] == 0] = self.BRDF_16_days[ 8, 0, :][self.BRDF_16_days[8, 1, :] == 0] #args = s, self.L_data, avker = np.ones((40, 40)) navker = avker / avker.sum() self.s = signal.fftconvolve(self.H_data, navker, mode='same') self.s[~self.val_mask] = np.nan min_val = [-40, -40] max_val = [40, 40] ps, distributions = create_training_set(['xs', 'ys'], min_val, max_val, n_train=50) solved = parmap(self.op1, ps, nprocs=10) paras, costs = np.array([i[0] for i in solved ]), np.array([i[1] for i in solved]) xs, ys = paras[costs == costs.min()][0] if costs.min() < 0.1: min_val = [5, 5, -15, xs - 5, ys - 5] max_val = [100, 100, 15, xs + 5, ys + 5] self.bounds = [5, 100], [5, 100], [-15, 15], [xs - 5, xs + 5], [ys - 5, ys + 5] ps, distributions = create_training_set( ['xstd', 'ystd', 'ang', 'xs', 'ys'], min_val, max_val, n_train=50) #ps = zip(xstd.ravel(), ystd.ravel()) print 'Start solving...' self.solved = parmap(self.op, ps, nprocs=10) #costs = np.array([i[1] for i in self.solved]) #paras = np.array([i[0] for i in self.solved]) #min_mask = costs==costs.min() #min_costs = costs.min() #para = paras[min_mask] print self.solved return self.solved, self.BRDF_16_days else: print 'Cost is too large, plese check!', xs, ys, costs.min() return [[xs, ys, costs.min()], self.BRDF_16_days] '''
def diamond_se(radius, dimension): se = generate_binary_structure(dimension, 1) return iterate_structure(se, radius)