def detect_peaks(image): """ http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array Takes an image and detect the peaks using the local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ from scipy.ndimage.filters import maximum_filter from scipy.ndimage.morphology import generate_binary_structure, binary_erosion # define an 8-connected neighborhood neighborhood = generate_binary_structure(2,2) #apply the local maximum filter; all pixel of maximal value #in their neighborhood are set to 1 local_max = maximum_filter(image, footprint=neighborhood)==image background = (image==0) #a little technicality: we must erode the background in order to #successfully subtract it form local_max, otherwise a line will #appear along the background border (artifact of the local maximum filter) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) detected_peaks = local_max - eroded_background peaks = np.array(np.where(detected_peaks)).T return peaks
def __surface_distances(input1, input2, voxelspacing=None, connectivity=1): """ The distances between the surface voxel of binary objects in input1 and their nearest partner surface voxel of a binary object in input2. """ input1 = numpy.atleast_1d(input1.astype(numpy.bool)) input2 = numpy.atleast_1d(input2.astype(numpy.bool)) if voxelspacing is not None: voxelspacing = _ni_support._normalize_sequence(voxelspacing, input1.ndim) voxelspacing = numpy.asarray(voxelspacing, dtype=numpy.float64) if not voxelspacing.flags.contiguous: voxelspacing = voxelspacing.copy() # binary structure footprint = generate_binary_structure(input1.ndim, connectivity) # extract only 1-pixel border line of objects input1_border = input1 - binary_erosion(input1, structure=footprint, iterations=1) input2_border = input2 - binary_erosion(input2, structure=footprint, iterations=1) # compute average surface distance # Note: scipys distance transform is calculated only inside the borders of the # foreground objects, therefore the input has to be reversed dt = distance_transform_edt(~input2_border, sampling=voxelspacing) sds = dt[input1_border] return sds
def find_local_maxima(arr): """ Find local maxima in a multidimensional array `arr`. Parameters ---------- arr : np.ndarray The array to find maxima in Returns ------- indices : tuple of np.ndarray The indices of local maxima in `arr` """ # http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array/3689710#3689710 # neighborhood is simply a 3x3x3 array of True neighborhood = morphology.generate_binary_structure(len(arr.shape), 2) local_max = ( filters.maximum_filter(arr, footprint=neighborhood) == arr ) # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#binary_erosion background = ( arr == 0 ) eroded_background = morphology.binary_erosion(background, structure=neighborhood, border_value=1) # we obtain the final mask, containing only peaks, # by removing the background from the local_min mask detected_max = local_max ^ eroded_background # ^ = XOR return np.where(detected_max)
def mkoutersurf(image, radius, outfile): #radius information is currently ignored #it is a little tougher to deal with the morphology in python fill = nib.load( image ) filld = fill.get_data() filld[filld==1] = 255 gaussian = np.ones((2,2))*.25 image_f = np.zeros((256,256,256)) for slice in xrange(256): temp = filld[:,:,slice] image_f[:,:,slice] = convolve(temp, gaussian, 'same') image2 = np.zeros((256,256,256)) image2[np.where(image_f <= 25)] = 0 image2[np.where(image_f > 25)] = 255 strel15 = generate_binary_structure(3, 1) BW2 = grey_closing(image2, structure=strel15) thresh = np.max(BW2)/2 BW2[np.where(BW2 <= thresh)] = 0 BW2[np.where(BW2 > thresh)] = 255 v, f = marching_cubes(BW2, 100) v2 = np.transpose( np.vstack( ( 128 - v[:,0], v[:,2] - 128, 128 - v[:,1], ))) write_surface(outfile, v2, f)
def detect_local_maxima(vol): """ Takes a 3D volume and detects the peaks using the local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ # define a 26-connected neighborhood neighborhood = morphology.generate_binary_structure(3,3) # first is dimension, next is relative connectivity # apply the local maximum filter; all locations of maximum value # in their neighborhood are set to 1 local_max = (filters.maximum_filter(vol, footprint=neighborhood)==vol) # Remove background local_max[vol==0] = 0 # Find endpoint indici [xOrig,yOrig,zOrig] = np.shape(vol) x = [] y = [] z = [] for i in range(0,xOrig): for j in range(0,yOrig): for k in range(0,zOrig): if local_max[i,j,k] > 0: x.append(i) y.append(j) z.append(k) return x, y, z
def __init__(self, geometric_model='affine', tps_grid_size=3, tps_reg_factor=0, h_matches=15, w_matches=15, use_conv_filter=False, dilation_filter=None, use_cuda=True, normalize_inlier_count=False, offset_factor=227/210): super(WeakInlierCount, self).__init__() self.normalize=normalize_inlier_count self.geometric_model = geometric_model self.geometricTnf = GeometricTnf(geometric_model=geometric_model, tps_grid_size=tps_grid_size, tps_reg_factor=tps_reg_factor, out_h=h_matches, out_w=w_matches, offset_factor = offset_factor, use_cuda=use_cuda) # define dilation filter if dilation_filter is None: dilation_filter = generate_binary_structure(2, 2) # define identity mask tensor (w,h are switched and will be permuted back later) mask_id = np.zeros((w_matches,h_matches,w_matches*h_matches)) idx_list = list(range(0, mask_id.size, mask_id.shape[2]+1)) mask_id.reshape((-1))[idx_list]=1 mask_id = mask_id.swapaxes(0,1) # perform 2D dilation to each channel if not use_conv_filter: if not (isinstance(dilation_filter,int) and dilation_filter==0): for i in range(mask_id.shape[2]): mask_id[:,:,i] = binary_dilation(mask_id[:,:,i],structure=dilation_filter).astype(mask_id.dtype) else: for i in range(mask_id.shape[2]): flt=np.array([[1/16,1/8,1/16], [1/8, 1/4, 1/8], [1/16,1/8,1/16]]) mask_id[:,:,i] = scipy.signal.convolve2d(mask_id[:,:,i], flt, mode='same', boundary='fill', fillvalue=0) # convert to PyTorch variable mask_id = Variable(torch.FloatTensor(mask_id).transpose(1,2).transpose(0,1).unsqueeze(0),requires_grad=False) self.mask_id = mask_id if use_cuda: self.mask_id = self.mask_id.cuda();
def getSaddlePoints(matrix, gaussian_filter_sigma=0., low=None, high=None): if low == None: low = matrix.min() if high == None: high = matrix.max() matrix = expandMatrix(matrix) neighborhood = morphology.generate_binary_structure(len(matrix.shape),2) # apply the local minimum filter; all locations of minimum value # in their neighborhood are set to 1 # http://www.scipy.org/doc/api_docs/SciPy.ndimage.filters.html#minimum_filter matrix = filters.minimum_filter(matrix, footprint=neighborhood) matrix = condenseMatrix(matrix) outPath, clusterPathMat, grad = minPath(matrix) flood = numpy.asarray(outPath) potential = [] for e in flood: i,j = e potential.append(matrix[i,j]) potential = numpy.asarray(potential) potential = scipy.ndimage.filters.gaussian_filter(potential, gaussian_filter_sigma) derivative = lambda x: numpy.array(zip(-x,x[1:])).sum(axis=1) signproduct = lambda x: numpy.array(zip(x,x[1:])).prod(axis=1) potential_prime = derivative(potential) signproducts = numpy.sign(signproduct(potential_prime)) extrema = flood[2:][numpy.where(signproducts<0)[0],:] bassinlimits = derivative(signproducts) saddlePoints = numpy.asarray(outPath[3:])[bassinlimits==-2] saddlePointValues = numpy.asarray(map(lambda x: matrix[x[0],x[1]], saddlePoints)) saddlePoints = saddlePoints[numpy.logical_and(saddlePointValues>=low, saddlePointValues<=high),:] return saddlePoints
def plotPeaks(arr2D, amp_min=DEFAULT_AMP_MIN): # http://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.morphology.iterate_structure.html#scipy.ndimage.morphology.iterate_structure struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE) # find local maxima using our fliter shape local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D background = (arr2D == 0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # Boolean mask of arr2D with True at peaks detected_peaks = local_max - eroded_background # extract peaks amps = arr2D[detected_peaks] j, i = np.where(detected_peaks) # filter peaks amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = [x for x in peaks if x[2] > amp_min] # freq, time, amp # get indices for frequency and time frequency_idx = [x[1] for x in peaks_filtered] time_idx = [x[0] for x in peaks_filtered] # scatter of the peaks fig, ax = plt.subplots() ax.imshow(arr2D) ax.scatter(time_idx, frequency_idx) ax.set_xlabel('Time') ax.set_ylabel('Frequency') ax.set_title("Spectrogram") plt.gca().invert_yaxis() plt.show()
def get2DPeaks(arr2D): """ Generates peaks of a spectogram. Args: arr2D: spectogram. Returns: List of pairs (time, frequency) of peaks. """ struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE) # find local maxima using our fliter shape local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D background = (arr2D == 0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # Boolean mask of arr2D with True at peaks detected_peaks = local_max - eroded_background # extract peaks amps = arr2D[detected_peaks] j, i = np.where(detected_peaks) # filter peaks amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = [x for x in peaks if x[2] > AMP_MIN] # freq, time, amp # get indices for frequency and time frequency_idx = [x[1] for x in peaks_filtered] time_idx = [x[0] for x in peaks_filtered] return zip(frequency_idx, time_idx)
def __surface_distances(result, reference, voxelspacing=None, connectivity=1): """ The distances between the surface voxel of binary objects in result and their nearest partner surface voxel of a binary object in reference. """ result = numpy.atleast_1d(result.astype(numpy.bool)) reference = numpy.atleast_1d(reference.astype(numpy.bool)) if voxelspacing is not None: voxelspacing = _ni_support._normalize_sequence(voxelspacing, result.ndim) voxelspacing = numpy.asarray(voxelspacing, dtype=numpy.float64) if not voxelspacing.flags.contiguous: voxelspacing = voxelspacing.copy() # binary structure footprint = generate_binary_structure(result.ndim, connectivity) # test for emptiness if 0 == numpy.count_nonzero(result): raise RuntimeError('The first supplied array does not contain any binary object.') if 0 == numpy.count_nonzero(reference): raise RuntimeError('The second supplied array does not contain any binary object.') # extract only 1-pixel border line of objects result_border = result - binary_erosion(result, structure=footprint, iterations=1) reference_border = reference - binary_erosion(reference, structure=footprint, iterations=1) # compute average surface distance # Note: scipys distance transform is calculated only inside the borders of the # foreground objects, therefore the input has to be reversed dt = distance_transform_edt(~reference_border, sampling=voxelspacing) sds = dt[result_border] return sds
def get_2D_peaks(arr2D, plot=False, amp_min=DEFAULT_AMP_MIN): struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE) # find local maxima using our fliter shape local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D background = (arr2D == 0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # Boolean mask of arr2D with True at peaks detected_peaks = local_max - eroded_background # extract peaks amps = arr2D[detected_peaks] j, i = np.where(detected_peaks) # filter peaks amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = [x for x in peaks if x[2] > amp_min] # freq, time, amp # get indices for frequency and time frequency_idx = [x[1] for x in peaks_filtered] time_idx = [x[0] for x in peaks_filtered] return zip(frequency_idx, time_idx)
def erodeDilate(self, img, method=None): """ Use morphological operators to erode or dilate the ridge structure. Dilate uses a recursive call to first dilate then erode. Dilation alone produces ridge structures that are too thick to look authentic. Recursive call introduces random spurious minutiae when some valley structures are bridged. """ img = np.array(img) if not method: method = random.choice(('erode', 'dilate', 'none')) inkIndex = np.where(img < 250) imgBin = np.zeros(np.shape(img)) imgBin[inkIndex] = 1 strel = morphology.generate_binary_structure(2,2) if method == 'erode': imgBin = morphology.binary_erosion(imgBin, strel) elif method == 'dilate': imgBin = morphology.binary_dilation(imgBin, strel) else: return img inkIndex = np.where(imgBin == 1) returnImg = 255*np.ones(np.shape(img)) returnImg[inkIndex] = 0 # Recursive call to erode after dilation to give more authentic # appearance. Erode after dilate introduces spurious minutiae # but does not make the ridge structure too thick if method == 'dilate': self.erodeDilate(returnImg, method='erode') return returnImg
def get_2D_peaks(arr2D, plot=False, amp_min=DEFAULT_AMP_MIN): struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE) local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D background = arr2D == 0 eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) detected_peaks = local_max - eroded_background amps = arr2D[detected_peaks] j, i = np.where(detected_peaks) amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = [x for x in peaks if x[2] > amp_min] # freq, time, amp frequency_idx = [x[1] for x in peaks_filtered] time_idx = [x[0] for x in peaks_filtered] if plot: # scatter of the peaks fig, ax = plt.subplots() ax.imshow(arr2D) ax.scatter(time_idx, frequency_idx) ax.set_xlabel("Time") ax.set_ylabel("Frequency") ax.set_title("Spectrogram") plt.gca().invert_yaxis() plt.show() return zip(frequency_idx, time_idx)
def __init__(self, image, perc): threshold = np.percentile(image.ravel(), perc) a = image.copy() # Keep only tail of image values distribution with signal a[a < threshold] = 0 s = generate_binary_structure(2, 2) # Label image labeled_array, num_features = label(a, structure=s) # Find objects objects = find_objects(labeled_array) # Container of object's properties _objects = np.empty(num_features, dtype=[('label', 'int'), ('dx', '<f8'), ('dy', '<f8'), ('max_pos', 'int', (2,))]) labels = np.arange(num_features) + 1 dx = [int(obj[1].stop - obj[1].start) for obj in objects] dy = [int(obj[0].stop - obj[0].start) for obj in objects] # Filling objects structured array _objects['label'] = labels _objects['dx'] = dx _objects['dy'] = dy self.objects = _objects self._classify(image, labeled_array) # Fetch positions of only successfuly classified objects self.max_pos = self._find_positions(image, labeled_array) self._sort()
def detect_peaks(image): """ from: http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array Takes an image and detect the peaks usingthe local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ # define an 8-connected neighborhood neighborhood = generate_binary_structure(2,2) #apply the local maximum filter; all pixel of maximal value #in their neighborhood are set to 1 local_max = maximum_filter(image, footprint=neighborhood)==image #local_max is a mask that contains the peaks we are #looking for, but also the background. #In order to isolate the peaks we must remove the background from the mask. #we create the mask of the background background = (image==0) #a little technicality: we must erode the background in order to #successfully subtract it form local_max, otherwise a line will #appear along the background border (artifact of the local maximum filter) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) #we obtain the final mask, containing only peaks, #by removing the background from the local_max mask detected_peaks = local_max - eroded_background return detected_peaks
def snapshot_shaded_area(file_name): """ Return an array of shaded cloud areas for a 3D model snapshot. Parameters ---------- file_name : netCDF file name Data has dimensions float x(x), float y(y) and float z(z) and the field has been conditionally sampled for condensed area in variable int condensed(z, y, x). Return ------ numpy.array Array of shaded cloud areas in units of grid cell. """ # Read netCDF dataset data = Dataset(file_name) condensed = data.variables['condensed'][...] # Determine shaded area mask (logical or along vertical axis) shaded_mask = np.any(condensed, axis=0) # Tag clouds; consider clouds connected even if they touch diagonally s = morphology.generate_binary_structure(2, 2) clouds, n = measurements.label(shaded_mask, s) # Count number of grid cells in each cloud; remove cloud free area cloud_areas = np.bincount(clouds.flatten().astype(int))[1:] return cloud_areas
def detect_local_minima(arr): # http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array/3689710#3689710 """ Takes an array and detects the troughs using the local maximum filter. Returns a boolean mask of the troughs (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ # define an connected neighborhood # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#generate_binary_structure neighborhood = morphology.generate_binary_structure(len(arr.shape),2) # apply the local minimum filter; all locations of minimum value # in their neighborhood are set to 1 # http://www.scipy.org/doc/api_docs/SciPy.ndimage.filters.html#minimum_filter local_min = (filters.minimum_filter(arr, footprint=neighborhood)==arr) # local_min is a mask that contains the peaks we are # looking for, but also the background. # In order to isolate the peaks we must remove the background from the mask. # # we create the mask of the background background = (arr==0) # # a little technicality: we must erode the background in order to # successfully subtract it from local_min, otherwise a line will # appear along the background border (artifact of the local minimum filter) # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#binary_erosion eroded_background = morphology.binary_erosion( background, structure=neighborhood, border_value=1) # # we obtain the final mask, containing only peaks, # by removing the background from the local_min mask detected_minima = local_min - eroded_background return np.where(detected_minima)
def fit_GLC_grid( subjdata, z_limit=ZLIMIT ): #thesedata=subjdata dims = len(subjdata[0])-1 bounds = [(.00001,50)] + [(-25,25) for _ in range( dims+1 ) ] optargs = ( subjdata, z_limit, None, True) xopt, fopt, grid, Jout = optimize.brute(func=negloglike_reduced , ranges = bounds , args = optargs , Ns = 5 , full_output=True ) from scipy.ndimage.filters import minimum_filter from scipy.ndimage.morphology import generate_binary_structure neighborhood = generate_binary_structure( dims+2, dims+2 ) local_mins = minimum_filter( Jout, footprint=neighborhood ) == Jout min_coords = np.array([ g[local_mins] for g in grid ]).T xoptglobal = xopt foptglobal = fopt for coords in min_coords: xopt, fopt, iter, im, sm = optimize.fmin(func=negloglike_reduced , x0 = coords , args = optargs , full_output=True ) if fopt < foptglobal: xoptglobal = xopt foptglobal = fopt return xoptglobal, foptglobal
def get_2D_peaks(array2D): # This function is based on the function 'get_2D_peaks()' available at the URL below. # https://github.com/worldveil/dejavu/blob/master/dejavu/fingerprint.py # Copyright (c) 2013 Will Drevo, use permitted under the terms of the open-source MIT License. # Create a filter to extract peaks from the image data. struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, 25) # Find local maxima using our fliter shape. These are boolean arrays. local_maxima = maximum_filter(array2D, footprint=neighborhood) == array2D background = (array2D == 0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # Boolean mask of array2D with True at peaks. detected_peaks = local_maxima - eroded_background # Extract peak amplitudes and locations. amps = array2D[detected_peaks] j, i = numpy.where(detected_peaks) # Filter peaks for those exceeding the minimum amplitude. amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = [x for x in peaks if x[2] > AMP_MIN] # Get frequency and time at peaks. frequency_idx = [x[1] for x in peaks_filtered] time_idx = [x[0] for x in peaks_filtered] return (frequency_idx, time_idx)
def incorporate_cells(binary_image): # invert input binary image inv_image = np.invert(binary_image) # matrix for binary_dilation struct = generate_binary_structure(2, 1) # do bunary dilation until the colony number even out plate_bin_dil = binary_dilation(inv_image, structure=struct) plate_dil_labels = label(plate_bin_dil) labels_number = len(np.unique(plate_dil_labels)) # initial number of colonies new_labels_number = labels_number - 1 # starting value cycle_number = 0 # starting value for dilation cycles while True: cycle_number += 1 if cycle_number >= 30: break # defence against infinite cycling else: if new_labels_number >= labels_number: break # further dilation is useless (in theory) else: labels_number = new_labels_number plate_bin_dil = binary_dilation(plate_bin_dil, structure=struct) plate_dil_labels = label(plate_bin_dil) new_labels_number = len(np.unique(plate_dil_labels)) return plate_bin_dil
def detect_local_maxima(image): neighborhood = generate_binary_structure(2,2) local_max = maximum_filter(image, footprint=neighborhood)==image background = (image==0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) detected_peaks = local_max - eroded_background return detected_peaks*256.0
def detect_local_minima(arr): neighborhood = morphology.generate_binary_structure(len(arr.shape), 2) local_min = (filters.minimum_filter(arr, footprint=neighborhood) == arr) background = (arr == 0) eroded_background = morphology.binary_erosion( background, structure=neighborhood, border_value=1) detected_minima = local_min - eroded_background return np.where(detected_minima)
def determine_search_location_per_channel(A, d1, d2, nr, method= 'ellipse', min_size=3, max_size=8, dist=3, expandCore=iterate_structure(generate_binary_structure(2,1),2).astype(int)): IND = np.array([]).reshape((d1*d2,0)) if type(method) == str: method = [method] for ni in xrange(len(np.unique(nr))): IND_ch = determine_search_location(A[:,comp_idx(nr, ni)], d1, d2, cm[comp_idx(nr,ni),:], method = method[ni], min_size = 3, max_size = 8, dist = 3, expandCore = iterate_structure(generate_binary_structure(2,1), 2).astype(int)) IND = np.hstack([IND, IND_ch]) return IND
def spatial_filter(skeleton,image_array,loops,iterations): # image_array = np.where(image_array > threshold, 1, 0) im = np.zeros_like(image_array) s = morphology.generate_binary_structure(2,2) for i in range(loops): print i grow = morphology.binary_dilation(skeleton+im, iterations = iterations, structure = s) im = np.where(grow !=0, image_array , 0) return im
def find_peaks(file): #read image data f=pyfits.open(file) img=f[0].data #set NaN pixels (empty pixels) to zero img[img != img]=0.0 img[img<0.]=0.0 if dim==4: img=img[0,0,:,:] #gets rid of 3 and 4 dimensions, since FIRST fits files have four axis but only first two have data T=ndimage.standard_deviation(img) sourcelabels,num_sources=ndimage.label(img>T) backgroundlabels,num_background=ndimage.label(img<T) # define an 8-connected neighbourhood neighborhood = generate_binary_structure(2,2) fimg=img*sourcelabels #apply the local maximum filter; all pixel of maximal value #in their neighbourhood are set to 1 local_max=maximum_filter(fimg,footprint=neighborhood)==fimg #In order to isolate the peaks we must remove the background from the mask. #we create the mask of the background background=img*backgroundlabels #we must erode the background in order to #successfully subtract it form local_max, otherwise a line will #appear along the background border (artifact of the local maximum filter) eroded_background=binary_erosion(background,structure=neighborhood,border_value=1) #we obtain the final mask, containing only peaks, #by removing the background from the local_max mask detected_peaks=local_max-eroded_background #contains some peaks not in source (background bright features), but code can remove these #now need to find positions of these maximum #label peaks peaklabels,num_peaks=ndimage.measurements.label(detected_peaks) #get peak positions slices = ndimage.find_objects(peaklabels) x, y = [], [] for dy,dx in slices: x_center = (dx.start + dx.stop - 1)/2 x.append(x_center) y_center = (dy.start + dy.stop - 1)/2 y.append(y_center) peak_positions=zip(x,y) #get peak values, in Jy/beam peak_fluxes=[] for coord in peak_positions: peak_fluxes.append(img[coord[1],coord[0]]) peaks=zip(peak_positions,peak_fluxes) #sort by peak_fluxes. Two brightest peaks will be the first two in the list peaks=sorted(peaks,key=lambda l: l[1]) peaks.reverse() return peaks,img,f
def extracts_minima_areas(arr): neighborhood = morphology.generate_binary_structure(len(arr.shape),2) local_min = (filters.minimum_filter(arr, footprint=neighborhood)==arr) labels = measurements.label(local_min)[0] objects = measurements.find_objects(labels) areas_and_indices_and_bounding_boxes = [] for idx, sl in enumerate(objects): areas_and_indices_and_bounding_boxes.append((len(arr[sl][labels[sl] == idx + 1]), idx + 1, sl)) # first area, then index, then bounding box return sorted(areas_and_indices_and_bounding_boxes), labels
def __distinct_binary_object_correspondences(reference, result, connectivity=1): """ Determines all distinct (where connectivity is defined by the connectivity parameter passed to scipy's `generate_binary_structure`) binary objects in both of the input parameters and returns a 1to1 mapping from the labelled objects in reference to the corresponding (whereas a one-voxel overlap suffices for correspondence) objects in result. All stems from the problem, that the relationship is non-surjective many-to-many. @return (labelmap1, labelmap2, n_lables1, n_labels2, labelmapping2to1) """ result = np.atleast_1d(result.astype(np.bool)) reference = np.atleast_1d(reference.astype(np.bool)) # binary structure footprint = generate_binary_structure(result.ndim, connectivity) # label distinct binary objects labelmap1, n_obj_result = label(result, footprint) labelmap2, n_obj_reference = label(reference, footprint) # find all overlaps from labelmap2 to labelmap1; collect one-to-one relationships and store all one-two-many for later processing slicers = find_objects(labelmap2) # get windows of labelled objects mapping = dict() # mappings from labels in labelmap2 to corresponding object labels in labelmap1 used_labels = set() # set to collect all already used labels from labelmap2 one_to_many = list() # list to collect all one-to-many mappings for l1id, slicer in enumerate(slicers): # iterate over object in labelmap2 and their windows l1id += 1 # labelled objects have ids sarting from 1 bobj = (l1id) == labelmap2[slicer] # find binary object corresponding to the label1 id in the segmentation l2ids = np.unique(labelmap1[slicer][ bobj]) # extract all unique object identifiers at the corresponding positions in the reference (i.e. the mapping) l2ids = l2ids[0 != l2ids] # remove background identifiers (=0) if 1 == len( l2ids): # one-to-one mapping: if target label not already used, add to final list of object-to-object mappings and mark target label as used l2id = l2ids[0] if not l2id in used_labels: mapping[l1id] = l2id used_labels.add(l2id) elif 1 < len(l2ids): # one-to-many mapping: store relationship for later processing one_to_many.append((l1id, set(l2ids))) # process one-to-many mappings, always choosing the one with the least labelmap2 correspondences first while True: one_to_many = [(l1id, l2ids - used_labels) for l1id, l2ids in one_to_many] # remove already used ids from all sets one_to_many = [x for x in one_to_many if x[1]] # remove empty sets one_to_many = sorted(one_to_many, key=lambda x: len(x[1])) # sort by set length if 0 == len(one_to_many): break l2id = one_to_many[0][1].pop() # select an arbitrary target label id from the shortest set mapping[one_to_many[0][0]] = l2id # add to one-to-one mappings used_labels.add(l2id) # mark target label as used one_to_many = one_to_many[1:] # delete the processed set from all sets return labelmap1, labelmap2, n_obj_result, n_obj_reference, mapping
def detect_peaks( image ): """ Detect the local maxima in a 2d matrix. Returns a 2d matrix where maxima are coded as 1s and the rest of the cells are 0s. """ neighborhood = generate_binary_structure( 2, 2 ) local_max = maximum_filter( image, footprint = neighborhood ) == image background = ( image == 0 ) eroded_background = binary_erosion( background, structure = neighborhood, border_value = 1 ) detected_peaks = local_max - eroded_background return detected_peaks
def detect_peaks(self, image): """ Takes an image and detect the peaks using the local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ neighborhood = generate_binary_structure(2,2) local_max = maximum_filter(image, 5)==image background = (image==0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) detected_peaks = local_max - eroded_background return detected_peaks
def detect_local_maxima(arr): # http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array/3689710#3689710 """ Takes an array and detects the troughs using the local maximum filter. Returns a boolean mask of the troughs (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ neighborhood = morphology.generate_binary_structure(len(arr.shape), 2) local_max = (filters.maximum_filter(arr, footprint=neighborhood) == arr) background = (arr == 0) eroded_background = morphology.binary_erosion(background, structure=neighborhood, border_value=1) detected_minima = local_max - eroded_background return np.where(detected_minima)
def __surface_distances(input1, input2, voxelspacing=None, connectivity=1): """ The distances between the surface voxel of binary objects in input1 and their nearest partner surface voxel of a binary object in input2. """ input1 = np.atleast_1d(input1.astype(np.bool)) input2 = np.atleast_1d(input2.astype(np.bool)) if voxelspacing is not None: voxelspacing = _ni_support._normalize_sequence(voxelspacing, input1.ndim) voxelspacing = np.asarray(voxelspacing, dtype=np.float64) if not voxelspacing.flags.contiguous: voxelspacing = voxelspacing.copy() # binary structure footprint = generate_binary_structure(input1.ndim, connectivity) # test for emptiness # if 0 == np.count_nonzero(input1): # raise RuntimeError('The first supplied array does not contain any binary object.') # if 0 == np.count_nonzero(input2): # raise RuntimeError('The second supplied array does not contain any binary object.') if np.count_nonzero(input1) == 0 or np.count_nonzero(input2) == 0: return np.max(np.shape(input1)) # extract only 1-pixel border line of objects input1_border = input1 - binary_erosion( input1, structure=footprint, iterations=1) input2_border = input2 - binary_erosion( input2, structure=footprint, iterations=1) # compute average surface distance # Note: scipys distance transform is calculated only inside the borders of the # foreground objects, therefore the input has to be reversed dt = distance_transform_edt(~input2_border, sampling=voxelspacing) sds = dt[input1_border] return sds
def __surface_distances(result, reference, voxelspacing=None, connectivity=1): """ The distances between the surface voxel of binary objects in result and their nearest partner surface voxel of a binary object in reference. """ result = np.atleast_1d(result.astype(np.bool)) reference = np.atleast_1d(reference.astype(np.bool)) if voxelspacing is not None: voxelspacing = _ni_support._normalize_sequence(voxelspacing, result.ndim) voxelspacing = np.asarray(voxelspacing, dtype=np.float64) if not voxelspacing.flags.contiguous: voxelspacing = voxelspacing.copy() # binary structure footprint = generate_binary_structure(result.ndim, connectivity) # test for emptiness if 0 == np.count_nonzero(result): raise RuntimeError( 'The first supplied array does not contain any binary object.') if 0 == np.count_nonzero(reference): raise RuntimeError( 'The second supplied array does not contain any binary object.') # extract only 1-pixel border line of objects result_border = result ^ binary_erosion( result, structure=footprint, iterations=1) reference_border = reference ^ binary_erosion( reference, structure=footprint, iterations=1) # compute average surface distance # Note: scipys distance transform is calculated only inside the borders of the # foreground objects, therefore the input has to be reversed dt = distance_transform_edt(~reference_border, sampling=voxelspacing) sds = dt[result_border] return sds
def process_mask(mask): """ 对一片肺叶的3D binary mask 进行处理 :param mask: 其中一片肺叶的3Dmask,和CT的尺寸一致 ndarray (D,H,W) np.bool :return: dilatedMask 尺寸与输入一致 """ convex_mask = np.copy(mask) for i_layer in range(convex_mask.shape[0]): mask1 = np.ascontiguousarray(mask[i_layer]) # 将mask1中的所有元素累计求和,并非完全是背景 if np.sum(mask1) > 0: # 计算能完全包住mask1前景区域的最小多边形 mask2 = convex_hull_image(mask1) if np.sum(mask2) > 2 * np.sum( mask1): # 这里取2还是1.5 扩张得过于严重,那么就让就取消 convex_hull_image 操作 mask2 = mask1 # 完全是背景 else: mask2 = mask1 convex_mask[i_layer] = mask2 struct = generate_binary_structure(3, 1) dilatedMask = binary_dilation(convex_mask, structure=struct, iterations=10) return dilatedMask
def spec_to_peaks(data, value, fp=iterate_structure( generate_binary_structure(rank=2, connectivity=2), 10)): """Gets the data and the cutoff and return the true and false values of the max Parameters ---------- data: numpy.ndarray, shape = (M, N) A 2-D array of the spectrum data value: integer Cutoff values fp: numpy.ndarray, boolean [Optional] A 2-D array Fingerprint for the surrounding Returns ------- isPeaks: numpy.ndarray, shape = (M, N) A 2-D array of true/false values of the data where peaks are located """ max_arr = maximum_filter(data, footprint=fp) return (data == max_arr) & (data > value)
def detect_largest_umap_areas_slice(slice_u_map, structure): binary_map = np.zeros(slice_u_map.shape).astype(np.bool) mask = slice_u_map != 0 binary_map[mask] = True binary_structure = generate_binary_structure(binary_map.ndim, connectivity=2) bin_labels, num_of_objects = scipy_label(binary_map, binary_structure) blob_area_sizes = [] blob_after_erosion_sizes = [] if num_of_objects >= 1: for i in np.arange(1, num_of_objects + 1): binary_map = np.zeros(bin_labels.shape).astype(np.bool) binary_map[bin_labels == i] = 1 blob_area_sizes.append(np.count_nonzero(binary_map)) remaining_blob = binary_erosion(binary_map, structure) blob_after_erosion_sizes.append(np.count_nonzero(remaining_blob)) blob_area_sizes = np.array(blob_area_sizes) blob_area_sizes[::-1].sort() # print(blob_after_erosion_sizes) blob_after_erosion_sizes = np.array(blob_after_erosion_sizes) blob_after_erosion_sizes[::-1].sort() return blob_area_sizes, blob_after_erosion_sizes
def mkoutersurf(image, radius, outfile): #radius information is currently ignored #it is a little tougher to deal with the morphology in python fill = nib.load(image) filld = fill.get_data() filld[filld == 1] = 255 gaussian = np.ones((2, 2)) * .25 image_f = np.zeros((256, 256, 256)) for slice in range(256): temp = filld[:, :, slice] image_f[:, :, slice] = convolve(temp, gaussian, 'same') image2 = np.zeros((256, 256, 256)) image2[np.where(image_f <= 25)] = 0 image2[np.where(image_f > 25)] = 255 strel15 = generate_binary_structure(3, 1) BW2 = grey_closing(image2, structure=strel15) thresh = np.max(BW2) / 2 BW2[np.where(BW2 <= thresh)] = 0 BW2[np.where(BW2 > thresh)] = 255 v, f, _, _ = measure.marching_cubes_lewiner(BW2, 100) v2 = np.transpose( np.vstack(( 128 - v[:, 0], v[:, 2] - 128, 128 - v[:, 1], ))) write_surface(outfile, v2, f)
def remove_parked_lows(lows): """ID_parked_lows Returns the locations of low pressure systems which stay in one location over the coarse of many days. Parameters: ------------------- lows: a binary array where '1' is the location of a low lat: an array of latitudes for the lows grid lon: an array of longitudes for the lows grid """ """ for t in times: buffer around each low find buffers that overlap delete these for now; just include moving storms """ lows_copy = np.copy(lows) time = lows.shape[0] for day in range(1,time): bi = morphology.generate_binary_structure(2,2) buffered_lows_new = morphology.binary_dilation(lows[day,:,:],structure = bi,iterations = 1).astype(int) buffered_lows_old = morphology.binary_dilation(lows[day-1,:,:],structure = bi,iterations = 1).astype(int) storms, _ = ndimage.label(buffered_lows_new) # id which storms are overlapping lowsum = buffered_lows_new + buffered_lows_old k = np.where(lowsum == 2) label_list = np.unique(storms[k]) # delete those storms l = np.select([lows[day,:,:] == 1],[storms]) lnew = np.in1d(l,label_list).astype(int) lnew = np.reshape(lnew,(lows.shape[1],lows.shape[2])) l[lnew == 1] = 0 l[l > 0] = 1 lows_copy[day,:,:] = l return lows_copy
def get_2D_peaks(arr2D, plot=False, amp_min=DEFAULT_AMP_MIN): # http://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.morphology.iterate_structure.html#scipy.ndimage.morphology.iterate_structure struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE) local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D background = (arr2D == 0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) detected_peaks = local_max ^ eroded_background # extract peaks amps = arr2D[detected_peaks] j, i = np.where(detected_peaks) # filter peaks amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = [x for x in peaks if x[2] > amp_min] # freq, time, amp # get indices for frequency and time frequency_idx = [x[1] for x in peaks_filtered] time_idx = [x[0] for x in peaks_filtered] # scatter of the peaks if plot: fig, ax = plt.subplots() ax.imshow(arr2D) ax.scatter(time_idx, frequency_idx) ax.set_xlabel('Time') ax.set_ylabel('Frequency') ax.set_title("Spectrogram") plt.gca().invert_yaxis() plt.show() return zip(frequency_idx, time_idx)
def detect_local_maxima(arr, toricMap=False): # http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array/3689710#3689710 """ Takes an array and detects the troughs using the local maximum filter. Returns a boolean mask of the troughs (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ # define an connected neighborhood # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#generate_binary_structure if toricMap: X,Y = arr.shape arr = expandMatrix(arr) neighborhood = morphology.generate_binary_structure(len(arr.shape),2) # apply the local minimum filter; all locations of minimum value # in their neighborhood are set to 1 # http://www.scipy.org/doc/api_docs/SciPy.ndimage.filters.html#minimum_filter local_max = (filters.maximum_filter(arr, footprint=neighborhood)==arr) # local_min is a mask that contains the peaks we are # looking for, but also the background. # In order to isolate the peaks we must remove the background from the mask. # # we create the mask of the background background = (arr==0) # # a little technicality: we must erode the background in order to # successfully subtract it from local_min, otherwise a line will # appear along the background border (artifact of the local minimum filter) # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#binary_erosion eroded_background = morphology.binary_erosion( background, structure=neighborhood, border_value=1) # # we obtain the final mask, containing only peaks, # by removing the background from the local_min mask detected_maxima = local_max - eroded_background if toricMap: detected_maxima = detected_maxima[X:2*X,Y:2*Y] return numpy.where(detected_maxima)
def detect_peaks(image): """ Takes an image and detect the peaks usingthe local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ # define an 8-connected neighborhood neighborhood = generate_binary_structure(2, 2) # apply the local maximum filter; all pixel of maximal value # in their neighborhood are set to 1 local_max = maximum_filter(image, footprint=neighborhood) == image # local_max is a mask that contains the peaks we are # looking for, but also the background. # In order to isolate the peaks we must remove the background from the mask. # we create the mask of the background background = (image == 0) # a little technicality: we must erode the background in order to # successfully subtract it form local_max, otherwise a line will # appear along the background border (artifact of the local maximum filter) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # we obtain the final mask, containing only peaks, # by removing the background from the local_max mask (xor operation) det_peaks = local_max ^ eroded_background dp = det_peaks.flatten() p = 0 for i in range(len(dp)): if dp[i] == 1: p += 1 return p, det_peaks
def surface_distance(result, reference, connectivity=1): result = result.astype(np.bool) reference = reference.astype(np.bool) # Create binary structure struct = generate_binary_structure(result.ndim, connectivity) # Test for empty images if np.count_nonzero(result) == 0: raise RuntimeError( 'The result image does not contain any binary object.') if np.count_nonzero(reference) == 0: raise RuntimeError( 'The reference image does not contain any binary object.') # Extract border images result_border = result ^ binary_erosion(result, structure=struct) reference_border = reference ^ binary_erosion(reference, structure=struct) # Compute average surface distance dt = distance_transform_edt(~reference_border) sds = dt[result_border] return sds
def spectrogram_to_peaks(spectrogram, cutoff, iterations=3): """ Given a spectrogram, return its peaks, which are all local peaks with values greater than the cutoff. Parameters ---------- spectrogram : numpy.ndarray, shape = (N, M) 2D array of shape (N, M) cutoff : float elements less than or equal to the cutoff are considered background elements iterations : int the number of iterations for the iterated footprint Returns ------- A bool array of the peaks of a spectrogram. """ fp = generate_binary_structure(rank=2, connectivity=1) spec_max = maximum_filter(spectrogram, footprint=iterate_structure(fp, iterations)) return (spectrogram == spec_max) & (spectrogram > cutoff)
def get_2D_peaks(self, plot=False): struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, 20) local_max = maximum_filter(self.spectrogram, footprint=neighborhood) == self.spectrogram background = (self.spectrogram == 0) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) detected_peaks = local_max ^ eroded_background # extract peaks amps = self.spectrogram[detected_peaks] j, i = np.where(detected_peaks) # filter peaks amps = amps.flatten() peaks = zip(i, j, amps) peaks_filtered = filter(lambda x: x[2] > 10, peaks) # freq, time, amp # get indices for frequency and time frequency_idx = [] time_idx = [] for x in peaks_filtered: frequency_idx.append(x[1]) time_idx.append(x[0]) return zip(frequency_idx, time_idx)
def detect_peaks(image): """ Takes an image and detect the peaks usingthe local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) Obtained from http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array """ from scipy.ndimage.filters import maximum_filter from scipy.ndimage.morphology import (generate_binary_structure, binary_erosion) # define an 8-connected neighborhood neighborhood = generate_binary_structure(2, 2) # apply the local maximum filter; all pixel of maximal value # in their neighborhood are set to 1 local_max = maximum_filter(image, footprint=neighborhood) == image # local_max is a mask that contains the peaks we are # looking for, but also the background. # In order to isolate the peaks we must remove the background from the mask # we create the mask of the background background = (image == 0) # a little technicality: we must erode the background in order to # successfully subtract it form local_max, otherwise a line will # appear along the background border (artifact of the local maximum filter) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # we obtain the final mask, containing only peaks, # by removing the background from the local_max mask detected_peaks = local_max - eroded_background return detected_peaks
def find_peaks(heatmap): """ We apply the "mask" method similar to github/erdogant/findpeaks directly""" # set threshold threshold = 0 # create the footprint # a neighborhood of a pixel connected to its 8 neighbors neighborhood = generate_binary_structure(2, 2) # apply local maximum filter mask, max pixels in neighborhood set to 1 local_max = maximum_filter(heatmap, footprint=neighborhood) == heatmap # Create mask of the background background = (heatmap <= threshold) # Erode background to prevent artifact when subtracting it from local max eroded = binary_erosion(background, structure=neighborhood, border_value=1) # XOR to remove background from the local max mask detected_peaks = local_max ^ eroded # Return obtain final mask of only peaks # Shape of heatmap but peaks are 1s & else is 0s return detected_peaks
def surfd(self,input1, input2, sampling=1, connectivity=1): ''' function to compute the surface distance input params: input1: predicted segmentation mask input2: ground truth mask sampling: default value connectivity: default value returns: sds : surface distance ''' input_1 = np.atleast_1d(input1.astype(np.bool)) input_2 = np.atleast_1d(input2.astype(np.bool)) conn = morphology.generate_binary_structure(input_1.ndim, connectivity) #binary erosion on input1 y=morphology.binary_erosion(input_1, conn) y=y.astype(np.float32) x=input_1.astype(np.float32) S=x-y #binary erosion on input2 y=morphology.binary_erosion(input_2, conn) y=y.astype(np.float32) x=input_2.astype(np.float32) Sprime=x-y S=S.astype(np.bool) Sprime=Sprime.astype(np.bool) dta = morphology.distance_transform_edt(~S,sampling) dtb = morphology.distance_transform_edt(~Sprime,sampling) sds = np.concatenate([np.ravel(dta[Sprime!=0]), np.ravel(dtb[S!=0])]) return sds
def local_maxima(arr): # http://stackoverflow.com/questions/3684484/peak-detection-in-a-2d-array/3689710#3689710 """ Takes an array and detects the troughs using the local maximum filter. Returns a boolean mask of the troughs (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) """ # define an connected neighborhood # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#generate_binary_structure neighborhood = morphology.generate_binary_structure(len(arr.shape), 2) # apply the local maximum filter; all locations of maximal value in their neighborhood are set to 1 # local_max is a mask that contains the peaks we are looking for, but also the background. In order to isolate # the peaks we must remove the background from the mask. # http://www.scipy.org/doc/api_docs/SciPy.ndimage.filters.html#maximum_filter local_max = (filters.maximum_filter(arr, footprint=neighborhood) == arr) # we create the mask of the background # mxu: in the original version, was background = (arr==0) background = (arr == arr.min()) # a little technicality: we must erode the background in order to # successfully subtract it from local_max, otherwise a line will # appear along the background border (artifact of the local maximum filter) # http://www.scipy.org/doc/api_docs/SciPy.ndimage.morphology.html#binary_erosion eroded_background = morphology.binary_erosion(background, structure=neighborhood, border_value=1) # # we obtain the final mask, containing only peaks, by removing the background from the local_max mask # # mxu: this is the old version, but the boolean minus operator is deprecated # detected_maxima = local_max - eroded_backround # Material nonimplication, see http://en.wikipedia.org/wiki/Material_nonimplication detected_maxima = np.bitwise_and(local_max, np.bitwise_not(eroded_background)) return np.where(detected_maxima)
def bwperim(BW, dim=3, conn=6): # generate_binary_structure and matlab bwperim interpret conn differently # Here we keep consist with Matlab function # vol = np.bool(vol) BW = BW > 0 if dim == 2: if conn in [4, 8]: if conn == 4: conn = 1 else: conn = 2 elif dim == 3: if conn in [6, 18, 26]: if conn == 6: conn = 1 elif conn == 18: conn = 2 else: conn = 3 # print(dim) conn_structure = generate_binary_structure(dim, conn) BWErode = binary_erosion(BW, structure=conn_structure) BWPeirm = BW ^ BWErode return BWPeirm
def detect_peaks(self, image): """ Takes an image and detect the peaks usingthe local maximum filter. Returns a boolean mask of the peaks (i.e. 1 when the pixel's value is the neighborhood maximum, 0 otherwise) :param image: An 2d input images :return: Binary output images of the same size as input with pixel value equal to 1 indicating that there is peak at that point """ # define an 8-connected neighborhood neighborhood = generate_binary_structure(2, 2) # apply the local maximum filter; all pixel of maximal value # in their neighborhood are set to 1 local_max = maximum_filter(image, footprint=neighborhood) == image # local_max is a mask that contains the peaks we are # looking for, but also the background. # In order to isolate the peaks we must remove the background from the mask. # we create the mask of the background background = (image == 0) # a little technicality: we must erode the background in order to # successfully subtract it form local_max, otherwise a line will # appear along the background border (artifact of the local maximum filter) eroded_background = binary_erosion(background, structure=neighborhood, border_value=1) # we obtain the final mask, containing only peaks, # by removing the background from the local_max mask (xor operation) detected_peaks = local_max ^ eroded_background return detected_peaks
def process_mask(mask): """ mask标注预处理相关 :param mask: :return: """ convex_mask = np.copy(mask) for i_layer in range(convex_mask.shape[0]): # np.ascontiguousarray方法copy返回一个跟参数array有一样shape的连续数组 mask1 = np.ascontiguousarray(mask[i_layer]) if np.sum(mask1) > 0: # 有mask目标 # convex_hull_image是CV高级形态学处理函数,输入为二值图像,输出一个逻辑二值图像,在凸包内的点为True, 否则为False mask2 = convex_hull_image(mask1) if np.sum(mask2) > 1.5 * np.sum( mask1): # 凸包生成的凸多边形太过分,掩盖了原始mask1的大致形状信息,则放弃凸包处理 mask2 = mask1 else: # 没有mask目标 mask2 = mask1 convex_mask[i_layer] = mask2 # 二值膨胀 struct = generate_binary_structure(3, 1) dilatedMask = binary_dilation(convex_mask, structure=struct, iterations=10) # 返回膨胀后的mask return dilatedMask
def _remove_outside_body(self, img_patch): h, w = img_patch.shape se = generate_binary_structure(2, 2) img_patch = binary_dilation(img_patch, structure=se).astype('uint8') for c in [0, w - 1]: x = np.where(img_patch[:, c] == 0)[0] while(len(x)): mask = region_growing([[x[0], c]], img_patch == 0) img_patch += mask x = np.where(img_patch[:, c] == 0)[0] for r in [0, h - 1]: x = np.where(img_patch[r, :] == 0)[0] while(len(x)): mask = region_growing([[r, x[0]]], img_patch == 0) img_patch += mask x = np.where(img_patch[r, :] == 0)[0] img_patch = binary_erosion(img_patch, structure=se).astype('uint8') return img_patch
print(channel1) print(type(channel1)) print(np.shape(channel1)) print(cqt) print(type(cqt)) print(np.shape(cqt)) print(cqt.ndim) sss # 2D Peaks (peaks in spectrogram) # Import from scipy.ndimage.filters import maximum_filter from scipy.ndimage.morphology import (generate_binary_structure, iterate_structure, binary_erosion) # variables: PEAK_NEIGHBORHOOD_SIZE = 20 axes = librosa_display.specshow(librosa.amplitude_to_db(cqt), sr=sr, x_axis='time', y_axis='hz', cmap='plasma') struct = generate_binary_structure(2, 1) neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE) # find local maxima using our filter shape local_max = maximum_filter(axes, footprint=neighborhood) == axes
def update_spatial_components(Y, C=None, f=None, A_in=None, sn=None, dims=None, min_size=3, max_size=8, dist=3, normalize_yyt_one=True, method='ellipse', expandCore=None, dview=None, n_pixels_per_process=128, medw=(3, 3), thr_method='nrg', maxthr=0.1, nrgthr=0.9999, extract_cc=True, b_in=None, se=np.ones((3, 3), dtype=np.int), ss=np.ones((3, 3), dtype=np.int), nb=1, method_ls='lasso_lars', update_background_components=True, low_rank_background=True, block_size=1000, num_blocks_per_run=20): """update spatial footprints and background through Basis Pursuit Denoising for each pixel i solve the problem [A(i,:),b(i)] = argmin sum(A(i,:)) subject to || Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T); for each pixel the search is limited to a few spatial components Parameters: ---------- Y: np.ndarray (2D or 3D) movie, raw data in 2D or 3D (pixels x time). C: np.ndarray calcium activity of each neuron. f: np.ndarray temporal profile of background activity. A_in: np.ndarray spatial profile of background activity. If A_in is boolean then it defines the spatial support of A. Otherwise it is used to determine it through determine_search_location b_in: np.ndarray you can pass background as input, especially in the case of one background per patch, since it will update using hals dims: [optional] tuple x, y[, z] movie dimensions min_size: [optional] int max_size: [optional] int dist: [optional] int sn: [optional] float noise associated with each pixel if known backend [optional] str 'ipyparallel', 'single_thread' single_thread:no parallelization. It can be used with small datasets. ipyparallel: uses ipython clusters and then send jobs to each of them SLURM: use the slurm scheduler n_pixels_per_process: [optional] int number of pixels to be processed by each thread method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion dview: view on ipyparallel client you need to create an ipyparallel client and pass a view on the processors (client = Client(), dview=client[:]) medw, thr_method, maxthr, nrgthr, extract_cc, se, ss: [optional] Parameters for components post-processing. Refer to spatial.threshold_components for more details nb: [optional] int Number of background components method_ls: method to perform the regression for the basis pursuit denoising. 'nnls_L0'. Nonnegative least square with L0 penalty 'lasso_lars' lasso lars function from scikit learn 'lasso_lars_old' lasso lars from old implementation, will be deprecated normalize_yyt_one: bool wheter to norrmalize the C and A matrices so that diag(C*C.T) are ones update_background_components:bool whether to update the background components in the spatial phase low_rank_background:bool whether to update the using a low rank approximation. In the False case all the nonzero elements of the background components are updated using hals (to be used with one background per patch) Returns: -------- A: np.ndarray new estimate of spatial footprints b: np.ndarray new estimate of spatial background C: np.ndarray temporal components (updated only when spatial components are completely removed) f: np.ndarray same as f_in except if empty component deleted. Raise: ------- Exception('You need to define the input dimensions') Exception('Dimension of Matrix Y must be pixels x time') Exception('Dimension of Matrix C must be neurons x time') Exception('Dimension of Matrix f must be background comps x time ') Exception('Either A or C need to be determined') Exception('Dimension of Matrix A must be pixels x neurons ') Exception('You need to provide estimate of C and f') Exception('Not implemented consistently') Exception("Failed to delete: " + folder) """ print('Initializing update of Spatial Components') if expandCore is None: expandCore = iterate_structure( generate_binary_structure(2, 1), 2).astype(int) if dims is None: raise Exception('You need to define the input dimensions') # shape transformation and tests Y, A_in, C, f, n_pixels_per_process, rank_f, d, T = test( Y, A_in, C, f, n_pixels_per_process, nb) start_time = time.time() print('computing the distance indicators') # we compute the indicator from distance indicator ind2_, nr, C, f, b_, A_in = computing_indicator( Y, A_in, b_in, C, f, nb, method, dims, min_size, max_size, dist, expandCore, dview) if normalize_yyt_one and C is not None: C = np.array(C) nr_C = np.shape(C)[0] d_ = scipy.sparse.lil_matrix((nr_C, nr_C)) d_.setdiag(np.sqrt(np.sum(C ** 2, 1))) A_in = A_in * d_ C = old_div(C, np.sqrt(np.sum(C ** 2, 1)[:, np.newaxis])) if b_in is None: b_in = b_ print('memmaping') # we create a memory map file if not already the case, we send Cf, a # matrix that include background components C_name, Y_name, folder = creatememmap(Y, np.vstack((C, f)), dview) # we create a pixel group array (chunks for the cnmf)for the parrallelization of the process print('Updating Spatial Components using lasso lars') cct = np.diag(C.dot(C.T)) pixel_groups = [] for i in range(0, np.prod(dims) - n_pixels_per_process + 1, n_pixels_per_process): pixel_groups.append([Y_name, C_name, sn, ind2_, list( range(i, i + n_pixels_per_process)), method_ls, cct, ]) if i < np.prod(dims): pixel_groups.append([Y_name, C_name, sn, ind2_, list( range(i, np.prod(dims))), method_ls, cct]) A_ = np.zeros((d, nr + np.size(f, 0))) # init A_ if dview is not None: if 'multiprocessing' in str(type(dview)): parallel_result = dview.map_async( regression_ipyparallel, pixel_groups).get(4294967) else: parallel_result = dview.map_sync( regression_ipyparallel, pixel_groups) dview.results.clear() else: parallel_result = list(map(regression_ipyparallel, pixel_groups)) for chunk in parallel_result: for pars in chunk: px, idxs_, a = pars A_[px, idxs_] = a print("thresholding components") A_ = threshold_components(A_, dims, dview=dview, medw=medw, thr_method=thr_method, maxthr=maxthr, nrgthr=nrgthr, extract_cc=extract_cc, se=se, ss=ss) ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components if np.size(ff) > 0: ff = ff[0] print('eliminating {} empty spatial components'.format(len(ff))) A_ = np.delete(A_, list(ff[ff < nr]), 1) C = np.delete(C, list(ff[ff < nr]), 0) nr = nr - len(ff[ff < nr]) if low_rank_background: background_ff = list(filter(lambda i: i >= nb, ff - nr)) f = np.delete(f, background_ff, 0) else: background_ff = list(filter(lambda i: i >= 0, ff - nr)) f = np.delete(f, background_ff, 0) b_in = np.delete(b_in, background_ff, 1) A_ = A_[:, :nr] A_ = coo_matrix(A_) print("Computing residuals") if 'memmap' in str(type(Y)): Y_resf = parallel_dot_product(Y, f.T, dview=dview, block_size=block_size, num_blocks_per_run=num_blocks_per_run) - \ A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) else: # Y*f' - A*(C*f') Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) if update_background_components: if b_in is None: # update baseline based on residual b = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T))), 0) else: ind_b = [np.where(_b)[0] for _b in b_in.T] b = HALS4shape_bckgrnd(Y_resf, b_in, f, ind_b) else: if b_in is None: raise Exception( 'If you set the update_background_components to True you have to pass them as input to update_spatial') # try: # b = np.delete(b_in, background_ff, 0) # except NameError: b = b_in print(("--- %s seconds ---" % (time.time() - start_time))) try: # clean up # remove temporary file created print("Removing tempfiles created") shutil.rmtree(folder) except: raise Exception("Failed to delete: " + folder) return A_, b, C, f
def determine_search_location(A, dims, method='ellipse', min_size=3, max_size=8, dist=3, expandCore=iterate_structure(generate_binary_structure(2, 1), 2).astype(int), dview=None): """ compute the indices of the distance from the cm to search for the spatial component does it by following an ellipse from the cm or doing a step by step dilatation around the cm Parameters: ---------- [parsed] cm[i]: center of mass of each neuron A[:, i]: the A of each components dims: the dimension of each A's ( same usually ) dist: computed distance matrix dims: [optional] tuple x, y[, z] movie dimensions method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion min_size: [optional] int max_size: [optional] int dist: [optional] int dims: [optional] tuple x, y[, z] movie dimensions Returns: -------- dist_indicator: np.ndarray distance from the cm to search for the spatial footprint Raise: ------- Exception('You cannot pass empty (all zeros) components!') """ from scipy.ndimage.morphology import grey_dilation # we initialize the values if len(dims) == 2: d1, d2 = dims elif len(dims) == 3: d1, d2, d3 = dims d, nr = np.shape(A) A = csc_matrix(A) dist_indicator = scipy.sparse.csc_matrix((d, nr),dtype= np.float32) if method == 'ellipse': Coor = dict() # we create a matrix of size A.x of each pixel coordinate in A.y and inverse if len(dims) == 2: Coor['x'] = np.kron(np.ones(d2), list(range(d1))) Coor['y'] = np.kron(list(range(d2)), np.ones(d1)) elif len(dims) == 3: Coor['x'] = np.kron(np.ones(d3 * d2), list(range(d1))) Coor['y'] = np.kron( np.kron(np.ones(d3), list(range(d2))), np.ones(d1)) Coor['z'] = np.kron(list(range(d3)), np.ones(d2 * d1)) if not dist == np.inf: # determine search area for each neuron cm = np.zeros((nr, len(dims))) # vector for center of mass Vr = [] # cell(nr,1); dist_indicator = [] pars = [] # for each dim for i, c in enumerate(['x', 'y', 'z'][:len(dims)]): # mass center in this dim = (coor*A)/sum(A) cm[:, i] = old_div( np.dot(Coor[c], A[:, :nr].todense()), A[:, :nr].sum(axis=0)) # parrallelizing process of the construct ellipse function for i in range(nr): pars.append([Coor, cm[i], A[:, i], Vr, dims, dist, max_size, min_size, d]) if dview is None: res = list(map(construct_ellipse_parallel, pars)) else: if 'multiprocessing' in str(type(dview)): res = dview.map_async( construct_ellipse_parallel, pars).get(4294967) else: res = dview.map_sync(construct_ellipse_parallel, pars) for r in res: dist_indicator.append(r) dist_indicator = (np.asarray(dist_indicator)).squeeze().T else: raise Exception('Not implemented') dist_indicator = True * np.ones((d, nr)) elif method == 'dilate': for i in range(nr): A_temp = np.reshape(A[:, i].toarray(), dims[::-1]) if len(expandCore) > 0: if len(expandCore.shape) < len(dims): # default for 3D expandCore = iterate_structure( generate_binary_structure(len(dims), 1), 2).astype(int) A_temp = grey_dilation(A_temp, footprint=expandCore) else: A_temp = grey_dilation(A_temp, [1] * len(dims)) dist_indicator[:, i] = scipy.sparse.coo_matrix(np.squeeze(np.reshape(A_temp, (d, 1)))[:,None] > 0) else: raise Exception('Not implemented') dist_indicator = True * np.ones((d, nr)) return dist_indicator
def find_activated_parcels(ppm_map, parcels_mask, ppm_threshold, cluster_size_threshold): """Find activated parcels for a given PPM map. Parameters ---------- ppm_map : ~numpy.ndarray Posterior Probability Map data parcels_mask : ~numpy.ndarray parcellation mask data ppm_threshold : float threshold for the PPM cluster_size_threshold : int threshold for the size of the cluster of activated neurons Returns ------- list of int Activated parcels Examples -------- >>> from nilearn.image import load_img >>> parcels_img = load_img(PARCELLATION_MASK) >>> parcels_img_data = parcels_img.get_data() >>> ppm_img = load_img(os.path.join('/home/jariasal/scipy_notebook/output/114/pyhrf_output', 'jde_vem_ppm_a_nrl_Finger.nii')) >>> print(find_activated_parcels(ppm_img.get_data(), parcels_img_data, 0.99999, 50)) [ 21 101 290 417 485] """ ppm_map[np.isnan(ppm_map)] = 0. ppm_map_mask = (ppm_map >= ppm_threshold) # binary ppm_map binary_structure = generate_binary_structure( 3, 2) # structuring element that defines neighborhood connections ppm_map_labeled, nb_labels = label(ppm_map_mask, structure=binary_structure) if not nb_labels: # always select at least one cluster ppm_threshold = ppm_map.max() * 0.9999 ppm_map_mask = (ppm_map >= ppm_threshold) ppm_map_labeled, nb_labels = label(ppm_map_mask, structure=binary_structure) print("ppm_threshold set to {}".format(ppm_threshold)) # number of active voxels for each cluster clusters_size = ndi_sum(ppm_map_mask, ppm_map_labeled, index=np.arange(nb_labels + 1)) # clusters (labels) whose number of active voxels is greater than the threshold active_clusters = clusters_size >= cluster_size_threshold if not active_clusters.any(): # threshold for cluster size is too high cluster_size_threshold = np.amax(clusters_size) active_clusters = clusters_size >= cluster_size_threshold print( "cluster_size_threshold set to {}".format(cluster_size_threshold)) # voxels belonging the clusters whose number of active voxels is greater than the threshold active_clusters_mask = active_clusters[ppm_map_labeled] active_parcels = np.unique(parcels_mask[active_clusters_mask]) return active_parcels
def generate_morph_drain_curv(seg_image_input,R_critical): # The method for this function follows Hilper & Miller AWR(2001) # 1. Perform erosion for the pore space with radius of R_critical # 2. Label the eroded pore space, and leave only the pore space that is still # connected with the non-wetting phase reservoir # 3. Perform the dilation for the labelled pore space with radius of R_critical # **************************************************************************** # Currently I am provided with a 3D SignDist image which has positive values # in the pore space and 0.0 in the solid phase. # **************************************************************************** if seg_image_input.ndim == 2: pore_vol = 1.0*(seg_image_input>0.0).sum() radius = R_critical print 'Morphological Drainage: processing critical radius: '+str(radius)+' now......' # Step 1.1: Create structuring element domain_size = int(np.rint(radius*2)+2) grid = np.indices((domain_size,domain_size)) mk_circle = (grid[0]-domain_size/2)**2 + (grid[1]-domain_size/2)**2 <= radius**2 circle = np.zeros((domain_size,domain_size),dtype=np.uint8) circle[mk_circle]=1 circle = extract_shape(circle).astype(bool) # Step 1.2: Perform erosion on the pore space # NOTE: the dtype of 'seg_im_ero' is 'bool' seg_im_ero = morphology.binary_erosion(seg_image_input>0.0,structure=circle,border_value=1) # NOTE: 'border_value' for erosion should be 'True' # Step 2: Label the eroded pore space # NOTE: Assume the NW phase reservoir is at the first layer of the domain # i.e. at seg_image[0,:] - adjust it if this does not suit your need # For erosion, assume that diagonals are not considered seg_im_ero_label_temp,num_features = measurements.label(seg_im_ero,structure=morphology.generate_binary_structure(2,1)) #seg_im_ero_label_temp,num_features = measurements.label(seg_im_ero,structure=morphology.generate_binary_structure(2,2)) # NOTE: Here I assume the inlet is at the first layer of the array's axis=2 (i.e. domain[0,:,:])\ # You can always change to any other layers as the inlet for this drainage. label_check = seg_im_ero_label_temp[0,seg_im_ero_label_temp[0,:]!=0] label_check = np.unique(label_check) # NOTE the following lines are only for your to check things # ******************** For check *******************************# # It assign the labelled array as: NW -> 1, W -> 2, Solid -> 0 #seg_im_ero_label_show = seg_im_ero_label.copy() #seg_im_ero_label_show[seg_im_ero_label_show !=1] = 2 #seg_im_ero_label_show[np.logical_not(seg_image_2d)]=0 # ******************** End: for check **************************# seg_im_ero_label = np.zeros_like(seg_im_ero_label_temp,dtype=bool) for labels in label_check: seg_im_ero_label = np.logical_or(seg_im_ero_label,seg_im_ero_label_temp==labels) #seg_im_ero_label = seg_im_ero_label.astype(np.uint8) # Step 3: perform dilation on the labelled pore space seg_im_ero_label_dil = morphology.binary_dilation(seg_im_ero_label,structure=circle,border_value=0) # NOTE: 'border_value' for dilation should be 'False' # NOTE: the dtype of 'seg_im_ero_label_dil' is 'bool' seg_im_ero_label_dil[seg_image_input<=0.0]=False Sw = 1.0 - seg_im_ero_label_dil.sum()/pore_vol else: pore_vol = 1.0*(seg_image_input>0.0).sum() radius = R_critical print 'Morphological Drainage: processing critical radius: '+str(radius)+' now......' # Step 1.1: Create structuring element domain_size = int(np.rint(radius*2)+2) grid = np.indices((domain_size,domain_size,domain_size)) mk_circle = (grid[0]-domain_size/2)**2 + (grid[1]-domain_size/2)**2 + (grid[2]-domain_size/2)**2 <= radius**2 circle = np.zeros((domain_size,domain_size,domain_size),dtype=np.uint8) circle[mk_circle]=1 circle = extract_shape(circle).astype(bool) # Step 1.2: Perform erosion on the pore space # NOTE: the dtype of 'seg_im_ero' is 'bool' seg_im_ero = morphology.binary_erosion(seg_image_input>0.0,structure=circle,border_value=1) # NOTE: 'border_value' for erosion should be 'True' # Step 2: Label the eroded pore space # NOTE: Assume the NW phase reservoir is at the first layer of the domain # i.e. at seg_image[0,:] - adjust it if this does not suit your need # For erosion, assume that diagonals are not considered seg_im_ero_label_temp,num_features = measurements.label(seg_im_ero,structure=morphology.generate_binary_structure(3,1)) #seg_im_ero_label_temp,num_features = measurements.label(seg_im_ero,structure=morphology.generate_binary_structure(3,3)) # NOTE: Here I assume the inlet is at the first layer of the array's axis=2 (i.e. domain[0,:,:])\ # You can always change to any other layers as the inlet for this drainage. label_check = seg_im_ero_label_temp[0,seg_im_ero_label_temp[0,:]!=0] label_check = np.unique(label_check) # NOTE the following lines are only for your to check things # ******************** For check *******************************# # It assign the labelled array as: NW -> 1, W -> 2, Solid -> 0 #seg_im_ero_label_show = seg_im_ero_label.copy() #seg_im_ero_label_show[seg_im_ero_label_show !=1] = 2 #seg_im_ero_label_show[np.logical_not(seg_image_2d)]=0 # ******************** End: for check **************************# seg_im_ero_label = np.zeros_like(seg_im_ero_label_temp,dtype=bool) for labels in label_check: seg_im_ero_label = np.logical_or(seg_im_ero_label,seg_im_ero_label_temp==labels) #seg_im_ero_label = seg_im_ero_label.astype(np.uint8) # Step 3: perform dilation on the labelled pore space seg_im_ero_label_dil = morphology.binary_dilation(seg_im_ero_label,structure=circle,border_value=0) # NOTE: 'border_value' for dilation should be 'False' # NOTE: the dtype of 'seg_im_ero_label_dil' is 'bool' seg_im_ero_label_dil[seg_image_input<=0.0]=False Sw = 1.0 - seg_im_ero_label_dil.sum()/pore_vol #end if return Sw
def __init__(self, fnames=None, dims=None, dxy=(1, 1), border_pix=0, del_duplicates=False, low_rank_background=True, memory_fact=1, n_processes=1, nb_patch=1, p_ssub=2, p_tsub=2, remove_very_bad_comps=False, rf=None, stride=None, check_nan=True, n_pixels_per_process=None, k=30, alpha_snmf=100, center_psf=False, gSig=[5, 5], gSiz=None, init_iter=2, method_init='greedy_roi', min_corr=.85, min_pnr=20, gnb=1, normalize_init=True, options_local_NMF=None, ring_size_factor=1.5, rolling_length=100, rolling_sum=True, ssub=2, ssub_B=2, tsub=2, block_size_spat=5000, num_blocks_per_run_spat=20, block_size_temp=5000, num_blocks_per_run_temp=20, update_background_components=True, method_deconvolution='oasis', p=2, s_min=None, do_merge=True, merge_thresh=0.8, decay_time=0.4, fr=30, min_SNR=2.5, rval_thr=0.8, N_samples_exceptionality=None, batch_update_suff_stat=False, expected_comps=500, iters_shape=5, max_comp_update_shape=np.inf, max_num_added=5, min_num_trial=5, minibatch_shape=100, minibatch_suff_stat=5, n_refit=0, num_times_comp_updated=np.inf, simultaneously=False, sniper_mode=False, test_both=False, thresh_CNN_noisy=0.5, thresh_fitness_delta=-50, thresh_fitness_raw=None, thresh_overlap=0.5, update_freq=200, update_num_comps=True, use_dense=True, use_peak_max=True, only_init_patch=True, var_name_hdf5='mov', max_merge_area=None, params_dict={}, ): """Class for setting the processing parameters. All parameters for CNMF, online-CNMF, quality testing, and motion correction can be set here and then used in the various processing pipeline steps. The prefered way to set parameters is by using the set function, where a subclass is determined and a dictionary is passed. The whole dictionary can also be initialized at once by passing a dictionary params_dict when initializing the CNMFParams object. Direct setting of the positional arguments in CNMFParams is only present for backwards compatibility reasons and should not be used if possible. Args: Any parameter that is not set get a default value specified by the dictionary default options DATA PARAMETERS (CNMFParams.data) ##### fnames: list[str] list of complete paths to files that need to be processed dims: (int, int), default: computed from fnames dimensions of the FOV in pixels fr: float, default: 30 imaging rate in frames per second decay_time: float, default: 0.4 length of typical transient in seconds dxy: (float, float) spatial resolution of FOV in pixels per um var_name_hdf5: str, default: 'mov' if loading from hdf5 name of the variable to load caiman_version: str version of CaImAn being used last_commit: str hash of last commit in the caiman repo mmap_F: list[str] paths to F-order memory mapped files after motion correction mmap_C: str path to C-order memory mapped file after motion correction PATCH PARAMS (CNMFParams.patch)###### rf: int or None, default: None Half-size of patch in pixels. If None, no patches are constructed and the whole FOV is processed jointly stride: int or None, default: None Overlap between neighboring patches in pixels. nb_patch: int, default: 1 Number of (local) background components per patch border_pix: int, default: 0 Number of pixels to exclude around each border. low_rank_background: bool, default: True Whether to update the background using a low rank approximation. If False all the nonzero elements of the background components are updated using hals (to be used with one background per patch) del_duplicates: bool, default: False Delete duplicate components in the overlaping regions between neighboring patches. If False, then merging is used. only_init: bool, default: True whether to run only the initialization p_patch: int, default: 0 order of AR dynamics when processing within a patch skip_refinement: bool, default: False Whether to skip refinement of components (deprecated?) remove_very_bad_comps: bool, default: True Whether to remove (very) bad quality components during patch processing p_ssub: float, default: 2 Spatial downsampling factor p_tsub: float, default: 2 Temporal downsampling factor memory_fact: float, default: 1 unitless number for increasing the amount of available memory n_processes: int Number of processes used for processing patches in parallel in_memory: bool, default: True Whether to load patches in memory PRE-PROCESS PARAMS (CNMFParams.preprocess) ############# sn: np.array or None, default: None noise level for each pixel noise_range: [float, float], default: [.25, .5] range of normalized frequencies over which to compute the PSD for noise determination noise_method: 'mean'|'median'|'logmexp', default: 'mean' PSD averaging method for computing the noise std max_num_samples_fft: int, default: 3*1024 Chunk size for computing the PSD of the data (for memory considerations) n_pixels_per_process: int, default: 1000 Number of pixels to be allocated to each process compute_g': bool, default: False whether to estimate global time constant p: int, default: 2 order of AR indicator dynamics lags: int, default: 5 number of lags to be considered for time constant estimation include_noise: bool, default: False flag for using noise values when estimating g pixels: list, default: None pixels to be excluded due to saturation check_nan: bool, default: True whether to check for NaNs INIT PARAMS (CNMFParams.init)############### K: int, default: 30 number of components to be found (per patch or whole FOV depending on whether rf=None) SC_kernel: {'heat', 'cos', binary'}, default: 'heat' kernel for graph affinity matrix SC_sigma: float, default: 1 variance for SC kernel SC_thr: float, default: 0, threshold for affinity matrix SC_normalize: bool, default: True standardize entries prior to computing the affinity matrix SC_use_NN: bool, default: False sparsify affinity matrix by using only nearest neighbors SC_nnn: int, default: 20 number of nearest neighbors to use gSig: [int, int], default: [5, 5] radius of average neurons (in pixels) gSiz: [int, int], default: [int(round((x * 2) + 1)) for x in gSig], half-size of bounding box for each neuron center_psf: bool, default: False whether to use 1p data processing mode. Set to true for 1p ssub: float, default: 2 spatial downsampling factor tsub: float, default: 2 temporal downsampling factor nb: int, default: 1 number of background components lambda_gnmf: float, default: 1. regularization weight for graph NMF maxIter: int, default: 5 number of HALS iterations during initialization method_init: 'greedy_roi'|'greedy_pnr'|'sparse_NMF'|'local_NMF' default: 'greedy_roi' initialization method. use 'greedy_pnr' for 1p processing and 'sparse_NMF' for dendritic processing. min_corr: float, default: 0.85 minimum value of correlation image for determining a candidate component during greedy_pnr min_pnr: float, default: 20 minimum value of psnr image for determining a candidate component during greedy_pnr ring_size_factor: float, default: 1.5 radius of ring (*gSig) for computing background during greedy_pnr ssub_B: float, default: 2 downsampling factor for background during greedy_pnr init_iter: int, default: 2 number of iterations during greedy_pnr (1p) initialization nIter: int, default: 5 number of rank-1 refinement iterations during greedy_roi initialization rolling_sum: bool, default: True use rolling sum (as opposed to full sum) for determining candidate centroids during greedy_roi rolling_length: int, default: 100 width of rolling window for rolling sum option kernel: np.array or None, default: None user specified template for greedyROI max_iter_snmf : int, default: 500 maximum number of iterations for sparse NMF initialization alpha_snmf: float, default: 100 sparse NMF sparsity regularization weight sigma_smooth_snmf : (float, float, float), default: (.5,.5,.5) std of Gaussian kernel for smoothing data in sparse_NMF perc_baseline_snmf: float, default: 20 percentile to be removed from the data in sparse_NMF prior to decomposition normalize_init: bool, default: True whether to equalize the movies during initialization options_local_NMF: dict dictionary with parameters to pass to local_NMF initializer SPATIAL PARAMS (CNMFParams.spatial) ########## method_exp: 'dilate'|'ellipse', default: 'dilate' method for expanding footprint of spatial components dist: float, default: 3 expansion factor of ellipse expandCore: morphological element, default: None(?) morphological element for expanding footprints under dilate nb: int, default: 1 number of global background components n_pixels_per_process: int, default: 1000 number of pixels to be processed by each worker thr_method: 'nrg'|'max', default: 'nrg' thresholding method maxthr: float, default: 0.1 Max threshold nrgthr: float, default: 0.9999 Energy threshold extract_cc: bool, default: True whether to extract connected components during thresholding (might want to turn to False for dendritic imaging) medw: (int, int) default: None window of median filter (set to (3,)*len(dims) in cnmf.fit) se: np.array or None, default: None Morphological closing structuring element (set to np.ones((3,)*len(dims), dtype=np.uint8) in cnmf.fit) ss: np.array or None, default: None Binary element for determining connectivity (set to np.ones((3,)*len(dims), dtype=np.uint8) in cnmf.fit) update_background_components: bool, default: True whether to update the spatial background components method_ls: 'lasso_lars'|'nnls_L0', default: 'lasso_lars' 'nnls_L0'. Nonnegative least square with L0 penalty 'lasso_lars' lasso lars function from scikit learn block_size : int, default: 5000 Number of pixels to process at the same time for dot product. Reduce if you face memory problems num_blocks_per_run: int, default: 20 Parallelization of A'*Y operation normalize_yyt_one: bool, default: True Whether to normalize the C and A matrices so that diag(C*C.T) = 1 during update spatial TEMPORAL PARAMS (CNMFParams.temporal)########### ITER: int, default: 2 block coordinate descent iterations method_deconvolution: 'oasis'|'cvxpy'|'oasis', default: 'oasis' method for solving the constrained deconvolution problem ('oasis','cvx' or 'cvxpy') if method cvxpy, primary and secondary (if problem unfeasible for approx solution) solvers: 'ECOS'|'SCS', default: ['ECOS', 'SCS'] solvers to be used with cvxpy, can be 'ECOS','SCS' or 'CVXOPT' p: 0|1|2, default: 2 order of AR indicator dynamics memory_efficient: False bas_nonneg: bool, default: True whether to set a non-negative baseline (otherwise b >= min(y)) noise_range: [float, float], default: [.25, .5] range of normalized frequencies over which to compute the PSD for noise determination noise_method: 'mean'|'median'|'logmexp', default: 'mean' PSD averaging method for computing the noise std lags: int, default: 5 number of autocovariance lags to be considered for time constant estimation optimize_g: bool, default: False flag for optimizing time constants fudge_factor: float (close but smaller than 1) default: .96 bias correction factor for discrete time constants nb: int, default: 1 number of global background components verbosity: bool, default: False whether to be verbose block_size : int, default: 5000 Number of pixels to process at the same time for dot product. Reduce if you face memory problems num_blocks_per_run: int, default: 20 Parallelization of A'*Y operation s_min: float or None, default: None Minimum spike threshold amplitude (computed in the code if used). MERGE PARAMS (CNMFParams.merge)##### do_merge: bool, default: True Whether or not to merge thr: float, default: 0.8 Trace correlation threshold for merging two components. merge_parallel: bool, default: False Perform merging in parallel max_merge_area: int or None, default: None maximum area (in pixels) of merged components, used to determine whether to merge components during fitting process QUALITY EVALUATION PARAMETERS (CNMFParams.quality)########### min_SNR: float, default: 2.5 trace SNR threshold. Traces with SNR above this will get accepted SNR_lowest: float, default: 0.5 minimum required trace SNR. Traces with SNR below this will get rejected rval_thr: float, default: 0.8 space correlation threshold. Components with correlation higher than this will get accepted rval_lowest: float, default: -1 minimum required space correlation. Components with correlation below this will get rejected use_cnn: bool, default: True flag for using the CNN classifier. min_cnn_thr: float, default: 0.9 CNN classifier threshold. Components with score higher than this will get accepted cnn_lowest: float, default: 0.1 minimum required CNN threshold. Components with score lower than this will get rejected. gSig_range: list or integers, default: None gSig scale values for CNN classifier. In not None, multiple values are tested in the CNN classifier. ONLINE CNMF (ONACID) PARAMETERS (CNMFParams.online)##### N_samples_exceptionality: int, default: np.ceil(decay_time*fr), Number of frames over which trace SNR is computed (usually length of a typical transient) batch_update_suff_stat: bool, default: False Whether to update sufficient statistics in batch mode ds_factor: int, default: 1, spatial downsampling factor for faster processing (if > 1) dist_shape_update: bool, default: False, update shapes in a distributed fashion epochs: int, default: 1, number of times to go over data expected_comps: int, default: 500 number of expected components (for memory allocation purposes) init_batch: int, default: 200, length of mini batch used for initialization init_method: 'bare'|'cnmf'|'seeded', default: 'bare', initialization method iters_shape: int, default: 5 Number of block-coordinate decent iterations for each shape update max_comp_update_shape: int, default: np.inf Maximum number of spatial components to be updated at each time max_num_added: int, default: 5 Maximum number of new components to be added in each frame max_shifts_online: int, default: 10, Maximum shifts for motion correction during online processing min_SNR: float, default: 2.5 Trace SNR threshold for accepting a new component min_num_trial: int, default: 5 Number of mew possible components for each frame minibatch_shape: int, default: 100 Number of frames stored in rolling buffer minibatch_suff_stat: int, default: 5 mini batch size for updating sufficient statistics motion_correct: bool, default: True Whether to perform motion correction during online processing movie_name_online: str, default: 'online_movie.avi' Name of saved movie (appended in the data directory) normalize: bool, default: False Whether to normalize each frame prior to online processing n_refit: int, default: 0 Number of additional iterations for computing traces num_times_comp_updated: int, default: np.inf path_to_model: str, default: os.path.join(caiman_datadir(), 'model', 'cnn_model_online.h5') Path to online CNN classifier rval_thr: float, default: 0.8 space correlation threshold for accepting a new component save_online_movie: bool, default: False Whether to save the results movie show_movie: bool, default: False Whether to display movie of online processing simultaneously: bool, default: False Whether to demix and deconvolve simultaneously sniper_mode: bool, default: False Whether to use the online CNN classifier for screening candidate components (otherwise space correlation is used) test_both: bool, default: False Whether to use both the CNN and space correlation for screening new components thresh_CNN_noisy: float, default: 0,5, Threshold for the online CNN classifier thresh_fitness_delta: float (negative) Derivative test for detecting traces thresh_fitness_raw: float (negative), default: computed from min_SNR Threshold value for testing trace SNR thresh_overlap: float, default: 0.5 Intersection-over-Union space overlap threshold for screening new components update_freq: int, default: 200 Update each shape at least once every X frames when in distributed mode update_num_comps: bool, default: True Whether to search for new components use_dense: bool, default: True Whether to store and represent A and b as a dense matrix use_peak_max: bool, default: True Whether to find candidate centroids using skimage's find local peaks function MOTION CORRECTION PARAMETERS (CNMFParams.motion)#### border_nan: bool or str, default: 'copy' flag for allowing NaN in the boundaries. True allows NaN, whereas 'copy' copies the value of the nearest data point. gSig_filt: int or None, default: None size of kernel for high pass spatial filtering in 1p data. If None no spatial filtering is performed is3D: bool, default: False flag for 3D recordings for motion correction max_deviation_rigid: int, default: 3 maximum deviation in pixels between rigid shifts and shifts of individual patches max_shifts: (int, int), default: (6,6) maximum shifts per dimension in pixels. min_mov: float or None, default: None minimum value of movie. If None it get computed. niter_rig: int, default: 1 number of iterations rigid motion correction. nonneg_movie: bool, default: True flag for producing a non-negative movie. num_frames_split: int, default: 80 split movie every x frames for parallel processing num_splits_to_process_els, default: [7, None] num_splits_to_process_rig, default: None overlaps: (int, int), default: (24, 24) overlap between patches in pixels in pw-rigid motion correction. pw_rigid: bool, default: False flag for performing pw-rigid motion correction. shifts_opencv: bool, default: True flag for applying shifts using cubic interpolation (otherwise FFT) splits_els: int, default: 14 number of splits across time for pw-rigid registration splits_rig: int, default: 14 number of splits across time for rigid registration strides: (int, int), default: (96, 96) how often to start a new patch in pw-rigid registration. Size of each patch will be strides + overlaps upsample_factor_grid" int, default: 4 motion field upsampling factor during FFT shifts. use_cuda: bool, default: False flag for using a GPU. """ self.data = { 'fnames': fnames, 'dims': dims, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'var_name_hdf5': var_name_hdf5, 'caiman_version': '1.6.2', 'last_commit': None, 'mmap_F': None, 'mmap_C': None } self.patch = { 'border_pix': border_pix, 'del_duplicates': del_duplicates, 'in_memory': True, 'low_rank_background': low_rank_background, 'memory_fact': memory_fact, 'n_processes': n_processes, 'nb_patch': nb_patch, 'only_init': only_init_patch, 'p_patch': 0, # AR order within patch 'remove_very_bad_comps': remove_very_bad_comps, 'rf': rf, 'skip_refinement': False, 'p_ssub': p_ssub, # spatial downsampling factor 'stride': stride, 'p_tsub': p_tsub, # temporal downsampling factor } self.preprocess = { 'check_nan': check_nan, 'compute_g': False, # flag for estimating global time constant 'include_noise': False, # flag for using noise values when estimating g # number of autocovariance lags to be considered for time constant estimation 'lags': 5, 'max_num_samples_fft': 3 * 1024, 'n_pixels_per_process': n_pixels_per_process, 'noise_method': 'mean', # averaging method ('mean','median','logmexp') 'noise_range': [0.25, 0.5], # range of normalized frequencies over which to average 'p': p, # order of AR indicator dynamics 'pixels': None, # pixels to be excluded due to saturation 'sn': None, # noise level for each pixel } self.init = { 'K': k, # number of components, 'SC_kernel': 'heat', # kernel for graph affinity matrix 'SC_sigma' : 1, # std for SC kernel 'SC_thr': 0, # threshold for affinity matrix 'SC_normalize': True, # standardize entries prior to # computing affinity matrix 'SC_use_NN': False, # sparsify affinity matrix by using # only nearest neighbors 'SC_nnn': 20, # number of nearest neighbors to use 'alpha_snmf': alpha_snmf, 'center_psf': center_psf, 'gSig': gSig, # size of bounding box 'gSiz': gSiz, 'init_iter': init_iter, 'kernel': None, # user specified template for greedyROI 'lambda_gnmf' :1, # regularization weight for graph NMF 'maxIter': 5, # number of HALS iterations 'max_iter_snmf': 500, 'method_init': method_init, # can be greedy_roi, greedy_pnr sparse_nmf, local_NMF 'min_corr': min_corr, 'min_pnr': min_pnr, 'nIter': 5, # number of refinement iterations 'nb': gnb, # number of global background components # whether to pixelwise equalize the movies during initialization 'normalize_init': normalize_init, # dictionary with parameters to pass to local_NMF initializaer 'options_local_NMF': options_local_NMF, 'perc_baseline_snmf': 20, 'ring_size_factor': ring_size_factor, 'rolling_length': rolling_length, 'rolling_sum': rolling_sum, 'sigma_smooth_snmf': (.5, .5, .5), 'ssub': ssub, # spatial downsampling factor 'ssub_B': ssub_B, 'tsub': tsub, # temporal downsampling factor } self.spatial = { 'block_size_spat': block_size_spat, # number of pixels to parallelize residual computation ** DECREASE IF MEMORY ISSUES 'dist': 3, # expansion factor of ellipse 'expandCore': iterate_structure(generate_binary_structure(2, 1), 2).astype(int), # Flag to extract connected components (might want to turn to False for dendritic imaging) 'extract_cc': True, 'maxthr': 0.1, # Max threshold 'medw': None, # window of median filter # method for determining footprint of spatial components ('ellipse' or 'dilate') 'method_exp': 'dilate', # 'nnls_L0'. Nonnegative least square with L0 penalty # 'lasso_lars' lasso lars function from scikit learn 'method_ls': 'lasso_lars', # number of pixels to be processed by each worker 'n_pixels_per_process': n_pixels_per_process, 'nb': gnb, # number of background components 'normalize_yyt_one': True, 'nrgthr': 0.9999, # Energy threshold 'num_blocks_per_run_spat': num_blocks_per_run_spat, # number of process to parallelize residual computation ** DECREASE IF MEMORY ISSUES 'se': np.ones((3, 3), dtype='uint8'), # Morphological closing structuring element 'ss': np.ones((3, 3), dtype='uint8'), # Binary element for determining connectivity 'thr_method': 'nrg', # Method of thresholding ('max' or 'nrg') # whether to update the background components in the spatial phase 'update_background_components': update_background_components, } self.temporal = { 'ITER': 2, # block coordinate descent iterations # flag for setting non-negative baseline (otherwise b >= min(y)) 'bas_nonneg': False, # number of pixels to process at the same time for dot product. Make it # smaller if memory problems 'block_size_temp': block_size_temp, # number of pixels to parallelize residual computation ** DECREASE IF MEMORY ISSUES # bias correction factor (between 0 and 1, close to 1) 'fudge_factor': .96, # number of autocovariance lags to be considered for time constant estimation 'lags': 5, 'optimize_g': False, # flag for optimizing time constants 'memory_efficient': False, # method for solving the constrained deconvolution problem ('oasis','cvx' or 'cvxpy') # if method cvxpy, primary and secondary (if problem unfeasible for approx # solution) solvers to be used with cvxpy, can be 'ECOS','SCS' or 'CVXOPT' 'method_deconvolution': method_deconvolution, # 'cvxpy', # 'oasis' 'nb': gnb, # number of background components 'noise_method': 'mean', # averaging method ('mean','median','logmexp') 'noise_range': [.25, .5], # range of normalized frequencies over which to average 'num_blocks_per_run_temp': num_blocks_per_run_temp, # number of process to parallelize residual computation ** DECREASE IF MEMORY ISSUES 'p': p, # order of AR indicator dynamics 's_min': s_min, # minimum spike threshold 'solvers': ['ECOS', 'SCS'], 'verbosity': False, } self.merging = { 'do_merge': do_merge, 'merge_thr': merge_thresh, 'merge_parallel': False, 'max_merge_area': max_merge_area } self.quality = { 'SNR_lowest': 0.5, # minimum accepted SNR value 'cnn_lowest': 0.1, # minimum accepted value for CNN classifier 'gSig_range': None, # range for gSig scale for CNN classifier 'min_SNR': min_SNR, # transient SNR threshold 'min_cnn_thr': 0.9, # threshold for CNN classifier 'rval_lowest': -1, # minimum accepted space correlation 'rval_thr': rval_thr, # space correlation threshold 'use_cnn': True, # use CNN based classifier } self.online = { 'N_samples_exceptionality': N_samples_exceptionality, # timesteps to compute SNR 'batch_update_suff_stat': batch_update_suff_stat, 'dist_shape_update': False, # update shapes in a distributed way 'ds_factor': 1, # spatial downsampling for faster processing 'epochs': 1, # number of epochs 'expected_comps': expected_comps, # number of expected components 'init_batch': 200, # length of mini batch for initialization 'init_method': 'bare', # initialization method for first batch, 'iters_shape': iters_shape, # number of block-CD iterations 'max_comp_update_shape': max_comp_update_shape, 'max_num_added': max_num_added, # maximum number of new components for each frame 'max_shifts_online': 10, # maximum shifts during motion correction 'min_SNR': min_SNR, # minimum SNR for accepting a new trace 'min_num_trial': min_num_trial, # number of mew possible components for each frame 'minibatch_shape': minibatch_shape, # number of frames in each minibatch 'minibatch_suff_stat': minibatch_suff_stat, 'motion_correct': True, # flag for motion correction 'movie_name_online': 'online_movie.avi', # filename of saved movie (appended to directory where data is located) 'normalize': False, # normalize frame 'n_refit': n_refit, # Additional iterations to simultaneously refit # path to CNN model for testing new comps 'num_times_comp_updated': num_times_comp_updated, 'path_to_model': os.path.join(caiman_datadir(), 'model', 'cnn_model_online.h5'), 'rval_thr': rval_thr, # space correlation threshold 'save_online_movie': False, # flag for saving online movie 'show_movie': False, # display movie online 'simultaneously': simultaneously, # demix and deconvolve simultaneously 'sniper_mode': sniper_mode, # flag for using CNN 'test_both': test_both, # flag for using both CNN and space correlation 'thresh_CNN_noisy': thresh_CNN_noisy, # threshold for online CNN classifier 'thresh_fitness_delta': thresh_fitness_delta, 'thresh_fitness_raw': thresh_fitness_raw, # threshold for trace SNR (computed below) 'thresh_overlap': thresh_overlap, 'update_freq': update_freq, # update every shape at least once every update_freq steps 'update_num_comps': update_num_comps, # flag for searching for new components 'use_dense': use_dense, # flag for representation and storing of A and b 'use_peak_max': use_peak_max, # flag for finding candidate centroids } self.motion = { 'border_nan': 'copy', # flag for allowing NaN in the boundaries 'gSig_filt': None, # size of kernel for high pass spatial filtering in 1p data 'is3D': False, # flag for 3D recordings for motion correction 'max_deviation_rigid': 3, # maximum deviation between rigid and non-rigid 'max_shifts': (6, 6), # maximum shifts per dimension (in pixels) 'min_mov': None, # minimum value of movie 'niter_rig': 1, # number of iterations rigid motion correction 'nonneg_movie': True, # flag for producing a non-negative movie 'num_frames_split': 80, # split across time every x frames 'num_splits_to_process_els': [7, None], 'num_splits_to_process_rig': None, 'overlaps': (32, 32), # overlap between patches in pw-rigid motion correction 'pw_rigid': False, # flag for performing pw-rigid motion correction 'shifts_opencv': True, # flag for applying shifts using cubic interpolation (otherwise FFT) 'splits_els': 14, # number of splits across time for pw-rigid registration 'splits_rig': 14, # number of splits across time for rigid registration 'strides': (96, 96), # how often to start a new patch in pw-rigid registration 'upsample_factor_grid': 4, # motion field upsampling factor during FFT shifts 'use_cuda': False # flag for using a GPU } self.change_params(params_dict) self.data['last_commit'] = '-'.join(caiman.utils.utils.get_caiman_version()) if self.data['dims'] is None and self.data['fnames'] is not None: self.data['dims'] = get_file_size(self.data['fnames'], var_name_hdf5=self.data['var_name_hdf5'])[0] if self.data['fnames'] is not None: if isinstance(self.data['fnames'], str): self.data['fnames'] = [self.data['fnames']] if self.motion['is3D']: T = get_file_size(self.data['fnames'], var_name_hdf5=self.data['var_name_hdf5'])[0][0] else: T = get_file_size(self.data['fnames'], var_name_hdf5=self.data['var_name_hdf5'])[1] if len(self.data['fnames']) > 1: T = T[0] num_splits = T//max(self.motion['num_frames_split'],10) self.motion['splits_els'] = num_splits self.motion['splits_rig'] = num_splits self.online['movie_name_online'] = os.path.join(os.path.dirname(self.data['fnames'][0]), self.online['movie_name_online']) if self.online['N_samples_exceptionality'] is None: self.online['N_samples_exceptionality'] = np.ceil(self.data['fr'] * self.data['decay_time']).astype('int') if self.online['thresh_fitness_raw'] is None: self.online['thresh_fitness_raw'] = scipy.special.log_ndtr( -self.online['min_SNR']) * self.online['N_samples_exceptionality'] self.online['max_shifts_online'] = (np.array(self.online['max_shifts_online']) / self.online['ds_factor']).astype(int) if self.init['gSig'] is None: self.init['gSig'] = [-1, -1] if self.init['gSiz'] is None: self.init['gSiz'] = [2*gs + 1 for gs in self.init['gSig']] self.init['gSiz'] = [gz if gz % 2 else gz + 1 for gz in self.init['gSiz']] if gnb <= 0: logging.warning("gnb={0}, hence setting keys nb_patch and low_rank_background ".format(gnb) + "in group patch automatically.") self.set('patch', {'nb_patch': gnb, 'low_rank_background': None}) if gnb == -1: logging.warning("gnb=-1, hence setting key update_background_components " + "in group spatial automatically to False.") self.set('spatial', {'update_background_components': False}) if method_init=='corr_pnr' and ring_size_factor is not None: logging.warning("using CNMF-E's ringmodel for background hence setting key " + "normalize_init in group init automatically to False.") self.set('init', {'normalize_init': False})
def update_spatial_components(Y, C=None, f=None, A_in=None, sn=None, dims=None, min_size=3, max_size=8, dist=3, normalize_yyt_one=True, method='ellipse', expandCore=None, dview=None, n_pixels_per_process=128, medw=(3, 3), thr_method='nrg', maxthr=0.1, nrgthr=0.9999, extract_cc=True, se=np.ones((3, 3), dtype=np.int), ss=np.ones((3, 3), dtype=np.int), nb=1, method_ls='nnls_L0'): """update spatial footprints and background through Basis Pursuit Denoising for each pixel i solve the problem [A(i,:),b(i)] = argmin sum(A(i,:)) subject to || Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T); for each pixel the search is limited to a few spatial components Parameters ---------- Y: np.ndarray (2D or 3D) movie, raw data in 2D or 3D (pixels x time). C: np.ndarray calcium activity of each neuron. f: np.ndarray temporal profile of background activity. A_in: np.ndarray spatial profile of background activity. If A_in is boolean then it defines the spatial support of A. Otherwise it is used to determine it through determine_search_location dims: [optional] tuple x, y[, z] movie dimensions min_size: [optional] int max_size: [optional] int dist: [optional] int sn: [optional] float noise associated with each pixel if known backend [optional] str 'ipyparallel', 'single_thread' single_thread:no parallelization. It can be used with small datasets. ipyparallel: uses ipython clusters and then send jobs to each of them SLURM: use the slurm scheduler n_pixels_per_process: [optional] int number of pixels to be processed by each thread method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion dview: view on ipyparallel client you need to create an ipyparallel client and pass a view on the processors (client = Client(), dview=client[:]) medw, thr_method, maxthr, nrgthr, extract_cc, se, ss: [optional] Parameters for components post-processing. Refer to spatial.threshold_components for more details nb: [optional] int Number of background components method_ls: method to perform the regression for the basis pursuit denoising. 'nnls_L0'. Nonnegative least square with L0 penalty 'lasso_lars' lasso lars function from scikit learn 'lasso_lars_old' lasso lars from old implementation, will be deprecated normalize_yyt_one: bool wheter to norrmalize the C and A matrices so that diag(C*C.T) are ones Returns -------- A: np.ndarray new estimate of spatial footprints b: np.ndarray new estimate of spatial background C: np.ndarray temporal components (updated only when spatial components are completely removed) f: np.ndarray same as f_in except if empty component deleted. """ C = np.array(C) if normalize_yyt_one: # cct=np.diag(C.dot(C.T)) nr_C = np.shape(C)[0] d = scipy.sparse.lil_matrix((nr_C, nr_C)) d.setdiag(np.sqrt(np.sum(C**2, 1))) A_in = A_in * d C = old_div(C, np.sqrt(np.sum(C**2, 1)[:, np.newaxis])) if expandCore is None: expandCore = iterate_structure(generate_binary_structure(2, 1), 2).astype(int) if dims is None: raise Exception('You need to define the input dimensions') if Y.ndim < 2 and not isinstance(Y, basestring): Y = np.atleast_2d(Y) if Y.shape[1] == 1: raise Exception('Dimension of Matrix Y must be pixels x time') if C is not None: C = np.atleast_2d(C) if C.shape[1] == 1: raise Exception('Dimension of Matrix C must be neurons x time') if f is not None: f = np.atleast_2d(f) if f.shape[1] == 1: raise Exception('Dimension of Matrix f must be background comps x time ') if (A_in is None) and (C is None): raise Exception('Either A or C need to be determined') if A_in is not None: if len(A_in.shape) == 1: A_in = np.atleast_2d(A_in).T if A_in.shape[0] == 1: raise Exception('Dimension of Matrix A must be pixels x neurons ') start_time = time.time() [d, T] = np.shape(Y) if A_in is None: A_in = np.ones((d, np.shape(C)[1]), dtype=bool) if n_pixels_per_process > d: print( 'The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decreasing suitably.') n_pixels_per_process = d if f is not None: nb = f.shape[0] else: if b is not None: nb = b.shape[1] if A_in.dtype == bool: IND = A_in.copy() print("spatial support for each components given by the user") if C is None: INDav = old_div(IND.astype('float32'), np.sum(IND, axis=0)) px = (np.sum(IND, axis=1) > 0) model = NMF(n_components=nb, init='random', random_state=0) b = model.fit_transform(np.maximum(Y[~px, :], 0)) f = model.components_.squeeze() #f = np.mean(Y[~px,:],axis=0) Y_resf = np.dot(Y, f.T) b = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T)), 0)) #b = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0) C = np.fmax(csr_matrix(INDav.T).dot(Y) - np.outer(INDav.T.dot(b), f), 0) f = np.atleast_2d(f) else: IND = determine_search_location( A_in, dims, method=method, min_size=min_size, max_size=max_size, dist=dist, expandCore=expandCore, dview=dview) print("found spatial support for each component") if C is None: raise Exception('You need to provide estimate of C and f') print((np.shape(A_in))) Cf = np.vstack((C, f)) # create matrix that include background components nr, _ = np.shape(C) # number of neurons ind2_ = [np.hstack((np.where(iid_)[0], nr + np.arange(f.shape[0]))) if np.size(np.where(iid_)[0]) > 0 else [] for iid_ in IND] if os.environ.get('SLURM_SUBMIT_DIR') is not None: tmpf = os.environ.get('SLURM_SUBMIT_DIR') print(('cluster temporary folder:' + tmpf)) folder = tempfile.mkdtemp(dir=tmpf) else: folder = tempfile.mkdtemp() if dview is None: Y_name = Y C_name = Cf else: C_name = os.path.join(folder, 'C_temp.npy') np.save(C_name, Cf) if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name = Y.filename # if not create a memory mapped version (necessary for parallelization) elif isinstance(Y, basestring) or dview is None: Y_name = Y else: raise Exception('Not implemented consistently') Y_name = os.path.join(folder, 'Y_temp.npy') np.save(Y_name, Y) Y, _, _, _ = load_memmap(Y_name) # create arguments to be passed to the function. Here we are grouping # bunch of pixels to be processed by each thread # pixel_groups = [(Y_name, C_name, sn, ind2_, range(i, i + n_pixels_per_process)) # for i in range(0, np.prod(dims) - n_pixels_per_process + 1, # n_pixels_per_process)] cct = np.diag(C.dot(C.T)) rank_f = nb pixel_groups = [] for i in range(0, np.prod(dims) - n_pixels_per_process + 1, n_pixels_per_process): pixel_groups.append([Y_name, C_name, sn, ind2_, list( range(i, i + n_pixels_per_process)), method_ls, cct, rank_f]) if i < np.prod(dims): pixel_groups.append([Y_name, C_name, sn, ind2_, list( range(i, np.prod(dims))), method_ls, cct, rank_f]) A_ = np.zeros((d, nr + np.size(f, 0))) print('Starting Update Spatial Components') #serial_result = map(lars_regression_noise_ipyparallel, pixel_groups) if dview is not None: parallel_result = dview.map_sync(regression_ipyparallel, pixel_groups) dview.results.clear() for chunk in parallel_result: for pars in chunk: px, idxs_, a = pars A_[px, idxs_] = a else: parallel_result = list(map(regression_ipyparallel, pixel_groups)) for chunk in parallel_result: for pars in chunk: px, idxs_, a = pars A_[px, idxs_] = a ## # Cf_ = [Cf[idx_, :] for idx_ in ind2_] # # #% LARS regression # A_ = np.hstack((np.zeros((d, nr)), np.zeros((d, np.size(f, 0))))) # # for c, y, s, id2_, px in zip(Cf_, Y, sn, ind2_, range(d)): # if px % 1000 == 0: # print px # if np.size(c) > 0: # _, _, a, _, _ = lars_regression_noise_old(y, np.array(c.T), 1, sn[px]**2 * T) # if np.isscalar(a): # A_[px, id2_] = a # else: # A_[px, id2_] = a.T ## #% print('Updated Spatial Components') A_ = threshold_components(A_, dims, dview=dview, medw=medw, thr_method=thr_method, maxthr=maxthr, nrgthr=nrgthr, extract_cc=extract_cc, se=se, ss=ss) print("threshold") ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components if np.size(ff) > 0: ff = ff[0] print('eliminating {} empty components!!'.format(len(ff))) A_ = np.delete(A_, list(ff), 1) C = np.delete(C, list(ff), 0) background_ff = list(filter(lambda i: i > 0, ff-nr)) f = np.delete(f, background_ff, 0) nr = nr - (len(ff) - len(background_ff)) nb = nb - len(background_ff) A_ = A_[:, :nr] A_ = coo_matrix(A_) #import pdb # pdb.set_trace() # Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) print("Computing residuals") if 'memmap' in str(type(Y)): Y_resf = parallel_dot_product(Y, f.T, block_size=1000, dview=dview) - \ A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) else: Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) print("Computing A_bas") A_bas = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T))), 0) # update baseline based on residual # A_bas = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0) # update baseline based on residual # baseline based on residual b = A_bas print(("--- %s seconds ---" % (time.time() - start_time))) try: # clean up # remove temporary file created print("Remove temporary file created") shutil.rmtree(folder) except: raise Exception("Failed to delete: " + folder) # if A_in.dtype == bool: # return A_, b, C, f # else: # return A_, b, C return A_, b, C, f
def determine_search_location(A, dims, method='ellipse', min_size=3, max_size=8, dist=3, expandCore=iterate_structure(generate_binary_structure(2, 1), 2).astype(int), dview=None): """ restrict search location to subset of pixels TODO """ from scipy.ndimage.morphology import grey_dilation from scipy.sparse import coo_matrix, issparse if len(dims) == 2: d1, d2 = dims elif len(dims) == 3: d1, d2, d3 = dims d, nr = np.shape(A) A = csc_matrix(A) IND = False * np.ones((d, nr)) if method == 'ellipse': Coor = dict() if len(dims) == 2: Coor['x'] = np.kron(np.ones(d2), list(range(d1))) Coor['y'] = np.kron(list(range(d2)), np.ones(d1)) elif len(dims) == 3: Coor['x'] = np.kron(np.ones(d3 * d2), list(range(d1))) Coor['y'] = np.kron(np.kron(np.ones(d3), list(range(d2))), np.ones(d1)) Coor['z'] = np.kron(list(range(d3)), np.ones(d2 * d1)) if not dist == np.inf: # determine search area for each neuron cm = np.zeros((nr, len(dims))) # vector for center of mass Vr = [] # cell(nr,1); IND = [] # indicator for distance for i, c in enumerate(['x', 'y', 'z'][:len(dims)]): cm[:, i] = old_div(np.dot(Coor[c], A[:, :nr].todense()), A[:, :nr].sum(axis=0)) # for i in range(nr): # calculation of variance for each component and construction of ellipses # dist_cm = coo_matrix(np.hstack([Coor[c].reshape(-1, 1) - cm[i, k] # for k, c in enumerate(['x', 'y', 'z'][:len(dims)])])) # Vr.append(dist_cm.T * spdiags(A[:, i].toarray().squeeze(), # 0, d, d) * dist_cm / A[:, i].sum(axis=0)) # # if np.sum(np.isnan(Vr)) > 0: # raise Exception('You cannot pass empty (all zeros) components!') # # D, V = eig(Vr[-1]) # # dkk = [np.min((max_size**2, np.max((min_size**2, dd.real)))) for dd in D] # # # search indexes for each component # IND.append(np.sqrt(np.sum([(dist_cm * V[:, k])**2 / dkk[k] # for k in range(len(dkk))], 0)) <= dist) # IND = (np.asarray(IND)).squeeze().T pars = [] for i in range(nr): pars.append([Coor, cm[i], A[:, i], Vr, dims, dist, max_size, min_size, d]) if dview is None: res = list(map(contruct_ellipse_parallel, pars)) else: res = dview.map_sync(contruct_ellipse_parallel, pars) for r in res: IND.append(r) IND = (np.asarray(IND)).squeeze().T else: IND = True * np.ones((d, nr)) elif method == 'dilate': for i in range(nr): A_temp = np.reshape(A[:, i].toarray(), dims[::-1]) # , order='F') # A_temp = np.reshape(A[:, i].toarray(), (d2, d1)) if len(expandCore) > 0: if len(expandCore.shape) < len(dims): # default for 3D expandCore = iterate_structure( generate_binary_structure(len(dims), 1), 2).astype(int) A_temp = grey_dilation(A_temp, footprint=expandCore) else: A_temp = grey_dilation(A_temp, [1] * len(dims)) IND[:, i] = np.squeeze(np.reshape(A_temp, (d, 1))) > 0 else: IND = True * np.ones((d, nr)) return IND