def multi_median(input, median_radius, numpass): """ Applies median filter multiple times on input data. Parameters ---------- input : ndarray The input volume to apply filter on. median_radius : int Radius (in voxels) of the applied median filter numpass: int Number of pass of the median filter Returns ------- input : ndarray Filtered input volume. """ outvol = np.zeros_like(input) # Array representing the size of the median window in each dimension. medarr = np.ones_like(input.shape) * ((median_radius * 2) + 1) # Multi pass for i in range(0, numpass): median_filter(input, medarr, output=input) return input
def create_background_map(data, bsx, bsy): '''Create a background map with a given mesh size''' sx, sy = data.shape mx = sx / bsx my = sy / bsy comp = [] rms = [] # Rows sp = numpy.split(data, numpy.arange(bsx, sx, bsx), axis=0) for s in sp: # Columns rp = numpy.split(s, numpy.arange(bsy, sy, bsy), axis=1) for r in rp: b, r = background_estimator(r) comp.append(b) rms.append(r) # Reconstructed image z = numpy.array(comp) z.shape = (mx, my) # median filter ndfilter.median_filter(z, size=(3, 3), output=z) # Interpolate to the original size new = _interpolation(z, sx, sy, mx, my) # Interpolate the rms z = numpy.array(rms) z.shape = (mx, my) nrms = _interpolation(z, sx, sy, mx, my) return new, nrms
def compute_snr(power_2d, fractional_window_size=0.05): """ Extract the central region of the 2D Fourier transform Parameters ---------- power_2d : numpy array The 2D Fourier transform of the data fractional_window_size : float Median filter window size as a fraction of the 1D power array Returns ------- snr : numpy array The 1D SNR """ power = np.median(power_2d, axis=0) p2p_scatter = abs(power[1:] - power[:-1]) power = power[1:] # Throw away DC term # Median filter window_size = get_odd_integer(fractional_window_size * len(power)) continuum = median_filter(power, size=window_size) pixel_to_pixel_scatter = median_filter(p2p_scatter, size=window_size) snr = (power - continuum) / pixel_to_pixel_scatter # Also divide out the global scatter for any residual structure that was not removed with the median filter global_scatter = robust_standard_deviation(snr) snr /= global_scatter return snr
def preprocess_image(image): # Copy the depth part of the image depth_pixels = image.pixels[..., 2].copy() depth_pixels = rescale_to_opencv_image(depth_pixels) filtered_depth_pixels = median_filter(depth_pixels, 5) # Build mask for floodfilling, this lets me ignore all the pixels # from the background and around the ears mask = np.zeros((depth_pixels.shape[0] + 2, depth_pixels.shape[1] + 2), dtype=np.uint8) # Flood fill from top left cv2.floodFill(filtered_depth_pixels, mask, (0, 0), (255, 255, 255), flags=cv2.FLOODFILL_MASK_ONLY) # Flood fill from top right cv2.floodFill(filtered_depth_pixels, mask, (depth_pixels.shape[1] - 1, 0), (255, 255, 255), flags=cv2.FLOODFILL_MASK_ONLY) # Truncate and negate the flood filled areas to find the facial region floodfill_mask = (~mask.astype(np.bool))[1:-1, 1:-1] # Build a mask of the areas inside the face that need inpainting inpaint_mask = ~image.mask.mask & floodfill_mask # Inpaint the image and filter to smooth inpainted_pixels = cv2.inpaint(depth_pixels, inpaint_mask.astype(np.uint8), 5, cv2.INPAINT_NS) inpainted_pixels = median_filter(inpainted_pixels, 5) # Back to depth pixels image.pixels[..., 2] = rescale_to_depth_image(image, inpainted_pixels) # Reset the mask! image.mask.pixels[..., 0] = ~np.isnan(image.pixels[..., 2])
def lacosmic(img, cat, oper, sci, imgrn): '''Maximum pixel on LA Cosmic "fine structure" image.''' lhs = median_filter(img['diff'], size=3) rhs = median_filter(median_filter(img['diff'], size=3), size=7) fsimg = lhs - rhs fsmax = fsimg.max() diffmax = img['diff'].max() return diffmax / fsmax
def _apply_median_filter(nr, footprint, three_d): if three_d: pancake = np.swapaxes(np.tile(footprint, (3, 1, 1)), 0, -1) nr = 1.0*median_filter(nr, footprint=pancake) else: nt = nr.shape[2] for i in range(0, nt): nr[:, :, i] = 1.0*median_filter(nr[:, :, i], footprint=footprint) return nr
def despike_full(data, window_length, rejection_threshold=7, preprocess_function=lpf256): preproc = np.abs(lpf256(data)) medians = filters.median_filter(preproc, window_length) abs_deviations = np.abs(preproc - medians) mads = filters.median_filter(abs_deviations, window_length) spike_flags = abs_deviations > (mads*rejection_threshold) nspikes = spike_flags.sum() data[spike_flags] = np.array(random.sample(data[~spike_flags],nspikes)) return data
def pearson(img, margin=3, dur=(0,25), SpaMed=False): ''' img : numpy array margin : shift range setting dur : to define F in frames SpaMed : can be a list of two boolean values or just a bool ''' height, width, nframes = img.shape offsetxy = np.zeros((nframes,4), dtype=np.float) c = np.zeros((margin*2+1, margin*2+1), dtype=np.float) # clip out subimage at the center of img. x1, y1 = margin, margin x2, y2 = width-margin, height-margin if type(dur)==int: ref = img[y1:y2, x1:x2, dur] ref[np.isnan(ref)] = 0 else: ref = np.mean(img[y1:y2, x1:x2, dur[0]:dur[1]], 2) if type(SpaMed) is bool: Filt_ref, Filt_sub = SpaMed, False elif type(SpaMed) is list: Filt_ref, Filt_sub = SpaMed if Filt_ref: ref = median_filter(ref, (3,3), mode='nearest') ref = ref.ravel() for z in range(nframes): if Filt_sub: target = median_filter(img[:,:,z], (3,3), mode='nearest') else: target = img[:,:,z] for xoff in range(-margin, margin+1): for yoff in range(-margin, margin+1): sub = target[y1+yoff:y2+yoff, x1+xoff:x2+xoff].ravel() # fastest #sub = img[y1+yoff:y2+yoff, x1+xoff:x2+xoff,z].flat #sub = img[y1+yoff:y2+yoff, x1+xoff:x2+xoff,z].reshape(-1) #sub = img[y1+yoff:y2+yoff, x1+xoff:x2+xoff,z].flatten() c[margin+yoff, margin+xoff] = pearsonr(ref,sub)[0] offset = np.nonzero(c.max() == c) #print offset, c.max(), c offsetxy[z,:] = [offset[0][0]-margin,\ offset[1][0]-margin,\ c.max(), z] return offsetxy
def preprocess_eyegaze(eyegaze, blink_margin=200, filter_width=40): from scipy.ndimage.morphology import binary_dilation from scipy.ndimage.filters import median_filter mask = binary_dilation(np.isnan(eyegaze['x']), iterations=blink_margin) # filter x and y coordinate separately eyegaze['x'] = median_filter(eyegaze['x'], size=filter_width) eyegaze['y'] = median_filter(eyegaze['y'], size=filter_width) # blank blink margin eyegaze['x'][mask] = np.nan eyegaze['y'][mask] = np.nan return eyegaze
def dtm_kraus_median(data,tr,lo,hi,ml=(4,4),mh=(40,40)): med4 = filters.median_filter(data,ml) med40 = filters.median_filter(data,mh) diff = med4 - med40 med4 = None med40 = None mask = diff > tr mask = ndimage.morphology.binary_erosion(mask,iterations=1) out = np.copy(data) out[mask] = out[mask] - diff[mask] out = denoise(out,lo,hi,(3,3)) return out
def dtm_my_median(data,size,tr,lo,hi): med4 = filters.median_filter(data,(4,4)) med40 = filters.median_filter(data,(size,size)) diff = med4 - med40 med4 = None med40 = None mask = diff > tr mask = ndimage.morphology.binary_erosion(mask,iterations=1) out = np.copy(data) out[mask] = out[mask] - diff[mask] out = denoise(out,lo,hi,(3,3)) return out
def run(self, rinput): from scipy.signal import savgol_filter self.logger.info('starting slit flat reduction') flow = self.init_filters(rinput, rinput.obresult.configuration) reduced = basic_processing_with_combination(rinput, flow, method=combine.median) hdr = reduced[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced, 'reduced_image.fits') # Using median filtering... In each channel l0 = reduced[0].data.shape[0] l1 = l0 // 2 channel1 = (slice(None, l1), slice(None, None, None)) channel2 = (slice(l1, None, None), slice(None, None, None)) m_window1 = (11, 11) self.logger.debug('median filtering by channel %s', m_window1) median1 = numpy.zeros_like(reduced[0].data) median1[channel1] = median_filter(reduced[0].data[channel1], m_window1) median1[channel2] = median_filter(reduced[0].data[channel2], m_window1) self.save_intermediate_array(median1, 'median_image1.fits') qe1 = reduced[0].data / median1 self.save_intermediate_array(qe1, 'qe_filter1.fits') m_window2 = (3, 21) self.logger.debug('median filtering %s', m_window2) median2 = median_filter(qe1, m_window2) self.save_intermediate_array(median2 , 'median_image2.fits') qe2 = qe1 / median2 self.save_intermediate_array(qe2, 'qe_filter2.fits') self.logger.debug('filtering Inf/NaN in result') qe2[numpy.isinf(qe2)] = 1.0 qe2[numpy.isnan(qe2)] = 1.0 hdu = fits.PrimaryHDU(qe2.astype('float32'), header=reduced[0].header) hdu.header['UUID'] = str(uuid.uuid1()) master_slitflat = fits.HDUList([hdu]) self.set_base_headers(master_slitflat[0].header) self.logger.info('end slit flat recipe') return self.create_result(master_slitflat=master_slitflat, reduced_image=reduced)
def __init__(self, bare_dir, dem_dir, outdir): #loads the GDAL derived DEMs to produce the crop height model if not os.path.exists(outdir): os.mkdir(outdir) os.path.join(dem_dir,bare_dir) bare = None crop_model = None for bare_dem in os.listdir(bare_dir): os.chdir(bare_dir) bare = LoadImage(bare_dem) bare_filtered = filters.median_filter(bare.stacked,size=(9,9)) bare_spatial = bare.spatial for survey in os.listdir(dem_dir): im_path = os.path.join(dem_dir, survey) os.chdir(im_path) for image in os.listdir(im_path): crop = LoadImage(image) crop_filtered = filters.median_filter(crop.stacked,size=(3,3)) crop_model = crop_filtered-bare_filtered bare_mask = ma.masked_where(bare_filtered==-999, bare_filtered) crop_mask = ma.masked_where(crop_filtered==-999, crop_filtered) #combine these into a single mask so that anything masked in either dataset #will be masked in the combined output combined_mask = ma.mask_or(bare_mask.mask, crop_mask.mask) #use this mask t omask the crop model crop_model = ma.array(crop_model, mask=combined_mask) #convert back to a bog-standard numpy array and fill the masked values crop_model = crop_model.filled(-999) name = survey+'_CropHeightModel.tif' self.writeimage(outdir, name, crop_model.shape[0], crop_model.shape[1], crop_model, bare_spatial)
def median_filter(tomo, size=3, axis=0, ind=None): """ Apply median filter to a 3D array along a specified axis. Parameters ---------- tomo : ndarray Arbitrary 3D array. size : int, optional The size of the filter. axis : int, optional Axis along which median filtering is performed. ind : array of int, optional Indices at which the filtering is applied. Returns ------- ndarray Median filtered 3D array. """ if type(tomo) == str and tomo == 'SHARED': tomo = mp.shared_data else: arr = mp.distribute_jobs( tomo, func=median_filter, axis=axis, args=(size, axis)) return arr dx, dy, dz = tomo.shape if ind is None: if axis == 0: ind = np.arange(0, dx) elif axis == 1: ind = np.arange(0, dy) elif axis == 2: ind = np.arange(0, dz) if axis == 0: for m in ind: tomo[m, :, :] = filters.median_filter( tomo[m, :, :], (size, size)) elif axis == 1: for m in ind: tomo[:, m, :] = filters.median_filter( tomo[:, m, :], (size, size)) elif axis == 2: for m in ind: tomo[:, :, m] = filters.median_filter( tomo[:, :, m], (size, size))
def detrend_using_mf(lc, flavor='SAP', size=14, mode='constant'): # either use a gap (size is int): # # size # <.......--------------.........................................> # _/▔﹀\_︿╱﹀╲/╲︿_/︺╲▁︹_/﹀\_︿╱▔︺\/\︹▁╱﹀▔╲︿_/︺▔╲▁︹_/﹀▔\⁄﹀\╱ # # # # or a gap with with a hole (size is 2-element array): # # size[0] size[1] size[0] # <.......--------_______--------................................> # _/▔﹀\_︿╱﹀╲/╲︿_/︺╲▁︹_/﹀\_︿╱▔︺\/\︹▁╱﹀▔╲︿_/︺▔╲▁︹_/﹀▔\⁄﹀\╱ pidx = {'SAP': 1, 'PDC': 3}[flavor] lc_chunks = slice_da_bitch_up(lc, pidx) dtlc = numpy.array(lc) # [:,pidx]) for i in range(0, len(lc_chunks), 2): try: len(size) sidesize = size[0] gapsize = size[1] footprint = [True]*sidesize + [False]*gapsize + [True]*sidesize dt = median_filter(# lc[:,pidx][lc_chunks[i]:lc_chunks[i+1]], # footprint=footprint, lc[lc_chunks[i]:lc_chunks[i+1]], footprint=footprint, mode=mode, cval=1.0) except TypeError: dt = median_filter(# lc[:,pidx][lc_chunks[i]:lc_chunks[i+1]],size=size, lc[lc_chunks[i]:lc_chunks[i+1]], size=size, mode=mode, cval=1.0) # dtlc[lc_chunks[i]:lc_chunks[i+1]] = lc[:,pidx][lc_chunks[i]:lc_chunks[i+1]] / dt dtlc[lc_chunks[i]:lc_chunks[i+1]] = lc[lc_chunks[i]:lc_chunks[i+1]] / dt return dtlc
def ehist_equalize_melhist(d, sr, refMelHist, edges): """ Modify a signal in the Mel domain by equalizing the Mel-subband histograms to match the passed-in ones """ # Calculate the (Mel) spectrograms, and histogram, and axes melHist, edges, D, DmeldB, melmx, freqs = mel_hist(d, sr, edges=edges) # Build mapping & modify mel spectrogram histmaps = make_hist_maps(melHist, refMelHist, edges) # for some reason, extrapolating madly below bottom edge - clip it DmeldBmapped = np.maximum(edges[0], np.minimum(edges[-1], apply_hist_maps(DmeldB, histmaps))) # Reconstruct audio based on mapped envelope # We map both original and modified Mel envelopes to FFT domain # then scale original STFT magnitudes by their ratio DmelInFFT = np.dot(melmx.T, idB(DmeldB)) DmappedInFFT = np.dot(melmx.T, idB(DmeldBmapped)) # Zero values in denominator will match to zeros in numerator, # so it's OK to drop them Dmask = DmappedInFFT / (DmelInFFT + (DmelInFFT==0)) # Median filter to remove short blips in gain medfiltwin = 7 DmaskF = median_filter(Dmask, size=(1, medfiltwin)) # Now scale each FFT val by their ratio Dmod = D * DmaskF # and resynthesize nfft = 2*(np.size(D, axis=0)-1) win = nfft hop = win/4 dout = istft(Dmod.T, win, hop) return dout
def threshold_components(A, d1, d2, medw = (3,3), thr = 0.9999, se = np.ones((3,3),dtype=np.int), ss = np.ones((3,3),dtype=np.int)): from scipy.ndimage.filters import median_filter from scipy.ndimage.morphology import binary_closing from scipy.ndimage.measurements import label d, nr = np.shape(A) Ath = np.zeros((d,nr)) for i in range(nr): A_temp = np.reshape(A[:,i],(d2,d1)) A_temp = median_filter(A_temp,medw) Asor = np.sort(np.squeeze(np.reshape(A_temp,(d,1))))[::-1] temp = np.cumsum(Asor**2) ff = np.squeeze(np.where(temp<(1-thr)*temp[-1])) if ff.size > 0: ind = ff[-1] A_temp[A_temp<Asor[ind]] = 0 BW = (A_temp>=Asor[ind]) else: BW = (A_temp>=0) Ath[:,i] = np.squeeze(np.reshape(A_temp,(d,1))) BW = binary_closing(BW.astype(np.int),structure = se) labeled_array, num_features = label(BW, structure=ss) BW = np.reshape(BW,(d,1)) labeled_array = np.squeeze(np.reshape(labeled_array,(d,1))) nrg = np.zeros((num_features,1)) for j in range(num_features): nrg[j] = np.sum(Ath[labeled_array==j+1,i]**2) indm = np.argmax(nrg) Ath[labeled_array==indm+1,i] = A[labeled_array==indm+1,i] return Ath
def applyMorphologicalCleaning(self, image): """ Applies a variety of morphological operations to improve the detection of worms in the image. Takes 0.030 s on MUSSORGSKY for a typical frame region Takes 0.030 s in MATLAB too """ # start with worm == 1 image = image.copy() segmentation.clear_border(image) # remove objects at edge (worm == 1) # fix defects in the thresholding by closing with a worm-width disk # worm == 1 wormSE = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (self.wormDiskRadius+1, self.wormDiskRadius+1)) imcl = cv2.morphologyEx(np.uint8(image), cv2.MORPH_CLOSE, wormSE) imcl = np.equal(imcl, 1) # fix defects by filling holes imholes = ndimage.binary_fill_holes(imcl) imcl = np.logical_or(imholes, imcl) # fix barely touching regions # majority with worm pixels == 1 (median filter same?) imcl = nf.median_filter(imcl, footprint=[[1, 1, 1], [1, 0, 1], [1, 1, 1]]) # diag with worm pixels == 0 imcl = np.logical_not(bwdiagfill(np.logical_not(imcl))) # open with worm pixels == 1 openSE = cv2.getStructuringElement(cv2.MORPH_RECT, (1, 1)) imcl = cv2.morphologyEx(np.uint8(imcl), cv2.MORPH_OPEN, openSE) return np.equal(imcl, 1)
def full_frame(dat): bias = np.median(dat[:,2045:], axis=1) bias = bias.astype(np.float) smooth = FI.median_filter(bias, size=50) return np.tile(smooth, (2048,1))
def filter_outliers(self, pattern, filter_size, std_cutoff): pattern[nonzero(pattern > 1e9)] = 0 pattern_filter = median_filter(pattern, size=filter_size) pattern_diff = pattern - pattern_filter pattern_index = nonzero( logical_or( -std(pattern_diff) * std_cutoff + pattern_diff.mean() > pattern_diff, pattern_diff > std(pattern_diff) * std_cutoff + pattern_diff.mean(), ) ) print pattern_index if pattern[pattern_index].any(): pattern_final = np.copy(pattern) print pattern_final[pattern_index] num_filter = len(pattern_final[pattern_index]) print num_filter pattern_final[pattern_index] = pattern_filter[pattern_index] pattern_diff_final = pattern - pattern_final print nonzero(pattern > pattern.max() * 0.8), pattern.max(), pattern_filter.max(), median( pattern_diff_final ), (pattern_diff_final).max() else: pattern_final = copy(pattern) num_filter = 0 return pattern_final, num_filter, pattern_diff
def check_alignment(image, r1, r2): """ Take a particular line though the image and check if the spheres were properly aligned in the z direction. It happens to be off by a pixel or two sometimes """ distance = dist_between_spheres(r1, r2, image.shape[0] / 2. + 10, image.shape[0] / 2.) gap_signal = [] denoised = median_filter(image.copy(), 3) for j in np.arange(0., image.shape[1]): # Take the region around the gap, which later on will be used # to define the intensity at the gap between the spheres. # The width of the gap is not exact if image.shape[1] / 2. + distance + 5 > j > image.shape[1] / 2. - distance - 5: gap_signal.append(denoised[image.shape[0] / 2. + 10, j]) centre = np.mean(np.argwhere(np.min(gap_signal) == gap_signal)) print centre print len(gap_signal) / 2. print if abs(centre - len(gap_signal) / 2.) <= 1.5: return True else: return False
def flatField(closeDist_img=None, inPlane_img=None, closeDist_bg=None, inPlane_bg=None, vignetting_model='different_objects', interpolation_method='kangWeiss', inPlane_scale_factor=None): # 1. Pixel sensitivity: if closeDist_img is not None: # TODO: find better name ff1 = flatFieldFromCalibration(closeDist_img, closeDist_bg) else: ff1 = 0 # 2. Vignetting from in-plane measurements: if inPlane_img is not None: bg = gaussian_filter(median_filter(ff1, 3), 9) ff1 -= bg ff2, mask = VIGNETTING_MODELS[vignetting_model](inPlane_img, inPlane_bg, inPlane_scale_factor) # import pylab as plt # plt.imshow(mask) # plt.show() ff2smooth = INTERPOLATION_METHODS[interpolation_method](ff2, mask) if isinstance(ff1, np.ndarray) and ff1.shape != ff2smooth.shape: ff2smooth = resize(ff2smooth, ff1.shape, mode='reflect') else: ff2 = 0 ff2smooth = 0 return ff1 + ff2smooth, ff2
def activate(self): image = np.array(self.display.widget.image) self._setupMenu() scale = mn = 0 # TRANSFORM TO UINT8: orig_dtype = image.dtype if orig_dtype != np.uint8: if self.pConvMethod.value() == 'clip': image = [np.uint8(np.clip(i, 0, 255)) for i in image] else: # scale med = median_filter(image[0], 3) mn = np.min(med) image -= mn # set min to 0 scale = np.max(med) / 255 image /= scale image = np.clip(image, 0, 255) image = image.astype(np.uint8) # if len(image)==1: # image = image[0] self.startThread( lambda image=image, scale=scale, mn=mn, orig_dtype=orig_dtype: self._process(image, scale, mn, orig_dtype), self._processDone)
def __init__(self, stack, psfwidth=1.68, **kwargs): super().__init__(stack) self.psfwidth = psfwidth # median filter to remove spikes self.peakfinder = PeakFinder(median_filter(self.stack.max(0), 3), self.psfwidth, **kwargs) self.peakfinder.find_blobs()
def medianFilter(self, size=2): """ Spatially smooth images using a median filter. Filtering will be applied to every image in the collection and can be applied to either images or volumes. For volumes, if an single scalar neighborhood is passed, it will be interpreted as the filter size in x and y, with no filtering in z. parameters ---------- size: int, optional, default=2 Size of the filter neighbourhood in pixels. A sequence is interpreted as the neighborhood size for each axis. For three-dimensional data, a single scalar is intrepreted as the neighborhood in x and y, with no filtering in z. """ from scipy.ndimage.filters import median_filter from numpy import isscalar dims = self.dims ndims = len(dims) if ndims == 3 and isscalar(size) == 1: # improved performance applying separately to each plane def filter_(im): im.setflags(write=True) for z in arange(0, dims[2]): im[:, :, z] = median_filter(im[:, :, z], size) return im else: filter_ = lambda im: median_filter(im, size) return self._constructor( self.rdd.mapValues(lambda v: filter_(v))).__finalize__(self)
def watershed_3d(sphere): """ Markers should be int8 Image should be uint8 """ sphere = median_filter(sphere, 3) thresh = threshold_otsu(sphere) sphere = (sphere >= thresh) * 1 sphere = sobel(sphere) size = (sphere.shape[0], sphere.shape[1], sphere.shape[2]) marker = np.zeros(size, dtype=np.int16) pl.imshow(sphere[:,:,50]) pl.show() # mark everything outside as background marker[5, :, :] = -1 marker[size[0] - 5, :, :] = -1 marker[:, :, 5] = -1 marker[:, :, size[2] - 5] = -1 marker[:, 5, :] = -1 marker[:, size[1] - 5, :] = -1 marker[:,0,0] = -1 # mark everything inside as a sphere marker[size[0] / 2., size[1] / 2., size[2] / 2.] = 5 result = measurements.watershed_ift(sphere.astype(dtype=np.uint16), marker) pl.imshow(result[:,:,50]) pl.show() return result
def get_init_centroid(image,mode,half_size = 100,median_window = 30): """ Initial guess for the centroid. The idea is pretty simple: get rid of deviating pixels with a median filter; hence, create a smooth version of the image with no outliers. Then, get as first guess of the centroid the maximum count value (i.e., the brightest star in the image). The idea is then to cut a sub-image around this star, and fit a gaussian to that. """ if mode == 'subimg': mf = median_filter(image,size=median_window) else: mf = image x0,y0 = np.where(mf == np.max(mf)) x0,y0 = x0[0],y0[0] x_init = np.max([0,int(x0)-half_size]) x_end = np.min([mf.shape[0],int(x0)+half_size]) y_init = np.max([0,int(y0)-half_size]) y_end = np.min([mf.shape[1],int(y0)+half_size]) if mode == 'subimg': # Get subimg: sub_img = image[x_init:x_end,\ y_init:y_end] return x0-x_init,y0-y_init,x_init,y_init,sub_img else: return x0,y0
def fitTrace(self,kwidth=10,porder=3,cwidth=30,pad=False): sh = self.sh xr1 = (0,sh[1]) xrs = [xr1] polys = [] for xr in xrs: xindex = np.arange(xr[0],xr[1]) kernel = np.median(self.image[int(sh[0]/2-kwidth):int(sh[0]/2+kwidth),xindex],0) centroids = [] totals = [] for i in np.arange(sh[0]): row = self.image[i,xindex] row_med = np.median(row) total = np.abs((row-row_med).sum()) cc = fp.ifft(fp.fft(kernel)*np.conj(fp.fft(row-row_med))) cc_sh = fp.fftshift(cc) centroid = helpers.calc_centroid(cc_sh,cwidth=cwidth).real - xindex.shape[0]/2. centroids.append(centroid) totals.append(total) centroids = np.array(centroids) yindex = np.arange(sh[0]) gsubs = np.where((np.isnan(centroids)==False)) centroids[gsubs] = median_filter(centroids[gsubs],size=20) coeffs = np.polyfit(yindex[gsubs],centroids[gsubs],porder) poly = np.poly1d(coeffs) polys.append(poly) return xrs,polys
def rescaleData(): # Rescale data so that they have roughly same size over channels. # Rescale to match the 90th percentile of each channel to a value of 100. resamp4D = nib.load(nifti4Dresamp) resampData = resamp4D.get_data().astype(np.float) val = 100.0 prctile = 90 for i in range(resampData.shape[3]): p = np.percentile(resampData[:,:,:,i], prctile) resampData[:,:,:,i] = resampData[:,:,:,i] * (val / p) # Clamp to a high percentile. maxVal = np.percentile(resampData, 99.9) resampData[resampData > maxVal] = maxVal # Make suitable for one-byte integers. resampData = np.round(resampData * 255.0 / maxVal, decimals=0).astype(np.uint8) # maxData = np.max(resampData, axis=3) tempIm = nib.Nifti1Image(maxData, resamp4D.get_affine()) nib.save(tempIm, nifti4DChanMax) # Median filtering medFiltSize=3 temp = median_filter(maxData, size=medFiltSize) # Save. tempIm = nib.Nifti1Image(temp, resamp4D.get_affine()) nib.save(tempIm, nifti4DChanMaxCleaned)
def watershed_slicing(image): """ Does the watershed algorithm slice by slice. Then use the labeled image to calculate the centres of mass for each slice. """ image = median_filter(image, 3) thresh = threshold_otsu(image) image = (image > thresh) * 1 N = len(image) slice_centroids = [] slice_radius = [] for i in range(N): slice = image[:, :, i] labels_slice = watershed_segmentation(slice) centroids, areas, bords, radius = centres_of_mass_2D(labels_slice) slice_centroids.append(centroids) slice_radius.append(radius) # if i > 49: # print centroids # pl.imshow(labels_slice) # pl.show() return slice_centroids, slice_radius
def compute_dff_windowed_median(traces, median_kernel_long=5401, median_kernel_short=101, noise_stds=None, n_small_baseline_frames=None, **kwargs): """Compute dF/F of a set of traces with median filter detrending. The operation is basically: T_long = windowed_median(T) # long timescale kernel T_dff1 = (T - T_long) / elementwise_max(T_long, noise_std(T)) T_short = windowed_median(T_dff1) # short timescale kernel T_dff = T_dff1 - elementwise_min(T_short, 2.5*noise_std(T_dff1)) Parameters ---------- traces : np.ndarray 2D array of traces to be analyzed. median_kernel_long : int Window size to use for long timescale median detrending. median_kernel_short : int Window size to use for short timescale median detrending. noise_stds : list List that will contain noise_std(T_dff1) for each trace. The value for each trace will be appended to the list if provided. n_small_baseline_frames : list List that will contain the number of frames for each trace where the long-timescale median window is less than noise_std(T). The value for each trace will be appended to the list if provided. kwargs: Additional keyword arguments are passed to :func:`noise_std` . Returns ------- dff : np.ndarray 2D array of dF/F traces. """ _check_kernel(median_kernel_long, traces.shape[1]) _check_kernel(median_kernel_short, traces.shape[1]) dff_traces = np.copy(traces) for dff in dff_traces: sigma_f = noise_std(dff, **kwargs) # long timescale median filter for baseline subtraction tf = median_filter(dff, median_kernel_long, mode='constant') dff -= tf dff /= np.maximum(tf, sigma_f) if n_small_baseline_frames is not None: n_small_baseline_frames.append(np.sum(tf <= sigma_f)) sigma_dff = noise_std(dff, **kwargs) if noise_stds is not None: noise_stds.append(sigma_dff) # short timescale detrending tf = median_filter(dff, median_kernel_short, mode='constant') tf = np.minimum(tf, 2.5 * sigma_dff) dff -= tf return dff_traces
def threshold_components_parallel(A_i, neuron, dims, medw, thr_method, maxthr, nrgthr, extract_cc, **kwargs): """ Post-processing of spatial components which includes the following steps (i) Median filtering (ii) Thresholding (iii) Morphological closing of spatial support (iv) Extraction of largest connected component ( to remove small unconnected pixel ) """ if 'se' in kwargs.keys(): se = kwargs['se'] else: se = np.ones((3, ) * len(dims), dtype='uint8') ss = np.ones((3, ) * len(dims), dtype='uint8') # we reshape this one dimension column of the 2d components into the 2D that A_temp = np.reshape(A_i, dims) # we apply a median filter of size medw A_temp = median_filter(A_temp, medw) if thr_method == 'max': BW = (A_temp > maxthr * np.max(A_temp)) elif thr_method == 'nrg': Asor = np.sort(A_temp.flatten())[::-1] temp = np.cumsum(Asor**2) ff = np.squeeze(np.where(temp < nrgthr * temp[-1])) if ff.size > 0: ind = ff if ff.ndim == 0 else ff[-1] A_temp[A_temp < Asor[ind]] = 0 BW = (A_temp >= Asor[ind]) else: BW = np.zeros_like(A_temp) # we want to remove the components that are valued 0 in this now 1d matrix Ath = A_temp.flatten() Ath2 = np.zeros_like(Ath) # we do that to have a full closed structure even if the values have been trehsolded BW = binary_closing(BW.astype(np.int), structure=se) # if we have deleted the element if BW.max() == 0: return Ath2, neuron # # we want to extract the largest connected component ( to remove small unconnected pixel ) if extract_cc: # we extract each future as independent with the cross structuring elemnt labeled_array, num_features = label(BW, structure=ss) labeled_array = labeled_array.flatten() nrg = np.zeros((num_features, 1)) # we extract the energy for each component for j in range(num_features): nrg[j] = np.sum(Ath[labeled_array == j + 1]**2) indm = np.argmax(nrg) Ath2[labeled_array == indm + 1] = Ath[labeled_array == indm + 1] else: BW = BW.flatten() Ath2[BW] = Ath[BW] return Ath2, neuron
def ffttc_traction_finite_thickness(u, v, pixelsize1, pixelsize2, h, young, sigma=0.49, filter="gaussian", fs=None): ''' FTTC with correction for finite substrate thikness according to Xavier Trepat, Physical forces during collective cell migration, 2009 :param u:deformation field in x direction in pixel of the deformation image :param v:deformation field in y direction in pixel of the deformation image :param young: youngs modulus in Pa :param pixelsize1: pixelsize of the original image, needed because u and v is given as displacment of these pixels :param pixelsize2: pixelsize of the deformation image :param h hight of the membrane the cells lie on, in µm :param sigma: poission ratio of the gel :param bf_image: give the brightfield image as an array before cells where removed :param filter: str, values: "mean","gaussian","median". Diffrent smoothing methods for the traction field. :param fs: float, size of the filter (std of gaussian or size of the filter window) in µm if fs :return: tx_filter,ty_filter: traction forces in x and y direction in Pa ''' # 0) substracting mean(better name for this step) u_shift = (u - np.mean(u)) * pixelsize1 v_shift = (v - np.mean(v)) * pixelsize1 ## bens algortithm: # 1)Zero padding to get sqauerd array with even index number ax1_length = np.shape(u_shift)[0] # u and v must have same dimensions ax2_length = np.shape(u_shift)[1] max_ind = int(np.max((ax1_length, ax2_length))) if max_ind % 2 != 0: max_ind += 1 u_expand = np.zeros((max_ind, max_ind)) v_expand = np.zeros((max_ind, max_ind)) u_expand[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] = u_shift v_expand[ max_ind - ax1_length:max_ind, max_ind - ax2_length: max_ind] = v_shift # note: seems to be snummerically slightly diffrent then in normal fnction ??? (dont know why # 2) producing wave vectors (FT-space) " # form 1:max_ind/2 then -(max_ind/2:1) kx1 = np.array([ list(range(0, int(max_ind / 2), 1)), ] * int(max_ind), dtype=np.float64) kx2 = np.array([ list(range(-int(max_ind / 2), 0, 1)), ] * int(max_ind), dtype=np.float64) # spatial frequencies: 1/wavelength,in 1/µm in fractions of total length kx = np.append(kx1, kx2, axis=1) * 2 * np.pi / (pixelsize2 * max_ind) ky = np.transpose(kx) k = np.sqrt(kx**2 + ky**2) # matrix with "relative" distances??# # np.save("/home/user/Desktop/k_test.npy",k) r = k * h c = np.cosh(r) s = np.sinh(r) s_c = np.tanh(r) #gamma = ((3 - 4 * sigma) * (c ** 2) + (1 - 2 * sigma) ** 2 + (k * h) ** 2) / ( # (3 - 4 * sigma) * s * c + k * h) ## inf values here because k goes to zero gamma = ((3 - 4 * sigma) + (((1 - 2 * sigma)**2) / (c**2)) + ((r**2) / (c**2))) / ((3 - 4 * sigma) * s_c + r / (c**2)) # 4) calculate fourier transform of displacements u_ft = scipy.fft.fft2(u_expand) v_ft = scipy.fft.fft2(v_expand) ''' #4.0*) approximation for large h according to this paper factor3=young/(2*(1-sigma**2)*k) factor3[0,0]=factor3[0,1] tx_ft=factor3*(u_ft*((k**2)*(1-sigma)+sigma*(kx**2)) + v_ft*kx*ky*sigma) ty_ft=factor3*(v_ft*((k**2)*(1-sigma)+sigma*(ky**2)) + u_ft*kx*ky*sigma) ## confirmed by comparisson with not corrected programm ''' # 4.1) calculate tractions in fourrier space factor1 = (v_ft * kx - u_ft * ky) factor2 = (u_ft * kx + v_ft * ky) tx_ft = ((-young * ky * c) / (2 * k * s * (1 + sigma))) * factor1 + ((young * kx) / (2 * k * (1 - sigma**2))) * gamma * factor2 tx_ft[ 0, 0] = 0 ## zero frequency would represent force every where(constant), so this is legit?? ty_ft = ((young * kx * c) / (2 * k * s * (1 + sigma))) * factor1 + ((young * ky) / (2 * k * (1 - sigma**2))) * gamma * factor2 ty_ft[0, 0] = 0 # 4.2) go back to real space tx = scipy.fft.ifft2(tx_ft.astype( np.complex128)).real ## maybe devide by 2pi here??? ty = scipy.fft.ifft2(ty_ft.astype(np.complex128)).real # 5.2) cut like in script from ben tx_cut = tx[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] ty_cut = ty[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] # 5.3) using filter if filter == "mean": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = uniform_filter(tx_cut, size=fs) ty_filter = uniform_filter(ty_cut, size=fs) if filter == "gaussian": fs = fs if isinstance(fs, (float, int)) else int(np.max( (ax1_length, ax2_length))) / 50 tx_filter = gaussian_filter(tx_cut, sigma=fs) ty_filter = gaussian_filter(ty_cut, sigma=fs) if filter == "median": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = median_filter(tx_cut, size=fs) ty_filter = median_filter(ty_cut, size=fs) if not isinstance(filter, str): tx_filter = tx_cut ty_filter = ty_cut # show_quiver(tx_filter,ty_filter) return (tx_filter, ty_filter)
def median_filt(img, Q): # Q = window size return median_filter(img, Q)
def get_ext_coeffs_imglist(self, lst, roi_ambient=None, apply_median=5, **kwargs): """Apply dilution fit to all images in an :class:`ImgList`. Parameters ---------- lst : ImgList image list for which the coefficients are supposed to be retrieved roi_ambient : list region of interest used to estimage ambient intensity, if None (default), usd :attr:`scale_rect` of :class:`PlumeBackgroundModel` of the input list apply_median : int if > 0, then a median filter of provided width is applied to the result time series (ext. coeffs and initial intensities) **kwargs : additional keyword args passed to dilution fit method :func:`apply_dilution_fit`. Returns ------- DataFrame pandas data frame containing time series of retrieved extinction coefficients and initial intensities as well as the ambient intensities used, access keys are: - ``coeffs``: retrieved extinction coefficients - ``i0``: retrieved initial intensities - ``ia``: retrieved ambient intensities """ if not isinstance(lst, ImgList): raise ValueError("Invalid input type for param lst, need ImgList") lst.vigncorr_mode = True if not check_roi(roi_ambient): try: roi_ambient = lst.bg_model.scale_rect except BaseException: pass if not check_roi(roi_ambient): raise ValueError("Input parameter roi_ambient is not a valied" "ROI and neither is scale_rect in background " "model of input image list...") cfn = lst.cfn lst.goto_img(0) nof = lst.nof times = lst.acq_times coeffs = [] i0s = [] ias = [] for k in range(nof): img = lst.current_img() try: ia = img.crop(roi_ambient, True).mean() ext, i0, _, _ = self.apply_dilution_fit(img=img, rad_ambient=ia, plot=False, **kwargs) coeffs.append(ext) i0s.append(i0) ias.append(ia) except BaseException: coeffs.append(nan) i0s.append(nan) ias.append(nan) lst.goto_next() lst.goto_img(cfn) if apply_median > 0: coeffs = median_filter(coeffs, apply_median) i0s = median_filter(i0s, apply_median) ias = median_filter(ias, apply_median) return DataFrame(dict(coeffs=coeffs, i0=i0s, ia=ias), index=times)
def middle( f, param, x=None, iterations=40, eps=0.001, poly=False, weight=1, lambda2=-1, mn=None, mx=None, ): """ middle tries to fit a smooth curve that is located along the "middle" of 1D data array f. Filter size "filter" together with the total number of iterations determine the smoothness and the quality of the fit. The total number of iterations can be controlled by limiting the maximum number of iterations (iter) and/or by setting the convergence criterion for the fit (eps) 04-Nov-2000 N.Piskunov wrote. 09-Nov-2011 NP added weights and 2nd derivative constraint as LAM2 Parameters ---------- f : Callable Function to fit filter : int Smoothing parameter of the optimal filter (or polynomial degree of poly is True) iter : int maximum number of iterations [def: 40] eps : float convergence level [def: 0.001] mn : float minimum function values to be considered [def: min(f)] mx : float maximum function values to be considered [def: max(f)] lam2 : float constraint on 2nd derivative weight : array(float) vector of weights. """ mn = mn if mn is not None else np.min(f) mx = mx if mx is not None else np.max(f) f = np.asarray(f) if x is None: xx = np.linspace(-1, 1, num=f.size) else: xx = np.asarray(x) if poly: j = (f >= mn) & (f <= mx) n = np.count_nonzero(j) if n <= round(param): return f fmin = np.min(f[j]) - 1 fmax = np.max(f[j]) + 1 ff = (f[j] - fmin) / (fmax - fmin) ff_old = ff else: fmin = np.min(f) - 1 fmax = np.max(f) + 1 ff = (f - fmin) / (fmax - fmin) ff_old = ff n = len(f) for _ in range(iterations): if poly: param = round(param) if param > 0: t = median_filter(np.polyval(np.polyfit(xx, ff, param), xx), 3) tmp = np.polyval(np.polyfit(xx, (t - ff)**2, param), xx) else: t = np.tile(np.polyfit(xx, ff, param), len(f)) tmp = np.tile(np.polyfit(xx, (t - ff)**2, param), len(f)) else: t = median_filter( opt_filter(ff, param, weight=weight, lambda2=lambda2), 3) tmp = opt_filter(weight * (t - ff)**2, param, weight=weight, lambda2=lambda2) dev = np.sqrt(np.clip(tmp, 0, None)) ff = np.clip(t - dev, ff, t + dev) dev2 = np.max(weight * np.abs(ff - ff_old)) ff_old = ff # print(dev2) if dev2 <= eps: break if poly: xx = np.linspace(-1, 1, len(f)) if param > 0: t = median_filter(np.polyval(np.polyfit(xx, ff, param), xx), 3) else: t = np.tile(np.polyfit(xx, ff, param), len(f)) return t * (fmax - fmin) + fmin
def top( f, order=1, iterations=40, eps=0.001, poly=False, weight=1, lambda2=-1, mn=None, mx=None, ): """ top tries to fit a smooth curve to the upper envelope of 1D data array f. Filter size "filter" together with the total number of iterations determine the smoothness and the quality of the fit. The total number of iterations can be controlled by limiting the maximum number of iterations (iter) and/or by setting the convergence criterion for the fit (eps) 04-Nov-2000 N.Piskunov wrote. 09-Nov-2011 NP added weights and 2nd derivative constraint as LAM2 Parameters ---------- f : Callable Function to fit filter : int Smoothing parameter of the optimal filter (or polynomial degree of poly is True) iter : int maximum number of iterations [def: 40] eps : float convergence level [def: 0.001] mn : float minimum function values to be considered [def: min(f)] mx : float maximum function values to be considered [def: max(f)] lam2 : float constraint on 2nd derivative weight : array(float) vector of weights. """ mn = mn if mn is not None else np.min(f) mx = mx if mx is not None else np.max(f) f = np.asarray(f) xx = np.linspace(-1, 1, num=f.size) if poly: j = (f >= mn) & (f <= mx) if np.count_nonzero(j) <= round(order): raise ValueError("Not enough points") fmin = np.min(f[j]) - 1 fmax = np.max(f[j]) + 1 ff = (f - fmin) / (fmax - fmin) ff_old = ff else: fff = middle( f, order, iterations=iterations, eps=eps, weight=weight, lambda2=lambda2, ) fmin = np.min(f) - 1 fmax = np.max(f) + 1 fff = (fff - fmin) / (fmax - fmin) ff = (f - fmin) / (fmax - fmin) / fff ff_old = ff for _ in range(iterations): order = round(order) if poly: t = median_filter(np.polyval(np.polyfit(xx, ff, order), xx), 3) tmp = np.polyval( np.polyfit(xx, np.clip(ff - t, 0, None)**2, order), xx) dev = np.sqrt(np.clip(tmp, 0, None)) else: t = median_filter( opt_filter(ff, order, weight=weight, lambda2=lambda2), 3) tmp = opt_filter( np.clip(weight * (ff - t), 0, None), order, weight=weight, lambda2=lambda2, ) dev = np.sqrt(np.clip(tmp, 0, None)) ff = np.clip(t - eps, ff, t + dev * 3) dev2 = np.max(weight * np.abs(ff - ff_old)) ff_old = ff if dev2 <= eps: break if poly: t = median_filter(np.polyval(np.polyfit(xx, ff, order), xx), 3) return t * (fmax - fmin) + fmin else: return t * fff * (fmax - fmin) + fmin
def counting(c_std, a_std, imageFile): image_list = [imageFile] c_std = float(c_std) if image_list[0][-3:] == 'tif': for f, filename in enumerate(image_list): subprocess.call(["convert", filename, filename + '.png']) image_list[f] = filename + '.png' output_filename_hash = (datetime.datetime.now().minute * 60 + datetime.datetime.now().second) images = [ scipy.misc.imread(filename).astype(np.uint16) for filename in image_list ] #correlation_matrix = np.array([[-1, -1, -1, -1, -1], # [-1, -1, -1, -1, -1], # [-1, -1, 24, -1, -1], # [-1, -1, -1, -1, -1], # [-1, -1, -1, -1, -1]]) correlation_matrix = np.array([[-5935, -5935, -5935, -5935, -5935], [-5935, 8027, 8027, 8027, -5935], [-5935, 8027, 30742, 8027, -5935], [-5935, 8027, 8027, 8027, -5935], [-5935, -5935, -5935, -5935, -5935]]) median_diameter = 5 local_max = 3 #c_std = 1 processed_images = [np.copy(image).astype(np.int64) for image in images] processed_images = \ [np.subtract(image, np.minimum(spf.median_filter(image, median_diameter), image)) for image in images] processed_images = \ [np.maximum(scipy.signal.correlate(image, correlation_matrix, mode='same'), np.zeros_like(image)).astype(np.int64) for image in processed_images] thresholded_images = [ image > np.mean(image) + c_std * np.std(image) #images[i] > np.mean(images[i]) + a_std * np.std(images[i]) #images[i] > a_std for i, image in enumerate(processed_images) ] for i, mask in enumerate(thresholded_images): for (h, w), valid in np.ndenumerate(mask): if valid: local_slice = \ np.copy(processed_images[i][h - local_max:h + local_max + 1, w - local_max:w + local_max + 1]) if (h + local_max >= mask.shape[0] or w + local_max >= mask.shape[1] or h - local_max < 0 or w - local_max < 0): mask[h, w] = False continue local_slice[local_max, local_max] = 0 if np.amax(local_slice) >= processed_images[i][h, w]: mask[h, w] = False size = 4 for i, image in enumerate(images): i_std = np.std(image) i_median = np.median(image) - i_std i_max = np.amax(image) for (h, w), value in np.ndenumerate(image): if image[h, w] > i_median: image[h, w] = int( np.around( math.sqrt(image[h, w] - i_median) / math.sqrt(i_max - i_median) * float(2**8 - 1))) else: image[h, w] = 0 output_images = [ ImageOps.colorize( Image.fromstring('L', image.shape, image.astype(np.uint8)), (0, 0, 0), (255, 255, 255)) for image in images ] for i, image in enumerate(images): for (h, w), value in np.ndenumerate(image): if thresholded_images[i][h, w]: box = ((w - size, h - size), (w + size, h + size)) draw = ImageDraw.Draw(output_images[i]) draw.rectangle(box, fill=None, outline='blue') # Editing the output file name; Will have the same as the filename + png [ image.save( str(i).zfill(int(np.ceil(math.log(len(output_images), 10)))) + ' SIMPLE ' + str(output_filename_hash) + '.png') for i, image in enumerate(output_images) ] for i, image in enumerate(thresholded_images): print( str(i).zfill(int(np.ceil(math.log(len(thresholded_images), 10)))) + ' ' + str(np.sum(np.sum(image))))
K=nb_median) print "Getting the harmonic part" estimated_spectrum_harmo, neighbors = regression.ann( learn_feats[:, -48:-36].T, learn_magspecs.T, test_feats[:, -48:-36].T, test_magspecs.T, K=nb_median) win_size = params['wintime'] * params['sr'] step_size = params['steptime'] * params['sr'] # sliding median filtering ? if l_medfilt > 1: estimated_spectrum = median_filter( estimated_spectrum_full + estimated_spectrum_harmo, (1, l_medfilt)) print "reconstruction" #init_vec = np.random.randn(step_size*Y_hat.shape[1]) init_vec = np.random.randn(step_size * estimated_spectrum.shape[1]) x_recon = transforms.gl_recons(estimated_spectrum, init_vec, nb_iter_gl, win_size, step_size, display=False) # Get the rythmic part by using all coefficients
def correctHeadTailIntWorm(trajectories_worm, skeletons_file, intensities_file, smooth_W = 5, gap_size = 0, min_block_size = 10, local_avg_win = 25, min_frac_in = 0.85): #get data with valid intensity maps (worm int profile) good = trajectories_worm['int_map_id'] != -1; int_map_id = trajectories_worm.loc[good, 'int_map_id'].values int_skeleton_id = trajectories_worm.loc[good, 'skeleton_id'].values int_frame_number = trajectories_worm.loc[good, 'frame_number'].values #only analyze data that contains at least min_block_size intensity profiles if int_map_id.size < min_block_size: return [] #read the worm intensity profiles with tables.File(intensities_file, 'r') as fid: worm_int_profile = fid.get_node('/straighten_worm_intensity_median')[int_map_id,:] #normalize intensities of each individual profile worm_int_profile -= np.median(worm_int_profile, axis=1)[:, np.newaxis] #reduce the importance of the head and tail. This parts are typically more noisy damp_factor = getDampFactor(worm_int_profile.shape[1]) worm_int_profile *= damp_factor #worm median intensity med_int = np.median(worm_int_profile, axis=0).astype(np.float) #%% #let's check for head tail errors by comparing the #total absolute difference between profiles using the original orientation ... diff_ori = np.sum(np.abs(med_int-worm_int_profile), axis = 1) #... and inverting the orientation diff_inv = np.sum(np.abs(med_int[::-1]-worm_int_profile), axis = 1) #%% DEPRECATED, it #check if signal noise will allow us to distinguish between the two signals #I am assuming that most of the images will have a correct head tail orientation #and the robust estimates will give us a good representation of the noise levels #if np.median(diff_inv) - medabsdev(diff_inv)/2 < np.median(diff_ori) + medabsdev(diff_ori)/2: # bad_worms.append(worm_index) # continue #%% #smooth data, it is easier for identification diff_ori_med = median_filter(diff_ori,smooth_W) diff_inv_med = median_filter(diff_inv,smooth_W) #this will increase the distance between the original and the inversion. #Therefore it will become more stringent on detection diff_orim = minimum_filter(diff_ori_med, smooth_W) diff_invM = maximum_filter(diff_inv_med, smooth_W) #a segment with a bad head-tail indentification should have a lower #difference with the median when the profile is inverted. bad_orientationM = diff_orim>diff_invM if np.all(bad_orientationM): return [] #let's create blocks of skeletons with a bad orientation blocks2correct = createBlocks(bad_orientationM, min_block_size) #print(blocks2correct) #let's refine blocks limits using the original unsmoothed differences bad_orientation = diff_ori>diff_inv blocks2correct = correctBlock(blocks2correct, bad_orientation, gap_size=0) #let's correct the blocks inversion boundaries by checking that they do not #travers a group of contigous skeletons. I am assuming that head tail errors #only can occur when we miss an skeleton. blocks2correct = removeBadSkelBlocks(blocks2correct, int_skeleton_id, trajectories_worm, min_frac_in, gap_size=gap_size) #Check in the boundaries between blocks if there is really a better local match if the block is inverted blocks2correct = checkLocalVariation(worm_int_profile, blocks2correct, local_avg_win) if not blocks2correct: return [] #redefine the limits in the skeleton_file and intensity_file rows using the final blocks boundaries skel_group = [(int_skeleton_id[ini], int_skeleton_id[fin]) for ini, fin in blocks2correct] int_group = [(int_map_id[ini], int_map_id[fin]) for ini, fin in blocks2correct] #finally switch all the data to correct for the wrong orientation in each group switchBlocks(skel_group, skeletons_file, int_group, intensities_file) #store data from the groups that were switched switched_blocks = [] for ini, fin in blocks2correct: switched_blocks.append((int_frame_number[ini], int_frame_number[fin])) return switched_blocks
def convert_2D_to_1D(array2Dlist, average_vspan=None, search_window=None): ''' # This function converts a 2D GISAXS reading to a 1D vertical average centered on the specular peak. # # In general the function first applies hot-pixel removal to deal with slightly faulty detectors, and then # finds the brightest pixel in the image and calls it the specular peak. If you know the specular peak will # be the brightest pixel, then you can just let the function find it for you (and it does so on an # image-by-image basis, so it automatically handles slight jitter between readings). # # However, if the specular peak is **not** the brightest pixel in the image, you can specify a smaller # sub-window in which to look for it, by listing four numbers in a file called "specular-peak-window.txt" ''' # get dimensions of (t, qx, qz) array datadims = np.shape(array2Dlist) tn = datadims[0] xn = datadims[1] zn = datadims[2] # obtain a median-filtered version of array (eliminates broken pixels -- borrowed from StackExchange) mf_data = np.array(array2Dlist) for kk in xrange(tn): arr = array2Dlist[kk, :, :] blurred = filters.median_filter(arr, size=2) difference = arr - blurred threshold = 3 * np.std( difference ) # too large and you don't remove pixels -- too small and you smooth hot_pixels = np.nonzero( (np.abs(difference[1:-1, 1:-1]) > threshold)) # ignore edges for simplicity; we'll discard anyway hot_pixels = np.array( hot_pixels ) + 1 # because we ignored the first row and first column for x, y in zip(hot_pixels[0], hot_pixels[1]): mf_data[kk, x, y] = blurred[x, y] # build the search window if search_window == None: print "Looking over the whole image for the specular peak." cxmin = 0 cxmax = xn cymin = 0 cymax = zn else: print "Looking for specular peak in provided window ", center_window cxmin = search_window[0] cxmax = search_window[1] cymin = search_window[2] cymax = search_window[3] # find the specular peak totalI = sum(mf_data, axis=0) cwindow = totalI[cxmin:cxmax, cymin:cymax] temp_coords = np.unravel_index(np.argmax(cwindow), cwindow.shape) max_coords = np.array([cxmin, cymin]) + temp_coords x0 = max_coords[0] z0 = max_coords[1] # report, with optional warning print "Peak found at ", max_coords if x0 == cxmin or x0 == cxmax or z0 == cymin or z0 == cymax: print "WARNING: specular peak found on edge of search window." print "Cannot confirm that value is a local maximum." print "Identified co-ordinates may be inaccurate." # identify the vertical analysis window (specular peak +- half the width/height specified in window_shape) zmin = z0 - (average_vspan - 1) / 2 zmax = z0 + (average_vspan - 1) / 2 iofqt = sum(mf_data[:, :, zmin:zmax + 1], axis=2) return iofqt
def run(self, rinput): self.logger.info('starting processing for bars detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) try: rotang = hdr['ROTANG'] tsutc1 = hdr['TSUTC1'] dtub, dtur = datamodel.get_dtur_from_header(hdr) csupos = datamodel.get_csup_from_header(hdr) csusens = datamodel.get_cs_from_header(hdr) except KeyError as error: self.logger.error(error) raise numina.exceptions.RecipeError(error) self.logger.debug('finding bars') # Processed array arr = hdulist[0].data # Median filter of processed array (two times) mfilter_size = rinput.median_filter_size self.logger.debug('median filtering X, %d columns', mfilter_size) arr_median = median_filter(arr, size=(1, mfilter_size)) self.logger.debug('median filtering X, %d rows', mfilter_size) arr_median = median_filter(arr_median, size=(mfilter_size, 1)) # Median filter of processed array (two times) in the other direction # for Y coordinates self.logger.debug('median filtering Y, %d rows', mfilter_size) arr_median_alt = median_filter(arr, size=(mfilter_size, 1)) self.logger.debug('median filtering Y, %d columns', mfilter_size) arr_median_alt = median_filter(arr_median_alt, size=(1, mfilter_size)) xfac = dtur[0] / EMIR_PIXSCALE yfac = -dtur[1] / EMIR_PIXSCALE vec = [yfac, xfac] self.logger.debug('DTU shift is %s', vec) # and the table of approx positions of the slits barstab = rinput.bars_nominal_positions # Currently, we only use fields 0 and 2 # of the nominal positions file # Number or rows used # These other parameters cab be tuned also bstart = 1 bend = 2047 self.logger.debug('ignoring columns outside %d - %d', bstart, bend - 1) # extract a region to average wy = (rinput.average_box_row_size // 2) wx = (rinput.average_box_col_size // 2) self.logger.debug('extraction window is %d rows, %d cols', 2 * wy + 1, 2 * wx + 1) # Fit the peak with these points wfit = 2 * (rinput.fit_peak_npoints // 2) + 1 self.logger.debug('fit with %d points', wfit) # Minimum threshold threshold = 5 * EMIR_RON # Savitsky and Golay (1964) filter to compute the X derivative # scipy >= xx has a savgol_filter function # for compatibility we do it manually allpos = {} ypos3_kernel = None slits = numpy.zeros((EMIR_NBARS, 8), dtype='float') self.logger.info('start finding bars') for ks in [3, 5, 7, 9]: self.logger.debug('kernel size is %d', ks) # S and G kernel for derivative kw = ks * (ks * ks - 1) / 12.0 coeffs_are = -numpy.arange((1 - ks) // 2, (ks - 1) // 2 + 1) / kw if ks == 3: ypos3_kernel = coeffs_are self.logger.debug('kernel weights are %s', coeffs_are) self.logger.debug('derive image in X direction') arr_deriv = convolve1d(arr_median, coeffs_are, axis=-1) # Axis 0 is # self.logger.debug('derive image in Y direction (with kernel=3)') arr_deriv_alt = convolve1d(arr_median_alt, ypos3_kernel, axis=0) positions = [] for coords in barstab: lbarid = int(coords[0]) rbarid = lbarid + EMIR_NBARS ref_y_coor = coords[1] + vec[1] poly_coeffs = coords[2:] prow = coor_to_pix_1d(ref_y_coor) - 1 fits_row = prow + 1 # FITS pixel index # A function that returns the center of the bar # given its X position def center_of_bar(x): # Pixel values are 0-based return polyval(x + 1 - vec[0], poly_coeffs) + vec[1] - 1 self.logger.debug('looking for bars with ids %d - %d', lbarid, rbarid) self.logger.debug('reference y position is Y %7.2f', ref_y_coor) # if ref_y_coor is outlimits, skip this bar # ref_y_coor is in FITS format if (ref_y_coor >= 2047) or (ref_y_coor <= 1): self.logger.debug( 'reference y position is outlimits, skipping') positions.append([lbarid, fits_row, fits_row, 1, 0, 3]) positions.append([rbarid, fits_row, fits_row, 1, 0, 3]) continue # Left bar self.logger.debug('measure left border (%d)', lbarid) centery, xpos, fwhm, st = char_bar_peak_l(arr_deriv, prow, bstart, bend, threshold, center_of_bar, wx=wx, wy=wy, wfit=wfit) xpos1 = xpos positions.append( [lbarid, centery + 1, fits_row, xpos + 1, fwhm, st]) # Right bar self.logger.debug('measure rigth border (%d)', rbarid) centery, xpos, fwhm, st = char_bar_peak_r(arr_deriv, prow, bstart, bend, threshold, center_of_bar, wx=wx, wy=wy, wfit=wfit) positions.append( [rbarid, centery + 1, fits_row, xpos + 1, fwhm, st]) xpos2 = xpos # if st == 0: self.logger.debug('measure top-bottom borders') try: y1, y2, statusy = char_bar_height(arr_deriv_alt, xpos1, xpos2, centery, threshold, wh=35, wfit=wfit) except Exception as error: self.logger.warning('Error computing height: %s', error) statusy = 44 if statusy in [0, 40]: # Main border is detected positions[-1][1] = y2 + 1 positions[-2][1] = y2 + 1 else: # Update status positions[-1][-1] = 4 positions[-2][-1] = 4 else: self.logger.debug('slit is not complete') y1, y2 = 0, 0 # Update positions self.logger.debug( 'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d', *positions[-2]) self.logger.debug( 'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d', *positions[-1]) if ks == 5: slits[lbarid - 1] = [xpos1, y2, xpos2, y2, xpos2, y1, xpos1, y1] # FITS coordinates slits[lbarid - 1] += 1.0 self.logger.debug('inserting bars %d-%d into "slits"', lbarid, rbarid) allpos[ks] = numpy.asarray( positions, dtype='float') # GCS doesn't like lists of lists self.logger.debug('end finding bars') result = self.create_result( frame=hdulist, slits=slits, positions9=allpos[9], positions7=allpos[7], positions5=allpos[5], positions3=allpos[3], DTU=dtub, ROTANG=rotang, TSUTC1=tsutc1, csupos=csupos, csusens=csusens, ) return result
def visitcomb(allvisit, starver, load=None, apred='r13', telescope='apo25m', nres=[5, 4.25, 3.5], bconly=False, plot=False, write=True, dorvfit=True, apstar_vers='stars', logger=None): """ Combine multiple visits with individual RVs to rest frame sum """ if logger is None: logger = dln.basiclogger() if load is None: load = apload.ApLoad(apred=apred, telescope=telescope) cspeed = 2.99792458e5 # speed of light in km/s logger.info('Doing visitcomb for {:s} '.format(allvisit['apogee_id'][0])) wnew = norm.apStarWave() nwave = len(wnew) nvisit = len(allvisit) # initialize array for stack of interpolated spectra zeros = np.zeros([nvisit, nwave]) izeros = np.zeros([nvisit, nwave], dtype=int) stack = apload.ApSpec(zeros, err=zeros.copy(), bitmask=izeros, cont=zeros.copy(), sky=zeros.copy(), skyerr=zeros.copy(), telluric=zeros.copy(), telerr=zeros.copy()) apogee_target1, apogee_target2, apogee_target3 = 0, 0, 0 apogee2_target1, apogee2_target2, apogee2_target3, apogee2_target4 = 0, 0, 0, 0 starflag, andflag = np.uint64(0), np.uint64(0) starmask = bitmask.StarBitMask() # Loop over each visit and interpolate to final wavelength grid if plot: fig, ax = plots.multi(1, 2, hspace=0.001) for i, visit in enumerate(allvisit): if bconly: vrel = -visit['bc'] else: vrel = visit['vrel'] # Skip if we don't have an RV if np.isfinite(vrel) is False: continue # Load the visit if load.telescope == 'apo1m': apvisit = load.apVisit1m(visit['plate'], visit['mjd'], visit['apogee_id'], load=True) else: apvisit = load.apVisit(int(visit['plate']), visit['mjd'], visit['fiberid'], load=True) pixelmask = bitmask.PixelBitMask() # Rest-frame wavelengths transformed to this visit spectra w = norm.apStarWave() * (1.0 + vrel / cspeed) # Loop over the chips for chip in range(3): # Get the pixel values to interpolate to pix = wave.wave2pix(w, apvisit.wave[chip, :]) gd, = np.where(np.isfinite(pix)) # Get a smoothed, filtered spectrum to use as replacement for bad values cont = gaussian_filter( median_filter(apvisit.flux[chip, :], [501], mode='reflect'), 100) errcont = gaussian_filter( median_filter(apvisit.flux[chip, :], [501], mode='reflect'), 100) bd, = np.where(apvisit.bitmask[chip, :] & pixelmask.badval()) if len(bd) > 0: apvisit.flux[chip, bd] = cont[bd] apvisit.err[chip, bd] = errcont[bd] # Load up quantity/error pairs for interpolation raw = [[apvisit.flux[chip, :], apvisit.err[chip, :]**2], [apvisit.sky[chip, :], apvisit.skyerr[chip, :]**2], [apvisit.telluric[chip, :], apvisit.telerr[chip, :]**2]] # Load up individual mask bits for ibit, name in enumerate(pixelmask.name): if name is not '' and len( np.where(apvisit.bitmask[chip, :] & 2**ibit)[0]) > 0: raw.append([ np.clip(apvisit.bitmask[chip, :] & 2**ibit, None, 1), None ]) # Do the sinc interpolation out = sincint.sincint(pix[gd], nres[chip], raw) # From output flux, get continuum to remove, so that all spectra are # on same scale. We'll later multiply in the median continuum flux = out[0][0] stack.cont[i, gd] = gaussian_filter( median_filter(flux, [501], mode='reflect'), 100) # Load interpolated spectra into output stack stack.flux[i, gd] = out[0][0] / stack.cont[i, gd] stack.err[i, gd] = out[0][1] / stack.cont[i, gd] stack.sky[i, gd] = out[1][0] stack.skyerr[i, gd] = out[1][1] stack.telluric[i, gd] = out[2][0] stack.telerr[i, gd] = out[2][1] # For mask, set bits where interpolated value is above some threshold # defined for each mask bit iout = 3 for ibit, name in enumerate(pixelmask.name): if name is not '' and len( np.where(apvisit.bitmask[chip, :] & 2**ibit)[0]) > 0: j = np.where( np.abs(out[iout][0]) > pixelmask.maskcontrib[ibit])[0] stack.bitmask[i, gd[j]] |= 2**ibit iout += 1 # Increase uncertainties for persistence pixels bd, = np.where( (stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) > 0) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(5) bd, = np.where(( (stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) == 0) & ( (stack.bitmask[i, :] & pixelmask.getval('PERSIST_MED')) > 0)) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(4) bd, = np.where( ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) == 0) & ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_MED')) == 0) & ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_LOW')) > 0)) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(3) bd, = np.where( (stack.bitmask[i, :] & pixelmask.getval('SIG_SKYLINE')) > 0) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(100) if plot: ax[0].plot(norm.apStarWave(), stack.flux[i, :]) ax[1].plot(norm.apStarWave(), stack.flux[i, :] / stack.err[i, :]) plt.draw() pdb.set_trace() # Accumulate for header of combined frame. Turn off visit specific RV flags first visitflag = visit['starflag'] & ~starmask.getval( 'RV_REJECT') & ~starmask.getval('RV_SUSPECT') starflag |= visitflag andflag &= visitflag if visit['survey'] == 'apogee': apogee_target1 |= visit['apogee_target1'] apogee_target2 |= visit['apogee_target2'] apogee_target3 |= visit['apogee_target3'] elif visit['survey'].find('apogee2') >= 0: apogee2_target1 |= visit['apogee_target1'] apogee2_target2 |= visit['apogee_target2'] apogee2_target3 |= visit['apogee_target3'] try: apogee2_target4 |= visit['apogee_target4'] except: pass # MWM target flags? # Create final spectrum zeros = np.zeros([nvisit + 2, nwave]) izeros = np.zeros([nvisit + 2, nwave], dtype=int) apstar = apload.ApSpec(zeros, err=zeros.copy(), bitmask=izeros, wave=norm.apStarWave(), sky=zeros.copy(), skyerr=zeros.copy(), telluric=zeros.copy(), telerr=zeros.copy(), cont=zeros.copy(), template=zeros.copy()) apstar.header['CRVAL1'] = norm.logw0 apstar.header['CDELT1'] = norm.dlogw apstar.header['CRPIX1'] = 1 apstar.header['CTYPE1'] = ( 'LOG-LINEAR', 'Logarithmic wavelength scale in subsequent HDU') apstar.header['DC-FLAG'] = 1 # Pixel-by-pixel weighted average cont = np.median(stack.cont, axis=0) apstar.flux[0, :] = np.sum(stack.flux / stack.err**2, axis=0) / np.sum( 1. / stack.err**2, axis=0) * cont apstar.err[0, :] = np.sqrt(1. / np.sum(1. / stack.err**2, axis=0)) * cont apstar.bitmask[0, :] = np.bitwise_and.reduce(stack.bitmask, 0) apstar.cont[0, :] = cont # Individual visits apstar.flux[2:, :] = stack.flux * stack.cont apstar.err[2:, :] = stack.err * stack.cont apstar.bitmask[2:, :] = stack.bitmask apstar.sky[2:, :] = stack.sky apstar.skyerr[2:, :] = stack.skyerr apstar.telluric[2:, :] = stack.telluric apstar.telerr[2:, :] = stack.telerr # Populate header apstar.header['OBJID'] = (allvisit['apogee_id'][0], 'APOGEE object name') apstar.header['APRED'] = (apred, 'APOGEE reduction version') apstar.header['STARVER'] = (starver, 'apStar version') apstar.header['HEALPIX'] = (apload.obj2healpix(allvisit['apogee_id'][0]), 'HEALPix location') try: apstar.header['SNR'] = (np.nanmedian(apstar.flux / apstar.err), 'Median S/N per apStar pixel') except: apstar.header['SNR'] = (0., 'Median S/N per apStar pixel') apstar.header['RA'] = (allvisit['ra'].max(), 'right ascension, deg, J2000') apstar.header['DEC'] = (allvisit['dec'].max(), 'declination, deg, J2000') apstar.header['GLON'] = (allvisit['glon'].max(), 'Galactic longitude') apstar.header['GLAT'] = (allvisit['glat'].max(), 'Galactic latitude') apstar.header['J'] = (allvisit['j'].max(), '2MASS J magnitude') apstar.header['J_ERR'] = (allvisit['j_err'].max(), '2MASS J magnitude uncertainty') apstar.header['H'] = (allvisit['h'].max(), '2MASS H magnitude') apstar.header['H_ERR'] = (allvisit['h_err'].max(), '2MASS H magnitude uncertainty') apstar.header['K'] = (allvisit['k'].max(), '2MASS K magnitude') apstar.header['K_ERR'] = (allvisit['k_err'].max(), '2MASS K magnitude uncertainty') try: apstar.header['SRC_H'] = (allvisit['src_h'][0], 'source of H magnitude') except KeyError: pass keys = [ 'wash_m', 'wash_t2', 'ddo51', 'irac_3_6', 'irac_4_5', 'irac_5_8', 'wise_4_5', 'targ_4_5' ] for key in keys: try: apstar.header[key] = allvisit[key].max() except KeyError: pass apstar.header['AKTARG'] = (allvisit['ak_targ'].max(), 'Extinction used for targeting') apstar.header['AKMETHOD'] = (allvisit['ak_targ_method'][0], 'Extinction method using for targeting') apstar.header['AKWISE'] = (allvisit['ak_wise'].max(), 'WISE all-sky extinction') apstar.header['SFD_EBV'] = (allvisit['sfd_ebv'].max(), 'SFD E(B-V)') apstar.header['APTARG1'] = (apogee_target1, 'APOGEE_TARGET1 targeting flag') apstar.header['APTARG2'] = (apogee_target2, 'APOGEE_TARGET2 targeting flag') apstar.header['APTARG3'] = (apogee_target3, 'APOGEE_TARGET3 targeting flag') apstar.header['AP2TARG1'] = (apogee2_target1, 'APOGEE2_TARGET1 targeting flag') apstar.header['AP2TARG2'] = (apogee2_target2, 'APOGEE2_TARGET2 targeting flag') apstar.header['AP2TARG3'] = (apogee2_target3, 'APOGEE2_TARGET3 targeting flag') apstar.header['AP2TARG4'] = (apogee2_target4, 'APOGEE2_TARGET4 targeting flag') apstar.header['NVISITS'] = (len(allvisit), 'Number of visit spectra combined flag') apstar.header['STARFLAG'] = (starflag, 'bitwise OR of individual visit starflags') apstar.header['ANDFLAG'] = (andflag, 'bitwise AND of individual visit starflags') try: apstar.header['N_COMP'] = (allvisit['n_components'].max(), 'Maximum number of components in RV CCFs') except: pass apstar.header['VHBARY'] = ( (allvisit['vheliobary'] * allvisit['snr']).sum() / allvisit['snr'].sum(), 'S/N weighted mean barycentric RV') if len(allvisit) > 1: apstar.header['vscatter'] = (allvisit['vheliobary'].std(ddof=1), 'standard deviation of visit RVs') else: apstar.header['VSCATTER'] = (0., 'standard deviation of visit RVs') apstar.header['VERR'] = (0., 'unused') apstar.header['RV_TEFF'] = (allvisit['rv_teff'].max(), 'Effective temperature from RV fit') apstar.header['RV_LOGG'] = (allvisit['rv_logg'].max(), 'Surface gravity from RV fit') apstar.header['RV_FEH'] = (allvisit['rv_feh'].max(), 'Metallicity from RV fit') if len(allvisit) > 0: meanfib = (allvisit['fiberid'] * allvisit['snr']).sum() / allvisit['snr'].sum() else: meanfib = 999999. if len(allvisit) > 1: sigfib = allvisit['fiberid'].std(ddof=1) else: sigfib = 0. apstar.header['MEANFIB'] = (meanfib, 'S/N weighted mean fiber number') apstar.header['SIGFIB'] = ( sigfib, 'standard deviation (unweighted) of fiber number') apstar.header['NRES'] = ('{:5.2f}{:5.2f}{:5.2f}'.format(*nres), 'number of pixels/resolution used for sinc') # individual visit information in header for i0, visit in enumerate(allvisit): i = i0 + 1 apstar.header['SFILE{:d}'.format(i)] = ( visit['file'], ' Visit #{:d} spectrum file'.format(i)) apstar.header['DATE{:d}'.format(i)] = ( visit['dateobs'], 'DATE-OBS of visit {:d}'.format(i)) apstar.header['JD{:d}'.format(i)] = ( visit['jd'], 'Julian date of visit {:d}'.format(i)) # hjd = helio_jd(visitstr[i].jd-2400000.0,visitstr[i].ra,visitstr[i].dec) #apstar.header['HJD{:d}'.format(i)] = apstar.header['FIBER{:d}'.format(i)] = (visit['fiberid'], ' Fiber, visit {:d}'.format(i)) apstar.header['BC{:d}'.format(i)] = ( visit['bc'], ' Barycentric correction (km/s), visit {:d}'.format(i)) apstar.header['VRAD{:d}'.format(i)] = ( visit['vrel'], ' Doppler shift (km/s) of visit {:d}'.format(i)) #apstar.header['VERR%d'.format(i)] = apstar.header['VHBARY{:d}'.format(i)] = ( visit['vheliobary'], ' Barycentric velocity (km/s), visit {:d}'.format(i)) apstar.header['SNRVIS{:d}'.format(i)] = ( visit['snr'], ' Signal/Noise ratio, visit {:d}'.format(i)) apstar.header['FLAG{:d}'.format(i)] = ( visit['starflag'], ' STARFLAG for visit {:d}'.format(i)) apstar.header.insert('SFILE{:d}'.format(i), ('COMMENT', 'VISIT {:d} INFORMATION'.format(i))) # Do a RV fit just to get a template and normalized spectrum, for plotting if dorvfit: try: apstar.setmask(pixelmask.badval()) spec = doppler.Spec1D(apstar.flux[0, :], err=apstar.err[0, :], bitmask=apstar.bitmask[0, :], mask=apstar.mask[0, :], wave=apstar.wave, lsfpars=np.array([0]), lsfsigma=apstar.wave / 22500 / 2.354, instrument='APOGEE', filename=apstar.filename) out = doppler.rv.jointfit([spec], verbose=False, plot=False, tweak=False, maxvel=[-50, 50]) apstar.cont = out[3][0].flux apstar.template = out[2][0].flux except ValueError as err: logger.error('Exception raised in visitcomb RV for: ', apstar.header['FIELD'], apstar.header['OBJID']) logger.error("ValueError: {0}".format(err)) except RuntimeError as err: logger.error('Exception raised in visitcomb RV for: ', apstar.header['FIELD'], apstar.header['OBJID']) logger.error("Runtime error: {0}".format(err)) except: logger.error('Exception raised in visitcomb RV fit for: ', apstar.header['FIELD'], apstar.header['OBJID']) # Write the spectrum to file if write: outfilenover = load.filename('Star', obj=apstar.header['OBJID']) outdir = os.path.dirname(outfilenover) outbase = os.path.splitext(os.path.basename(outfilenover))[0] outbase += '-' + starver # add star version outfile = outdir + '/' + outbase + '.fits' if apstar_vers != 'stars': outfile = outfile.replace('/stars/', '/' + apstar_vers + '/') outdir = os.path.dirname(outfile) try: os.makedirs(os.path.dirname(outfile)) except: pass logger.info('Writing apStar file to ' + outfile) apstar.write(outfile) apstar.filename = outfile mwm_root = os.environ['MWM_ROOT'] apstar.uri = outfile[len(mwm_root) + 1:] # Create symlink no file with no version if os.path.exists(outfilenover) or os.path.islink(outfilenover): os.remove(outfilenover) os.symlink(os.path.basename(outfile), outfilenover) # relative path # Plot gd, = np.where( (apstar.bitmask[0, :] & (pixelmask.badval() | pixelmask.getval('SIG_SKYLINE'))) == 0) fig, ax = plots.multi(1, 3, hspace=0.001, figsize=(48, 6)) med = np.nanmedian(apstar.flux[0, :]) plots.plotl(ax[0], norm.apStarWave(), apstar.flux[0, :], color='k', yr=[0, 2 * med]) ax[0].plot(norm.apStarWave()[gd], apstar.flux[0, gd], color='g') ax[0].set_ylabel('Flux') try: ax[1].plot(norm.apStarWave()[gd], apstar.cont[gd], color='g') ax[1].set_ylabel('Normalized') ax[1].plot(norm.apStarWave(), apstar.template, color='r') except: pass plots.plotl(ax[2], norm.apStarWave(), apstar.flux[0, :] / apstar.err[0, :], yt='S/N') for i in range(3): ax[i].set_xlim(15100, 17000) ax[0].set_xlabel('Wavelength') fig.savefig(outdir + '/plots/' + outbase + '.png') # Plot if plot: ax[0].plot(norm.apStarWave(), apstar.flux, color='k') ax[1].plot(norm.apStarWave(), apstar.flux / apstar.err, color='k') plt.draw() pdb.set_trace() return apstar
mFilterSize = 3 mFilterEvery = 30 mIter = 1001 # step 5, mFilterEvery 8, mFilterSize 3 # we run gradient ascent for 20 steps for i in range(mIter): loss_value, grads_value = iterate([input_img_data]) input_img_data += grads_value * step input_img_data = np.clip(input_img_data, 0., 255.) if mFilterSize is not 0 and i % mFilterEvery == 0: input_img_data = median_filter(input_img_data, size=(1, 1, mFilterSize, mFilterSize)) if i % 50 == 0: print('\t%d, Current loss value:%f' % (i, loss_value)) # decode the resulting input image img = deprocess_image(input_img_data[0]) kept_filters.append((img, loss_value, filter_index)) end_time = time.time() print('%d, Filter %d processed in %ds' % (ix, filter_index, end_time - start_time)) print("=" * 80) print("Finish!")
def _medianf(f): for i in range(niter): f = median_filter(f, size) return f
def ffttc_traction_pure_shear(u, v, pixelsize1, pixelsize2, h, young, sigma=0.49, spatial_filter="mean", fs=None): """ limiting case for h*k==0 Xavier Trepat, Physical forces during collective cell migration, 2009 :param u:deformation field in x direction in pixel of the deformation image :param v:deformation field in y direction in pixel of the deformation image :param young: Young's modulus in Pa :param pixelsize1: pixelsize of the original image, needed because u and v is given as displacement of these pixels :param pixelsize2: pixelsize of the deformation image :param h: height of the membrane the cells lie on, in µm :param sigma: Poisson's ratio of the gel :param spatial_filter: str, values: "mean","gaussian","median". Different smoothing methods for the traction field. :return: tx_filter,ty_filter: traction forces in x and y direction in Pa """ # 0) subtracting mean(better name for this step) u_shift = (u - np.mean(u)) * pixelsize1 v_shift = (v - np.mean(v)) * pixelsize1 # Ben's algorithm: # 1)Zero padding to get square array with even index number ax1_length = np.shape(u_shift)[0] # u and v must have same dimensions ax2_length = np.shape(u_shift)[1] max_ind = int(np.max((ax1_length, ax2_length))) if max_ind % 2 != 0: max_ind += 1 u_expand = np.zeros((max_ind, max_ind)) v_expand = np.zeros((max_ind, max_ind)) u_expand[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] = u_shift v_expand[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] = v_shift u_ft = scipy.fft.fft2(u_expand) v_ft = scipy.fft.fft2(v_expand) # 4.1) calculate tractions in Fourier space mu = young / (2 * (1 + sigma)) tx_ft = mu * u_ft / h tx_ft[0, 0] = 0 # zero frequency would represent force everywhere (constant) ty_ft = mu * v_ft / h ty_ft[0, 0] = 0 # 4.2) go back to real space tx = scipy.fft.ifft2(tx_ft).real ty = scipy.fft.ifft2(ty_ft).real # 5.2) cut like in script from ben tx_cut = tx[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] ty_cut = ty[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] # 5.3) using filter tx_filter = tx_cut ty_filter = ty_cut if spatial_filter == "mean": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = uniform_filter(tx_cut, size=fs) ty_filter = uniform_filter(ty_cut, size=fs) if spatial_filter == "gaussian": fs = fs if isinstance(fs, (float, int)) else int(np.max( (ax1_length, ax2_length))) / 50 tx_filter = gaussian_filter(tx_cut, sigma=fs) ty_filter = gaussian_filter(ty_cut, sigma=fs) if spatial_filter == "median": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = median_filter(tx_cut, size=fs) ty_filter = median_filter(ty_cut, size=fs) return tx_filter, ty_filter
def AGD(vel, data, errors, alpha1 = None, alpha2 = None, plot = False, mode ='c', verbose = False, SNR_thresh = 5.0, BLFrac = 0.1, SNR2_thresh=5.0, deblend=True, perform_final_fit = True, phase='one'): """ Autonomous Gaussian Decomposition """ if type(SNR2_thresh) != type([]): SNR2_thresh = [SNR2_thresh, SNR2_thresh] if type(SNR_thresh) != type([]): SNR_thresh = [SNR_thresh, SNR_thresh] say('\n --> AGD() \n',verbose) if (not alpha2) and (phase=='two'): print 'alpha2 value required' return dv = np.abs(vel[1] - vel[0]) v_to_i = interp1d(vel, np.arange(len(vel))) #--------------------------------------# # Find phase-one guesses # #--------------------------------------# agd1 = initialGuess(vel, data, errors = None, alpha = alpha1, plot = plot, mode = mode, verbose = verbose, SNR_thresh = SNR_thresh[0], BLFrac = BLFrac, SNR2_thresh = SNR2_thresh[0], deblend = deblend) amps_g1, widths_g1, offsets_g1, u2 = agd1['amps'], agd1['FWHMs'], agd1['means'], agd1['u2'] params_g1 = np.append(np.append(amps_g1, widths_g1), offsets_g1) ncomps_g1 = len(params_g1) / 3 ncomps_g2 = 0 # Default ncomps_f1 = 0 # Default # ----------------------------# # Find phase-two guesses # # ----------------------------# if phase == 'two': say('Beginning phase-two AGD... ', verbose) ncomps_g2 = 0. # ----------------------------------------------------------# # Produce the residual signal # # -- Either the original data, or intermediate subtraction # # ----------------------------------------------------------# if ncomps_g1 == 0: say('Phase 2 with no narrow comps -> No intermediate subtration... ', verbose) residuals = data else: # "Else" Narrow components were found, and Phase == 2, so perform intermediate subtraction... # The "fitmask" is a collection of windows around the a list of phase-one components fitmask, fitmaskw = create_fitmask(len(vel), v_to_i(offsets_g1), widths_g1 / dv / 2.355 * 0.9) notfitmask = 1 - fitmask notfitmaskw = np.logical_not(fitmaskw) # Error function for intermediate optimization def objectiveD2_leastsq(paramslm): params = vals_vec_from_lmfit(paramslm) model0 = func(vel, *params) model2 = np.diff(np.diff(model0.ravel()))/dv/dv resids1 = fitmask[1:-1] * (model2 - u2[1:-1]) / errors[1:-1] resids2 = notfitmask * (model0 - data) / errors /10. return np.append(resids1, resids2) # Perform the intermediate fit using LMFIT t0 = time.time() say('Running LMFIT on initial narrow components...', verbose) lmfit_params = paramvec_to_lmfit(params_g1) result = lmfit_minimize(objectiveD2_leastsq, lmfit_params, method='leastsq') params_f1 = vals_vec_from_lmfit(result.params) ncomps_f1 = len(params_f1) / 3 # Make "FWHMS" positive params_f1[0:ncomps_f1][params_f1[0:ncomps_f1] < 0.0] = -1 * params_f1[0:ncomps_f1][params_f1[0:ncomps_f1] < 0.0] del lmfit_params say('LMFIT fit took {0} seconds.'.format(time.time()-t0)) if result.success: # Compute intermediate residuals # Median filter on 2x effective scale to remove poor subtractions of strong components intermediate_model = func(vel, *params_f1).ravel() # Explicit final (narrow) model median_window = 2. * 10**((np.log10(alpha1) + 2.187) / 3.859) residuals = median_filter(data - intermediate_model, np.int(median_window)) else: residuals = data # Finished producing residual signal # --------------------------- # Search for phase-two guesses agd2 = initialGuess(vel, residuals, errors = None, alpha = alpha2, mode = mode, verbose = verbose, SNR_thresh = SNR_thresh[1], BLFrac = BLFrac, SNR2_thresh = SNR2_thresh[1], # June 9 2014, change deblend=deblend, plot=plot) ncomps_g2 = agd2['N_components'] if ncomps_g2 > 0: params_g2 = np.concatenate([agd2['amps'],agd2['FWHMs'], agd2['means']]) else: params_g2 = [] u22 = agd2['u2'] # END PHASE 2 <<< # Check for phase two components, make final guess list # ------------------------------------------------------ if phase=='two' and (ncomps_g2 > 0): amps_gf = np.append(params_g1[0:ncomps_g1], params_g2[0:ncomps_g2]) widths_gf = np.append(params_g1[ncomps_g1:2*ncomps_g1], params_g2[ncomps_g2:2*ncomps_g2]) offsets_gf = np.append(params_g1[2*ncomps_g1:3*ncomps_g1], params_g2[2*ncomps_g2:3*ncomps_g2]) params_gf = np.concatenate([amps_gf, widths_gf, offsets_gf]) ncomps_gf = len(params_gf) / 3 else: params_gf = params_g1 ncomps_gf = len(params_gf) / 3 # Sort final guess list by amplitude # ---------------------------------- say('N final parameter guesses: ' + str(ncomps_gf)) amps_temp = params_gf[0:ncomps_gf] widths_temp = params_gf[ncomps_gf:2*ncomps_gf] offsets_temp = params_gf[2*ncomps_gf:3*ncomps_gf] w_sort_amp = np.argsort(amps_temp)[::-1] params_gf = np.concatenate([amps_temp[w_sort_amp], widths_temp[w_sort_amp], offsets_temp[w_sort_amp]]) if (perform_final_fit == True) and (ncomps_gf > 0): say('\n\n --> Final Fitting... \n',verbose) # Objective functions for final fit def objective_leastsq(paramslm): params = vals_vec_from_lmfit(paramslm) resids = (func(vel, *params).ravel() - data.ravel()) / errors return resids # Final fit using unconstrained parameters t0 = time.time() lmfit_params = paramvec_to_lmfit(params_gf) result2 = lmfit_minimize(objective_leastsq, lmfit_params, method='leastsq') params_fit = vals_vec_from_lmfit(result2.params) params_errs = errs_vec_from_lmfit(result2.params) ncomps_fit = len(params_fit)/3 del lmfit_params say('Final fit took {0} seconds.'.format(time.time()-t0), verbose) # Make "FWHMS" positive params_fit[0:ncomps_fit][params_fit[0:ncomps_fit] < 0.0] = -1 * params_fit[0:ncomps_fit][params_fit[0:ncomps_fit] < 0.0] best_fit_final = func(vel, *params_fit).ravel() rchi2 = np.sum( (data - best_fit_final)**2 / errors**2) / len(data) # Check if any amplitudes are identically zero, if so, remove them. if np.any(params_fit[0:ncomps_gf] == 0.0): amps_fit = params_fit[0:ncomps_gf] fwhms_fit = params_fit[ncomps_gf:2*ncomps_gf] offsets_fit = params_fit[2*ncomps_gf:3*ncomps_gf] w_keep = amps_fit > 0. params_fit = np.concatenate([amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]]) ncomps_fit = len(params_fit)/3 if plot: datamax = np.max(data) # Set up figure fig = plt.figure('AGD results', [12,12]) ax1 = fig.add_axes([0.1,0.5,0.4,0.4]) # Initial guesses (alpha1) ax2 = fig.add_axes([0.5,0.5,0.4,0.4]) # D2 fit to peaks(alpha2) ax3 = fig.add_axes([0.1,0.1,0.4,0.4]) # Initial guesses (alpha2) ax4 = fig.add_axes([0.5,0.1,0.4,0.4]) # Final fit # Decorations plt.figtext(0.52,0.47,'Final fit') if perform_final_fit: try: plt.figtext(0.52,0.45,'Reduced Chi2: {0:3.1f}'.format(rchi2)) plt.figtext(0.52,0.43,'N components: {0}'.format(ncomps_fit)) except UnboundLocalError: pass plt.figtext(0.12,0.47,'Phase-two initial guess') plt.figtext(0.12,0.45,'N components: {0}'.format(ncomps_g2)) plt.figtext(0.12,0.87,'Phase-one initial guess') plt.figtext(0.12,0.85,'N components: {0}'.format(ncomps_g1)) plt.figtext(0.52,0.87,'Intermediate fit') # Initial Guesses (Panel 1) # ------------------------- ax1.xaxis.tick_top() u2_scale = 1. / np.max(np.abs(u2)) * datamax * 0.5 ax1.plot(vel, data, '-k') ax1.plot(vel, u2 * u2_scale, '-r') ax1.plot(vel, vel / vel * agd1['thresh'], '-k') ax1.plot(vel, vel / vel * agd1['thresh2'] * u2_scale, '--r') for i in range(ncomps_g1): one_component = gaussian(params_g1[i], params_g1[i+ncomps_g1], params_g1[i+2*ncomps_g1])(vel) ax1.plot(vel, one_component,'-g') # Plot intermediate fit components (Panel 2) # ------------------------------------------ ax2.xaxis.tick_top() ax2.plot(vel, data, '-k') ax2.yaxis.tick_right() for i in range(ncomps_f1): one_component = gaussian(params_f1[i], params_f1[i+ncomps_f1], params_f1[i+2*ncomps_f1])(vel) ax2.plot(vel, one_component,'-',color='blue') # Residual spectrum (Panel 3) # ----------------------------- if phase == 'two': u22_scale = 1. / np.abs(u22).max() * np.max(residuals) * 0.5 ax3.plot(vel, residuals, '-k') ax3.plot(vel, vel / vel * agd2['thresh'], '--k') ax3.plot(vel, vel / vel * agd2['thresh2'] * u22_scale, '--r') ax3.plot(vel, u22 * u22_scale, '-r') for i in range(ncomps_g2): one_component = gaussian(params_g2[i], params_g2[i+ncomps_g2], params_g2[i+2*ncomps_g2])(vel) ax3.plot(vel, one_component,'-g') # Plot best-fit model (Panel 4) # ----------------------------- if perform_final_fit: ax4.yaxis.tick_right() try: ax4.plot(vel, best_fit_final, label='final model',color='purple') ax4.plot(vel, data, label='data',color='black') for i in range(ncomps_fit): one_component = gaussian(params_fit[i], params_fit[i+ncomps_fit], params_fit[i+2*ncomps_fit])(vel) ax4.plot(vel, one_component,'-',color='purple') ax4.plot(vel ,best_fit_final, '-', color='purple') except UnboundLocalError: pass plt.show() # Construct output dictionary (odict) # ----------------------------------- odict = {} odict['initial_parameters'] = params_gf odict['N_components'] = ncomps_gf if (perform_final_fit == True) and (ncomps_gf > 0): odict['best_fit_parameters'] = params_fit odict['best_fit_errors'] = params_errs odict['rchi2'] = rchi2 return (1, odict)
def ffttc_traction_pure_shear(u, v, pixelsize1, pixelsize2, h, young, sigma=0.49, filter="mean", fs=None): ''' limiting case for h*k==0 Xavier Trepat, Physical forces during collective cell migration, 2009 :param u:deformation field in x direction in pixel of the deformation image :param v:deformation field in y direction in pixel of the deformation image :param young: youngs modulus in Pa :param pixelsize1: pixelsize of the original image, needed because u and v is given as displacment of these pixels :param pixelsize2: pixelsize of the deformation image :param h hight of the membrane the cells lie on, in µm :param sigma: poission ratio of the gel :param bf_image: give the brightfield image as an array before cells where removed :param filter: str, values: "mean","gaussian","median". Diffrent smoothing methods for the traction field. :return: tx_filter,ty_filter: traction forces in x and y direction in Pa ''' # 0) substracting mean(better name for this step) u_shift = (u - np.mean(u)) * pixelsize1 # also try without dis v_shift = (v - np.mean(v)) * pixelsize1 ## bens algortithm: # 1)Zero padding to get sqauerd array with even index number ax1_length = np.shape(u_shift)[0] # u and v must have same dimensions ax2_length = np.shape(u_shift)[1] max_ind = int(np.max((ax1_length, ax2_length))) if max_ind % 2 != 0: max_ind += 1 u_expand = np.zeros((max_ind, max_ind)) v_expand = np.zeros((max_ind, max_ind)) u_expand[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] = u_shift v_expand[ max_ind - ax1_length:max_ind, max_ind - ax2_length: max_ind] = v_shift # note: seems to be snummerically slightly diffrent then in normal fnction ??? (dont know why # 2) producing wave vectors (FT-space) " # form 1:max_ind/2 then -(max_ind/2:1) kx1 = np.array([ list(range(0, int(max_ind / 2), 1)), ] * int(max_ind)) kx2 = np.array([ list(range(-int(max_ind / 2), 0, 1)), ] * int(max_ind)) kx = np.append(kx1, kx2, axis=1) / ( pixelsize2 * max_ind ) * 2 * np.pi # spatial frequencies: 1/wavelength,in 1/µm in fractions of total length ky = np.transpose(kx) k = np.sqrt(kx**2 + ky**2) # matrix with "relative" distances??# u_ft = scipy.fft.fft2(u_expand) v_ft = scipy.fft.fft2(v_expand) # 4.1) calculate tractions in fourrier space mu = young / (2 * (1 + sigma)) tx_ft = mu * u_ft / h tx_ft[ 0, 0] = 0 ## zero frequency would represent force every where(constant), so this is legit?? ty_ft = mu * v_ft / h ty_ft[0, 0] = 0 # 4.2) go back to real space tx = scipy.fft.ifft2(tx_ft).real ## maybe devide by 2pi here??? ty = scipy.fft.ifft2(ty_ft).real # 5.2) cut like in script from ben tx_cut = tx[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] ty_cut = ty[max_ind - ax1_length:max_ind, max_ind - ax2_length:max_ind] # 5.3) using filter if filter == "mean": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = uniform_filter(tx_cut, size=fs) ty_filter = uniform_filter(ty_cut, size=fs) if filter == "gaussian": fs = fs if isinstance(fs, (float, int)) else int(np.max( (ax1_length, ax2_length))) / 50 tx_filter = gaussian_filter(tx_cut, sigma=fs) ty_filter = gaussian_filter(ty_cut, sigma=fs) if filter == "median": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = median_filter(tx_cut, size=fs) ty_filter = median_filter(ty_cut, size=fs) if not isinstance(filter, str): tx_filter = tx_cut ty_filter = ty_cut # show_quiver(tx_filter,ty_filter) return (tx_filter, ty_filter)
def slice_wise_fft(in_file, ftmask=None, spike_thres=3., out_prefix=None): """Search for spikes in slices using the 2D FFT""" import os.path as op import numpy as np import nibabel as nb from mriqc.workflows.utils import spectrum_mask from scipy.ndimage.filters import median_filter from scipy.ndimage import generate_binary_structure, binary_erosion from statsmodels.robust.scale import mad if out_prefix is None: fname, ext = op.splitext(op.basename(in_file)) if ext == '.gz': fname, _ = op.splitext(fname) out_prefix = op.abspath(fname) func_data = nb.load(in_file).get_data() if ftmask is None: ftmask = spectrum_mask(tuple(func_data.shape[:2])) fft_data = [] for t in range(func_data.shape[-1]): func_frame = func_data[..., t] fft_slices = [] for z in range(func_frame.shape[2]): sl = func_frame[..., z] fftsl = median_filter(np.real(np.fft.fft2(sl)).astype(np.float32), size=(5, 5), mode='constant') * ftmask fft_slices.append(fftsl) fft_data.append(np.stack(fft_slices, axis=-1)) # Recompose the 4D FFT timeseries fft_data = np.stack(fft_data, -1) # Z-score across t, using robust statistics mu = np.median(fft_data, axis=3) sigma = np.stack([mad(fft_data, axis=3)] * fft_data.shape[-1], -1) idxs = np.where(np.abs(sigma) > 1e-4) fft_zscored = fft_data - mu[..., np.newaxis] fft_zscored[idxs] /= sigma[idxs] # save fft z-scored out_fft = op.abspath(out_prefix + '_zsfft.nii.gz') nii = nb.Nifti1Image(fft_zscored.astype(np.float32), np.eye(4), None) nii.to_filename(out_fft) # Find peaks spikes_list = [] for t in range(fft_zscored.shape[-1]): fft_frame = fft_zscored[..., t] for z in range(fft_frame.shape[-1]): sl = fft_frame[..., z] if np.all(sl < spike_thres): continue # Any zscore over spike_thres will be called a spike sl[sl <= spike_thres] = 0 sl[sl > 0] = 1 # Erode peaks and see how many survive struc = generate_binary_structure(2, 2) sl = binary_erosion(sl.astype(np.uint8), structure=struc).astype(np.uint8) if sl.sum() > 10: spikes_list.append((t, z)) out_spikes = op.abspath(out_prefix + '_spikes.tsv') np.savetxt(out_spikes, spikes_list, fmt=b'%d', delimiter=b'\t', header='TR\tZ') return len(spikes_list), out_spikes, out_fft
def ffttc_traction(u, v, pixelsize1, pixelsize2, young, sigma=0.49, filter="gaussian", fs=None): ''' fourier transform based calculation of the traction force. U and v must be given as deformations in pixel. Size of these pixels must be the pixelsize (size of a pixel in the deformation field u or v). Note that thePiv deformation returns deformation in pixel of the size of pixels in the images of beads before and after. If bf_image is provided this script will return a traction field that is zoomed to the size of the brightfield image, by interpolation. It is not recommended to use this for any calculations. The function can use diffretn filters. Recommended filter is gaussian. Mean filter should yield similar results. :param u:deformation field in x direction in pixel of the deformation image :param v:deformation field in y direction in pixel of the deformation image :param young: youngs modulus in Pa :param pixelsize1: pixelsize in m/pixel of the original image, needed because u and v is given as displacement of these pixels :param pixelsize2: pixelsize of m/pixel the deformation image :param sigma: posson ratio of the gel :param bf_image: give the brightfield image as an array before cells where removed :param filter: str, values: "mean","gaussian","median". Diffrent smoothing methods for the traction field :return: tx_filter,ty_filter: traction forces in x and y direction in Pa ''' # 0) substracting mean(better name for this step) u_shift = ( u - np.mean(u) ) # shifting to zero mean (translating to pixelsize of u-image is done later) v_shift = (v - np.mean(v)) ## bens algortithm: # 1)Zeropadding to get sqauerd array with even index number ax1_length = np.shape(u_shift)[0] # u and v must have same dimensions ax2_length = np.shape(u_shift)[1] max_ind = int(np.max((ax1_length, ax2_length))) if max_ind % 2 != 0: max_ind += 1 u_expand = np.zeros((max_ind, max_ind)) v_expand = np.zeros((max_ind, max_ind)) u_expand[:ax1_length, :ax2_length] = u_shift v_expand[:ax1_length, :ax2_length] = v_shift # 2) produceing wave vectors ## why this form # form 1:max_ind/2 then -(max_ind/2:1) kx1 = np.array([ list(range(0, int(max_ind / 2), 1)), ] * int(max_ind)) kx2 = np.array([ list(range(-int(max_ind / 2), 0, 1)), ] * int(max_ind)) kx = np.append( kx1, kx2, axis=1) * 2 * np.pi # fourier transform in this case is defined as # F(kx)=1/2pi integral(exp(i*kx*x)dk therefore kx must be expressed as a spatial frequency in distance*2*pi ky = np.transpose(kx) k = np.sqrt(kx**2 + ky**2) / (pixelsize2 * max_ind) # np.save("/home/user/Desktop/k_test.npy",k) # 2.1) calculating angle between k and kx with atan 2 function (what is this exactely??) just if statemments to get # angel from x1 to x2 in fixed direction... (better explanation) alpha = np.arctan2(ky, kx) alpha[0, 0] = np.pi / 2 # np.save("/home/user/Desktop/alpha_test.npy",alpha) # 3) calculation of K --> Tensor to calculate displacements from Tractions. We calculate inverse of K # (check if correct inversion by your self) # K⁻¹=[[kix kid], # [kid,kiy]] ,,, so is "diagonal, kid appears two times kix = ((k * young) / (2 * (1 - sigma**2))) * ((1 - sigma + sigma * np.cos(alpha)**2)) kiy = ((k * young) / (2 * (1 - sigma**2))) * ((1 - sigma + sigma * np.sin(alpha)**2)) kid = ((k * young) / (2 * (1 - sigma**2))) * (sigma * np.sin(alpha) * np.cos(alpha)) # np.save("/home/user/Desktop/kid_seg2_test.npy",(sigma * np.sin(alpha) * np.cos(alpha))) ## adding zeros in kid diagonals(?? why do i need this) kid[:, int(max_ind / 2)] = np.zeros(max_ind) kid[int(max_ind / 2), :] = np.zeros(max_ind) # 4) calculate fourrier transform of dsiplacements # u_ft=np.fft.fft2(u_expand*pixelsize1*2*np.pi) # v_ft=np.fft.fft2(v_expand*pixelsize1*2*np.pi) # u_ft = scipy.fft.fft2(u_expand * pixelsize1) v_ft = scipy.fft.fft2(v_expand * pixelsize1) # 4.1) calculate tractions in fourrier space T=K⁻¹*U, U=[u,v] here with individual matrix elelemnts.. tx_ft = kix * u_ft + kid * v_ft ty_ft = kid * u_ft + kiy * v_ft # 4.2) go back to real space tx = scipy.fft.ifft2(tx_ft).real ty = scipy.fft.ifft2(ty_ft).real # 5.2) cut back to oringinal shape tx_cut = tx[0:ax1_length, 0:ax2_length] ty_cut = ty[0:ax1_length, 0:ax2_length] # 5.3) using filter if filter == "mean": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = uniform_filter(tx_cut, size=fs) ty_filter = uniform_filter(ty_cut, size=fs) if filter == "gaussian": fs = fs if isinstance(fs, (float, int)) else int(np.max( (ax1_length, ax2_length))) / 50 tx_filter = gaussian_filter(tx_cut, sigma=fs) ty_filter = gaussian_filter(ty_cut, sigma=fs) if filter == "median": fs = fs if isinstance(fs, (float, int)) else int( int(np.max((ax1_length, ax2_length))) / 16) tx_filter = median_filter(tx_cut, size=fs) ty_filter = median_filter(ty_cut, size=fs) if not isinstance(filter, str): tx_filter = tx_cut ty_filter = ty_cut return (tx_filter, ty_filter)
x = width - 1 if y < 0: y = 0 elif y >= height: y = height - 1 return x, y print "CREATING..." depths = create_depths_from_pix(pix) print "CREATED" print "SHIFTING..." depths = shift_depths(depths) print "SHIFTED" # print "DILATING..." # depths = dilate(depths) # print "DILATED" # print "ERODING..." # depths = erode(depths) # print "ERODED" print "MEDIANING..." depths = filters.median_filter(depths, size=(13, 5)) print "MEDIANED" print "ADDING NOISE..." depths = add_noise(depths) print "DONE" new_img = depths_to_image(depths) new_img.show()
def Blink_Tracker(EAR, IF_Closed_Eyes, Counter4blinks, TOTAL_BLINKS, skip): BLINK_READY = False if int(IF_Closed_Eyes) == 1: # If the eyes are closed Current_Blink.values.append(EAR) Current_Blink.EAR_of_FOI = EAR # Save to use later if Counter4blinks > 0: skip = False if Counter4blinks == 0: Current_Blink.startEAR = EAR # EAR_series[6] is the EAR for the frame of interest(the middle one) Current_Blink.start = reference_frame - 6 # reference-6 points to the frame of interest which will be the 'start' of the blink Counter4blinks += 1 if Current_Blink.peakEAR >= EAR: # deciding the min point of the EAR signal Current_Blink.peakEAR = EAR Current_Blink.peak = reference_frame - 6 else: # otherwise, the eyes are open in this frame if Counter4blinks < 2 and skip == False: # Wait to approve or reject the last blink if Last_Blink.duration > 15: FRAME_MARGIN_BTW_2BLINKS = 8 else: FRAME_MARGIN_BTW_2BLINKS = 1 if ((reference_frame - 6) - Last_Blink.end) > FRAME_MARGIN_BTW_2BLINKS: # Check so the prev blink signal is not monotonic or too small (noise) if Last_Blink.peakEAR < Last_Blink.startEAR and Last_Blink.peakEAR < Last_Blink.endEAR and Last_Blink.amplitude > MIN_AMPLITUDE and Last_Blink.start < Last_Blink.peak: if ((Last_Blink.startEAR - Last_Blink.peakEAR) > (Last_Blink.endEAR - Last_Blink.peakEAR) * 0.25 and (Last_Blink.startEAR - Last_Blink.peakEAR) * 0.25 < (Last_Blink.endEAR - Last_Blink.peakEAR)): # the amplitude is balanced BLINK_READY = True #THE ULTIMATE BLINK Check Last_Blink.values = signal.convolve1d( Last_Blink.values, [1 / 3.0, 1 / 3.0, 1 / 3.0], mode='nearest') Last_Blink.values = signal.median_filter( Last_Blink.values, 3, mode='reflect') # #smoothing the signal [MISSED_BLINKS, retrieved_blinks] = Ultimate_Blink_Check() TOTAL_BLINKS = TOTAL_BLINKS + len( retrieved_blinks ) # Finally, approving/counting the previous blink candidate ###Now You can count on the info of the last separate and valid blink and analyze it Counter4blinks = 0 print("RETRIVED BLINKS= {}".format( len(retrieved_blinks))) return retrieved_blinks, int( TOTAL_BLINKS ), Counter4blinks, BLINK_READY, skip else: skip = True print('rejected due to imbalance') else: skip = True print('rejected due to noise,magnitude is {}'.format( Last_Blink.amplitude)) print(Last_Blink.start < Last_Blink.peak) # if the eyes were closed for a sufficient number of frames (2 or more) # then this is a valid CANDIDATE for a blink if Counter4blinks > 1: Current_Blink.end = reference_frame - 7 # reference-7 points to the last frame that eyes were closed Current_Blink.endEAR = Current_Blink.EAR_of_FOI Current_Blink.amplitude = (Current_Blink.startEAR + Current_Blink.endEAR - 2 * Current_Blink.peakEAR) / 2 Current_Blink.duration = Current_Blink.end - Current_Blink.start + 1 if Last_Blink.duration > 15: FRAME_MARGIN_BTW_2BLINKS = 8 else: FRAME_MARGIN_BTW_2BLINKS = 1 if ( Current_Blink.start - Last_Blink.end ) <= FRAME_MARGIN_BTW_2BLINKS + 1: # Merging two close blinks print('Merging...') frames_in_between = Current_Blink.start - Last_Blink.end - 1 print(Current_Blink.start, Last_Blink.end, frames_in_between) valuesBTW = Linear_Interpolate(Last_Blink.endEAR, Current_Blink.startEAR, frames_in_between) Last_Blink.values = Last_Blink.values + valuesBTW + Current_Blink.values Last_Blink.end = Current_Blink.end # update the end Last_Blink.endEAR = Current_Blink.endEAR if Last_Blink.peakEAR > Current_Blink.peakEAR: # update the peak Last_Blink.peakEAR = Current_Blink.peakEAR Last_Blink.peak = Current_Blink.peak # update duration and amplitude Last_Blink.amplitude = (Last_Blink.startEAR + Last_Blink.endEAR - 2 * Last_Blink.peakEAR) / 2 Last_Blink.duration = Last_Blink.end - Last_Blink.start + 1 else: # Should not Merge (a Separate blink) Last_Blink.values = Current_Blink.values # update the EAR list Last_Blink.end = Current_Blink.end # update the end Last_Blink.endEAR = Current_Blink.endEAR Last_Blink.start = Current_Blink.start # update the start Last_Blink.startEAR = Current_Blink.startEAR Last_Blink.peakEAR = Current_Blink.peakEAR # update the peak Last_Blink.peak = Current_Blink.peak Last_Blink.amplitude = Current_Blink.amplitude Last_Blink.duration = Current_Blink.duration # reset the eye frame counter Counter4blinks = 0 retrieved_blinks = 0 return retrieved_blinks, int( TOTAL_BLINKS), Counter4blinks, BLINK_READY, skip
coords = [] img = images_hu[k] root = tk.Tk() im = Image.fromarray(img) photo = ImageTk.PhotoImage(im) canvas = tk.Canvas(root, width=512, height=512) canvas.pack() canvas.create_image(256, 256, image=photo) canvas.bind("<Button-1>", onclick) root.mainloop() for i in range(img.shape[0]): for j in range(img.shape[0]): if img[i][j] > 0: img[i][j] *= 12 blurred_f = ndimage.gaussian_filter(img, sigma=3) filter_blurred_f = filters.median_filter(img, 3) filter_blurred_f2 = ndimage.gaussian_filter(filter_blurred_f, sigma=1) alpha = 10 sharpened = filter_blurred_f + alpha * (filter_blurred_f - filter_blurred_f2) seed = np.zeros(img.shape) for jj in range(len(coords)): seed[int(coords[jj][1]), int(coords[jj][0])] = 1 seg, phi, its = lvlset(sharpened, seed, max_its=1800, display=True, alpha=0.2, thresh=0.1) segment = seg * 1.0
def smooth_data(data, mbox=25): mdata = median_filter(data, size=(mbox, mbox)) return data - mdata
def threshold_components_parallel(pars): """ Post-processing of spatial components which includes the following steps (i) Median filtering (ii) Thresholding (iii) Morphological closing of spatial support (iv) Extraction of largest connected component ( to remove small unconnected pixel ) /!\ need to be called through the function threshold components Parameters: --------- [parsed] A: np.ndarray 2d matrix with spatial components dims: tuple dimensions of spatial components medw: [optional] tuple window of median filter thr_method: [optional] string Method of thresholding: 'max' sets to zero pixels that have value less than a fraction of the max value 'nrg' keeps the pixels that contribute up to a specified fraction of the energy maxthr: [optional] scalar Threshold of max value nrgthr: [optional] scalar Threshold of energy extract_cc: [optional] bool Flag to extract connected components (might want to turn to False for dendritic imaging) se: [optional] np.intarray Morphological closing structuring element ss: [optinoal] np.intarray Binary element for determining connectivity Returns: ------- Ath: np.ndarray 2d matrix with spatial components thresholded """ A_i, i, dims, medw, d, thr_method, se, ss, maxthr, nrgthr, extract_cc = pars # we reshape this one dimension column of the 2d components into the 2D that A_temp = np.reshape(A_i, dims[::-1]) # we apply a median filter of size medw A_temp = median_filter(A_temp, medw) if thr_method == 'max': BW = (A_temp > maxthr * np.max(A_temp)) elif thr_method == 'nrg': Asor = np.sort(np.squeeze(np.reshape(A_temp, (d, 1))))[::-1] temp = np.cumsum(Asor**2) ff = np.squeeze(np.where(temp < nrgthr * temp[-1])) if ff.size > 0: ind = ff if ff.ndim == 0 else ff[-1] A_temp[A_temp < Asor[ind]] = 0 BW = (A_temp >= Asor[ind]) else: BW = np.zeros_like(A_temp) # we want to remove the components that are valued 0 in this now 1d matrix Ath = np.squeeze(np.reshape(A_temp, (d, 1))) Ath2 = np.zeros((d)) # we do that to have a full closed structure even if the values have been trehsolded BW = binary_closing(BW.astype(np.int), structure=se) # if we have deleted the element if BW.max() == 0: return Ath2, i # if extract_cc: # we want to extract the largest connected component ( to remove small unconnected pixel ) # we extract each future as independent with the cross structuring elemnt labeled_array, num_features = label(BW, structure=ss) labeled_array = np.squeeze(np.reshape(labeled_array, (d, 1))) nrg = np.zeros((num_features, 1)) # we extract the energy for each component for j in range(num_features): nrg[j] = np.sum(Ath[labeled_array == j + 1]**2) indm = np.argmax(nrg) Ath2[labeled_array == indm + 1] = Ath[labeled_array == indm + 1] else: BW = BW.flatten() Ath2[BW] = Ath[BW] return Ath2, i
def reduce_singlefiber_phase3(config, logtable): """Reduce the single fiber data of Xinglong 2.16m HRS. Args: config (:class:`configparser.ConfigParser`): Config object. logtable (:class:`astropy.table.Table`): Table of observing log. """ # extract keywords from config file section = config['data'] rawpath = section.get('rawpath') statime_key = section.get('statime_key') exptime_key = section.get('exptime_key') direction = section.get('direction') section = config['reduce'] midpath = section.get('midpath') odspath = section.get('odspath') figpath = section.get('figpath') mode = section.get('mode') fig_format = section.get('fig_format') oned_suffix = section.get('oned_suffix') ncores = section.get('ncores') # create folders if not exist if not os.path.exists(figpath): os.mkdir(figpath) if not os.path.exists(odspath): os.mkdir(odspath) if not os.path.exists(midpath): os.mkdir(midpath) # determine number of cores to be used if ncores == 'max': ncores = os.cpu_count() else: ncores = min(os.cpu_count(), int(ncores)) # initialize general card list general_card_lst = {} ### Parse bias bias, bias_card_lst = get_bias(config, logtable) ### define dtype of 1-d spectra if bias is None: ndisp = 4096 else: ncros, ndisp = bias.shape types = [ ('aperture', np.int16), ('order', np.int16), ('points', np.int16), ('wavelength', (np.float64, ndisp)), ('flux', (np.float32, ndisp)), ('error', (np.float32, ndisp)), ('background', (np.float32, ndisp)), ('mask', (np.int16, ndisp)), ] names, formats = list(zip(*types)) spectype = np.dtype({'names': names, 'formats': formats}) ### Combine the flats and trace the orders # fiter flat frames filterfunc = lambda item: item['object'].lower() == 'flat' logitem_lst = list(filter(filterfunc, logtable)) nflat = len(logitem_lst) flat_filename = os.path.join(midpath, 'flat.fits') aperset_filename = os.path.join(midpath, 'trace.trc') aperset_regname = os.path.join(midpath, 'trace.reg') trace_figname = os.path.join(figpath, 'trace.{}'.format(fig_format)) profile_filename = os.path.join(midpath, 'profile.fits') if mode=='debug' and os.path.exists(flat_filename) \ and os.path.exists(aperset_filename) \ and os.path.exists(profile_filename): # read flat data and mask array hdu_lst = fits.open(flat_filename) flat_data = hdu_lst[0].data flat_mask = hdu_lst[1].data flat_norm = hdu_lst[2].data flat_sens = hdu_lst[3].data flat_spec = hdu_lst[4].data exptime = hdu_lst[0].header[exptime_key] hdu_lst.close() aperset = load_aperture_set(aperset_filename) disp_x_lst, profile_x, profile_lst = read_crossprofile( profile_filename) else: data_lst = [] head_lst = [] exptime_lst = [] print('* Combine {} Flat Images: {}'.format(nflat, flat_filename)) fmt_str = ' - {:>7s} {:^11} {:^8s} {:^7} {:^23s} {:^8} {:^6}' head_str = fmt_str.format('frameid', 'FileID', 'Object', 'exptime', 'obsdate', 'N(sat)', 'Q95') for iframe, logitem in enumerate(logitem_lst): # read each individual flat frame fname = '{}.fits'.format(logitem['fileid']) filename = os.path.join(rawpath, fname) data, head = fits.getdata(filename, header=True) exptime_lst.append(head[exptime_key]) mask = get_mask(data, head) sat_mask = (mask & 4 > 0) bad_mask = (mask & 2 > 0) if iframe == 0: allmask = np.zeros_like(mask, dtype=np.int16) allmask += sat_mask # correct overscan for flat data, card_lst = correct_overscan(data, head, logitem['amp']) for key, value in card_lst: head.append((key, value)) # correct bias for flat, if has bias if bias is None: message = 'No bias. skipped bias correction' else: data = data - bias message = 'Bias corrected' logger.info(message) # print info if iframe == 0: print(head_str) message = fmt_str.format('[{:d}]'.format(logitem['frameid']), logitem['fileid'], logitem['object'], logitem['exptime'], logitem['obsdate'], logitem['nsat'], logitem['q95']) print(message) data_lst.append(data) if nflat == 1: flat_data = data_lst[0] else: data_lst = np.array(data_lst) flat_data = combine_images( data_lst, mode='mean', upper_clip=10, maxiter=5, maskmode=(None, 'max')[nflat > 3], ncores=ncores, ) # get mean exposure time and write it to header head = fits.Header() exptime = np.array(exptime_lst).mean() head[exptime_key] = exptime # find saturation mask sat_mask = allmask > nflat / 2. flat_mask = np.int16(sat_mask) * 4 + np.int16(bad_mask) * 2 # get exposure time normalized flats flat_norm = flat_data / exptime # create the trace figure tracefig = TraceFigure() section = config['reduce.trace'] aperset = find_apertures( flat_data, flat_mask, scan_step=section.getint('scan_step'), minimum=section.getfloat('minimum'), separation=section.get('separation'), align_deg=section.getint('align_deg'), filling=section.getfloat('filling'), degree=section.getint('degree'), display=section.getboolean('display'), fig=tracefig, ) aperset.fill(tol=10) # save the trace figure tracefig.adjust_positions() title = 'Trace for {}'.format(flat_filename) tracefig.suptitle(title, fontsize=15) tracefig.savefig(trace_figname) aperset.save_txt(aperset_filename) aperset.save_reg(aperset_regname) # do the flat fielding # prepare the output mid-prococess figures in debug mode if mode == 'debug': figname = 'flat_aperpar_{}_%03d.{}'.format(flatname, fig_format) fig_aperpar = os.path.join(figpath, figname) else: fig_aperpar = None # prepare the name for slit figure figname = 'slit.{}'.format(fig_format) fig_slit = os.path.join(figpath, figname) # prepare the name for slit file fname = 'slit.dat' slit_file = os.path.join(midpath, fname) section = config['reduce.flat'] p1, p2, pstep = -8, 8, 0.1 profile_x = np.arange(p1, p2 + 1e-4, pstep) disp_x_lst = np.arange(48, ndisp, 500) fig_spatial = SpatialProfileFigure() flat_sens, flatspec_lst, profile_lst = get_flat( data=flat_data, mask=flat_mask, apertureset=aperset, nflat=nflat, q_threshold=section.getfloat('q_threshold'), smooth_A_func=smooth_aperpar_A, smooth_c_func=smooth_aperpar_c, smooth_bkg_func=smooth_aperpar_bkg, mode='debug', fig_spatial=fig_spatial, flatname='flat_normal', profile_x=profile_x, disp_x_lst=disp_x_lst, ) figname = os.path.join(figpath, 'spatial_profile.png') title = 'Spatial Profile of Flat' fig_spatial.suptitle(title) fig_spatial.savefig(figname) fig_spatial.close() # pack 1-d spectra of flat flat_spec = [] for aper, flatspec in sorted(flatspec_lst.items()): n = flatspec.size row = ( aper, 0, n, np.zeros(n, dtype=np.float64), # wavelength flatspec, # flux np.zeros(n, dtype=np.float32), # error np.zeros(n, dtype=np.float32), # background np.zeros(n, dtype=np.int16), # mask ) flat_spec.append(row) flat_spec = np.array(flat_spec, dtype=spectype) # save cross-profiles save_crossprofile(profile_filename, disp_x_lst, p1, p2, pstep, profile_lst) # pack results and save to fits hdu_lst = fits.HDUList([ fits.PrimaryHDU(flat_data, head), fits.ImageHDU(flat_mask), fits.ImageHDU(flat_norm), fits.ImageHDU(flat_sens), fits.BinTableHDU(flat_spec), ]) hdu_lst.writeto(flat_filename, overwrite=True) ############################## Extract ThAr ################################ # get the data shape ny, nx = flat_sens.shape calib_lst = {} # filter ThAr frames filter_thar = lambda item: item['object'].lower() == 'thar' thar_items = list(filter(filter_thar, logtable)) for ithar, logitem in enumerate(thar_items): # logitem alias frameid = logitem['frameid'] fileid = logitem['fileid'] objname = logitem['object'] imgtype = logitem['imgtype'] exptime = logitem['exptime'] amp = logitem['amp'] # prepare message prefix logger_prefix = 'FileID: {} - '.format(fileid) screen_prefix = ' - ' fmt_str = 'FileID: {} ({}) OBJECT: {} - wavelength identification' message = fmt_str.format(fileid, imgtype, objname) logger.info(message) print(message) fname = '{}.fits'.format(fileid) filename = os.path.join(rawpath, fname) data, head = fits.getdata(filename, header=True) mask = get_mask(data, head) # correct overscan for ThAr data, card_lst = correct_overscan(data, head, amp) for key, value in card_lst: head.append((key, value)) message = 'Overscan corrected.' logger.info(logger_prefix + message) print(screen_prefix + message) # correct bias for ThAr, if has bias if bias is None: message = 'No bias' else: data = data - bias message = 'Bias corrected. Mean = {:.2f}'.format(bias.mean()) logger.info(logger_prefix + message) print(screen_prefix + message) head.append(('HIERARCH GAMSE BACKGROUND CORRECTED', False)) # extract ThAr spectra lower_limit = 7 upper_limit = 7 spectra1d = extract_aperset( data, mask, apertureset=aperset, lower_limit=lower_limit, upper_limit=upper_limit, ) head = aperset.to_fitsheader(head) message = '1D spectra extracted for {:d} orders'.format(len(spectra1d)) logger.info(logger_prefix + message) print(screen_prefix + message) # pack to a structured array spec = [] for aper, item in sorted(spectra1d.items()): flux_sum = item['flux_sum'] n = flux_sum.size # pack to table row = ( aper, 0, n, np.zeros(n, dtype=np.float64), # wavelength flux_sum, # flux np.zeros(n, dtype=np.float32), # error np.zeros(n, dtype=np.float32), # background np.zeros(n, dtype=np.int16), # mask ) spec.append(row) spec = np.array(spec, dtype=spectype) figname = 'wlcalib_{}.{}'.format(fileid, fig_format) wlcalib_fig = os.path.join(figpath, figname) section = config['reduce.wlcalib'] title = '{}.fits'.format(fileid) if ithar == 0: # this is the first ThAr frame in this observing run if section.getboolean('search_database'): # find previouse calibration results index_file = os.path.join( os.path.dirname(__file__), '../../data/calib/wlcalib_xinglong216hrs.dat') message = ('Searching for archive wavelength calibration' 'file in "{}"'.format(os.path.basename(index_file))) logger.info(logger_prefix + message) print(screen_prefix + message) ref_spec, ref_calib = select_calib_from_database( index_file, head[statime_key]) if ref_spec is None or ref_calib is None: message = ('Did not find nay archive wavelength' 'calibration file') logger.info(logger_prefix + message) print(screen_prefix + message) # if failed, pop up a calibration window and identify # the wavelengths manually calib = wlcalib( spec, figfilename=wlcalib_fig, title=title, linelist=section.get('linelist'), window_size=section.getint('window_size'), xorder=section.getint('xorder'), yorder=section.getint('yorder'), maxiter=section.getint('maxiter'), clipping=section.getfloat('clipping'), q_threshold=section.getfloat('q_threshold'), ) else: # if success, run recalib # determien the direction message = 'Found archive wavelength calibration file' logger.info(message) print(screen_prefix + message) ref_direction = ref_calib['direction'] if direction[1] == '?': aperture_k = None elif direction[1] == ref_direction[1]: aperture_k = 1 else: aperture_k = -1 if direction[2] == '?': pixel_k = None elif direction[2] == ref_direction[2]: pixel_k = 1 else: pixel_k = -1 result = find_caliblamp_offset( ref_spec, spec, aperture_k=aperture_k, pixel_k=pixel_k, pixel_range=(-50, 50), mode=mode, ) aperture_koffset = (result[0], result[1]) pixel_koffset = (result[2], result[3]) message = 'Aperture offset = {}; Pixel offset = {}' message = message.format(aperture_koffset, pixel_koffset) logger.info(logger_prefix + message) print(screen_prefix + message) use = section.getboolean('use_prev_fitpar') xorder = (section.getint('xorder'), None)[use] yorder = (section.getint('yorder'), None)[use] maxiter = (section.getint('maxiter'), None)[use] clipping = (section.getfloat('clipping'), None)[use] window_size = (section.getint('window_size'), None)[use] q_threshold = (section.getfloat('q_threshold'), None)[use] calib = recalib( spec, figfilename=wlcalib_fig, title=title, ref_spec=ref_spec, linelist=section.get('linelist'), aperture_koffset=aperture_koffset, pixel_koffset=pixel_koffset, ref_calib=ref_calib, xorder=xorder, yorder=yorder, maxiter=maxiter, clipping=clipping, window_size=window_size, q_threshold=q_threshold, direction=direction, ) else: message = 'No database searching. Identify lines manually' logger.info(logger_prefix + message) print(screen_prefix + message) # do not search the database calib = wlcalib( spec, figfilename=wlcalib_fig, title=title, identfilename=section.get('ident_file', None), linelist=section.get('linelist'), window_size=section.getint('window_size'), xorder=section.getint('xorder'), yorder=section.getint('yorder'), maxiter=section.getint('maxiter'), clipping=section.getfloat('clipping'), q_threshold=section.getfloat('q_threshold'), ) message = ('Wavelength calibration finished.' '(k, offset) = ({}, {})'.format( calib['k'], calib['offset'])) logger.info(logger_prefix + message) # then use this thar as reference ref_calib = calib ref_spec = spec message = 'Reference calib and spec are selected' logger.info(logger_prefix + message) else: message = 'Use reference calib and spec' logger.info(logger_prefix + message) # for other ThArs, no aperture offset calib = recalib( spec, figfilename=wlcalib_fig, title=title, ref_spec=ref_spec, linelist=section.get('linelist'), ref_calib=ref_calib, aperture_koffset=(1, 0), pixel_koffset=(1, 0), xorder=ref_calib['xorder'], yorder=ref_calib['yorder'], maxiter=ref_calib['maxiter'], clipping=ref_calib['clipping'], window_size=ref_calib['window_size'], q_threshold=ref_calib['q_threshold'], direction=direction, ) # add more infos in calib calib['fileid'] = fileid calib['date-obs'] = head[statime_key] calib['exptime'] = head[exptime_key] message = 'Add more info in calib of {}'.format(fileid) logger.info(logger_prefix + message) # reference the ThAr spectra spec, card_lst, identlist = reference_self_wavelength(spec, calib) message = 'Wavelength solution added' logger.info(logger_prefix + message) prefix = 'HIERARCH GAMSE WLCALIB ' for key, value in card_lst: head.append((prefix + key, value)) hdu_lst = fits.HDUList([ fits.PrimaryHDU(header=head), fits.BinTableHDU(spec), fits.BinTableHDU(identlist), ]) # save in midproc path as a wlcalib reference file fname = 'wlcalib_{}.fits'.format(fileid) filename = os.path.join(midpath, fname) hdu_lst.writeto(filename, overwrite=True) message = 'Wavelength calibrated spectra written to {}'.format( filename) logger.info(logger_prefix + message) # save in onedspec path fname = '{}_{}.fits'.format(fileid, oned_suffix) filename = os.path.join(odspath, fname) hdu_lst.writeto(filename, overwrite=True) message = 'Wavelength calibrated spectra written to {}'.format( filename) logger.info(logger_prefix + message) # pack to calib_lst calib_lst[frameid] = calib # print fitting summary fmt_string = ' [{:3d}] {} - ({:4g} sec) - {:4d}/{:4d} RMS = {:7.5f}' section = config['reduce.wlcalib'] auto_selection = section.getboolean('auto_selection') if auto_selection: rms_threshold = section.getfloat('rms_threshold', 0.005) group_contiguous = section.getboolean('group_contiguous', True) time_diff = section.getfloat('time_diff', 120) ref_calib_lst = select_calib_auto( calib_lst, rms_threshold=rms_threshold, group_contiguous=group_contiguous, time_diff=time_diff, ) ref_fileid_lst = [calib['fileid'] for calib in ref_calib_lst] # print ThAr summary and selected calib for frameid, calib in sorted(calib_lst.items()): string = fmt_string.format(frameid, calib['fileid'], calib['exptime'], calib['nuse'], calib['ntot'], calib['std']) if calib['fileid'] in ref_fileid_lst: string = '\033[91m{} [selected]\033[0m'.format(string) print(string) else: # print the fitting summary for frameid, calib in sorted(calib_lst.items()): string = fmt_string.format(frameid, calib['fileid'], calib['exptime'], calib['nuse'], calib['ntot'], calib['std']) print(string) promotion = 'Select References: ' ref_calib_lst = select_calib_manu(calib_lst, promotion=promotion) ########### Extract Science Spectrum ########## # filter science items in logtable #extr_filter = config['reduce.extract'].get('extract', # 'lambda row: row["imgtype"]=="sci"') #extr_filter = eval(extr_filter) extr_filter = lambda row: row['imgtype'] == 'sci' extr_items = list(filter(extr_filter, logtable)) for logitem in extr_items: # logitem alias frameid = logitem['frameid'] fileid = logitem['fileid'] imgtype = logitem['imgtype'] objname = logitem['object'] exptime = logitem['exptime'] amp = logitem['amp'] # prepare message prefix logger_prefix = 'FileID: {} - '.format(fileid) screen_prefix = ' - ' filename = os.path.join(rawpath, '{}.fits'.format(fileid)) message = 'FileID: {} ({}) OBJECT: {}'.format(fileid, imgtype, objname) logger.info(message) print(message) # read raw data data, head = fits.getdata(filename, header=True) mask = get_mask(data, head) # correct overscan data, card_lst = correct_overscan(data, head, amp) for key, value in card_lst: head.append((key, value)) message = 'Overscan corrected.' logger.info(logger_prefix + message) print(screen_prefix + message) # correct bias if bias is None: message = 'No bias' else: data = data - bias message = 'Bias corrected. Mean = {:.2f}'.format(bias.mean()) logger.info(logger_prefix + message) print(screen_prefix + message) # correct flat data = data / flat_sens message = 'Flat field corrected.' logger.info(logger_prefix + message) print(screen_prefix + message) ny, nx = data.shape allx = np.arange(nx) # get background lights background = get_interorder_background(data, mask, aperset) for y in np.arange(ny): m = mask[y, :] == 0 f = intp.InterpolatedUnivariateSpline(allx[m], background[y, :][m], k=3) background[y, :][~m] = f(allx[~m]) background = median_filter(background, size=(9, 5), mode='nearest') background = savitzky_golay_2d(background, window_length=(21, 101), order=3, mode='nearest') # plot stray light figname = 'bkg2d_{}.{}'.format(fileid, fig_format) figfilename = os.path.join(figpath, figname) fig_bkg = BackgroundFigure( data, background, title='Background Correction for {}'.format(fileid), figname=figfilename, ) fig_bkg.close() data = data - background message = 'Background corrected. Max = {:.2f}; Mean = {:.2f}'.format( background.max(), background.mean()) logger.info(logger_prefix + message) print(screen_prefix + message) # extract 1d spectrum section = config['reduce.extract'] method = section.get('method') if method == 'optimal': result = extract_aperset_optimal( data, mask, background=background, apertureset=aperset, gain=1.02, ron=3.29, profilex=profile_x, disp_x_lst=disp_x_lst, main_disp='x', profile_lst=profile_lst, ) flux_opt_lst = result[0] flux_err_lst = result[1] back_opt_lst = result[2] flux_sum_lst = result[3] back_sum_lst = result[4] # pack spectrum spec = [] for aper in sorted(flux_opt_lst.keys()): n = flux_opt_lst[aper].size row = ( aper, 0, n, np.zeros(n, dtype=np.float64), # wavelength flux_opt_lst[aper], # flux flux_err_lst[aper], # error back_opt_lst[aper], # background np.zeros(n, dtype=np.int16), # mask ) spec.append(row) spec = np.array(spec, dtype=spectype) elif method == 'sum': lower_limit = section.getfloat('lower_limit') upper_limit = section.getfloat('upper_limit') # extract 1d spectra of the object spectra1d = extract_aperset( data, mask, apertureset=aperset, lower_limit=lower_limit, upper_limit=upper_limit, ) norder = len(spectra1d) message = '1D spectra of {} orders extracted'.format(norder) logger.info(logger_prefix + message) print(screen_prefix + message) # extract 1d spectra for straylight/background light background1d = extract_aperset( background, mask, apertureset=aperset, lower_limit=lower_limit, upper_limit=upper_limit, ) message = '1D straylight of {} orders extracted'.format( len(background1d)) logger.info(logger_prefix + message) print(screen_prefix + message) prefix = 'HIERARCH GAMSE EXTRACTION ' head.append((prefix + 'LOWER LIMIT', lower_limit)) head.append((prefix + 'UPPER LIMIT', upper_limit)) # pack spectrum spec = [] for aper, item in sorted(spectra1d.items()): flux_sum = item['flux_sum'] n = flux_sum.size # background 1d flux back_flux = background1d[aper]['flux_sum'] row = ( aper, 0, n, np.zeros(n, dtype=np.float64), # wavelength flux_sum, # flux np.zeros(n, dtype=np.float32), # error back_flux, # background np.zeros(n, dtype=np.int16), # mask ) spec.append(row) spec = np.array(spec, dtype=spectype) # wavelength calibration weight_lst = get_calib_weight_lst( ref_calib_lst, obsdate=head[statime_key], exptime=head[exptime_key], ) message_lst = ['Wavelength calibration:'] for i, calib in enumerate(ref_calib_lst): string = ' ' * len(screen_prefix) string = string + '{} ({:4g} sec) {} weight = {:5.3f}'.format( calib['fileid'], calib['exptime'], calib['date-obs'], weight_lst[i]) message_lst.append(string) message = os.linesep.join(message_lst) logger.info(logger_prefix + message) print(screen_prefix + message) spec, card_lst = reference_spec_wavelength(spec, ref_calib_lst, weight_lst) prefix = 'HIERARCH GAMSE WLCALIB ' for key, value in card_lst: head.append((prefix + key, value)) # pack and save wavelength referenced spectra hdu_lst = fits.HDUList([ fits.PrimaryHDU(header=head), fits.BinTableHDU(spec), ]) fname = '{}_{}.fits'.format(fileid, oned_suffix) filename = os.path.join(odspath, fname) hdu_lst.writeto(filename, overwrite=True) message = '1D spectra written to "{}"'.format(filename) logger.info(logger_prefix + message) print(screen_prefix + message)
def vectorizeRaster(infile, outfile, classes, classfile, weight, nodata, smoothing, band, cartoCSS, axonometrize, nosimple, setNoData, nibbleMask, outvar): band = int(band) src = gdal.Open(infile) bandData = src.GetRasterBand(band) inarr = bandData.ReadAsArray() if (inarr is None) or (len(inarr) == 0): gdal.SetConfigOption('GDAL_NETCDF_BOTTOMUP', 'NO') src = gdal.Open(infile) bandData = src.GetRasterBand(band) inarr = bandData.ReadAsArray() oshape = np.shape(inarr) if len(src.GetProjectionRef()) > 0: new_cs = osr.SpatialReference() new_cs.ImportFromEPSG(4326) old_cs = osr.SpatialReference() old_cs.ImportFromWkt(src.GetProjectionRef()) transform = osr.CoordinateTransformation(old_cs, new_cs) oaff = Affine.from_gdal(*src.GetGeoTransform()) bbox = src.GetGeoTransform() nodata = None if type(bandData.GetNoDataValue()) == float: nodata = bandData.GetNoDataValue() if (type(setNoData) == int or type(setNoData) == float) and hasattr( inarr, 'mask'): inarr[np.where(inarr.mask == True)] = setNoData nodata = True nlat, nlon = np.shape(inarr) dataY = np.arange(nlat) * bbox[5] + bbox[3] dataX = np.arange(nlon) * bbox[1] + bbox[0] if len(src.GetProjectionRef()) > 0: ul = transform.TransformPoint(min(dataX), max(dataY)) ll = transform.TransformPoint(min(dataX), min(dataY)) ur = transform.TransformPoint(max(dataX), max(dataY)) lr = transform.TransformPoint(max(dataX), min(dataY)) simplestY1 = (abs(ul[1] - ll[1]) / float(oshape[0])) simplestY2 = (abs(ur[1] - lr[1]) / float(oshape[0])) simplestX1 = (abs(ur[0] - ul[0]) / float(oshape[1])) simplestX2 = (abs(lr[0] - ll[0]) / float(oshape[1])) simplest = 2 * max(simplestX1, simplestY1, simplestX2, simplestY2) else: simplestY = ((max(dataY) - min(dataY)) / float(oshape[0])) simplestX = ((max(dataX) - min(dataX)) / float(oshape[1])) simplest = 2 * max(simplestX, simplestY) if nodata == 'min': maskArr = np.zeros(inarr.shape, dtype=np.bool) maskArr[np.where(inarr == inarr.min())] = True inarr = np.ma.array(inarr, mask=maskArr) del maskArr elif type(nodata) == int or type(nodata) == float: maskArr = np.zeros(inarr.shape, dtype=np.bool) maskArr[np.where(inarr == nodata)] = True inarr[np.where(inarr == nodata)] = np.nan inarr = np.ma.array(inarr, mask=maskArr) del maskArr elif nodata == None or np.isnan(nodata) or nodata: maskArr = np.zeros(inarr.shape, dtype=np.bool) inarr = np.ma.array(inarr, mask=maskArr) del maskArr elif (type(nodata) == int or type(nodata) == float) and hasattr( inarr, 'mask'): nodata = True if nibbleMask: inarr.mask = maximum_filter(inarr.mask, size=3) if smoothing and smoothing > 1: inarr, oaff = zoomSmooth(inarr, smoothing, oaff) else: smoothing = 1 with open(classfile, 'r') as ofile: classifiers = ofile.read().split(',') classRas, breaks = classifyManual( inarr, np.array(classifiers).astype(inarr.dtype)) # filtering for speckling classRas = median_filter(classRas, size=2) # print out cartocss for classes if cartoCSS: for i in breaks: click.echo('[value = ' + str(breaks[i]) + '] { polygon-fill: @class' + str(i) + '}') if outfile: outputHandler = tools.dataOutput(True) else: outputHandler = tools.dataOutput() #polys = [] #vals = [] for i, br in enumerate(breaks): if i == 0: continue tRas = (classRas == i).astype(np.uint8) if nodata: tRas[np.where(classRas == 0)] = 0 for feature, shapes in features.shapes(np.asarray(tRas, order='C'), transform=oaff): if shapes == 1: featurelist = [] for c, f in enumerate(feature['coordinates']): if len(src.GetProjectionRef()) > 0: for ix in range(len(f)): px = transform.TransformPoint(f[ix][0], f[ix][1]) lst = list() lst.append(px[0]) lst.append(px[1]) f[ix] = tuple(lst) if len(f) > 3 or c == 0: if axonometrize: f = np.array(f) f[:, 1] += (axonometrize * br) if nosimple: poly = Polygon(f) else: poly = Polygon(f).simplify(simplest / float(smoothing), preserve_topology=True) if c == 0: poly = polygon.orient(poly, sign=-1.0) else: poly = polygon.orient(poly, sign=1.0) featurelist.append(poly) if len(featurelist) != 0: #polys.append(MultiPolygon(featurelist)) #vals.append(breaks[br]) outputHandler.out({ 'type': 'Feature', 'geometry': mapping(MultiPolygon(featurelist)), 'properties': { outvar: breaks[br] } }) #for pa in range(0,len(polys)): # for pb in range(0,len(polys)): # if pa==pb: # continue # if polys[pa].contains(polys[pb]) & (polys[pa].area>polys[pb].area): # try: # polys[pa] = polys[pa].difference(polys[pb]) # print polys[pa].area # print '---' # break # except: # a = 1 # #for pc in range(0,len(polys)): # outputHandler.out({ # 'type': 'Feature', # 'geometry': mapping(polys[pc]), # 'properties': { # outvar: vals[pc] # } # }) if outfile: with open(outfile, 'w') as ofile: ofile.write( json.dumps({ "type": "FeatureCollection", "features": outputHandler.data }))
def main(argv): generate_output = True selected_channel = 1 # 0 or 1: 2017-09-25 sample has flipped channels input_h5_file = '/groups/mousebrainmicro/mousebrainmicro/users/base/AnnotationData/h5repo/2017-09-25_G-003_Consensus/2017-09-25_G-003_Consensus-carved.h5' output_folder = '/groups/mousebrainmicro/mousebrainmicro/users/base/AnnotationData/h5repo/2017-09-25_G-003_Consensus/' try: opts, args = getopt.getopt(argv, "hi:o:", ["input_h5_file=", "output_folder="]) except getopt.GetoptError: print('segmentAxon.py -i <input_h5_file> -o <output_folder>') sys.exit(2) for opt, arg in opts: print('opt:', opt, 'arg:', arg) if opt == '-h': print('segmentAxon.py -i <input_h5_file> -o <output_folder>') sys.exit() elif opt in ("-i", "--input_h5_file"): input_h5_file = arg print('SWCFILE :', input_h5_file) elif opt in ("-o", "--output_folder"): output_folder = arg print('OUTPUT :', output_folder) # swc_name = '2017-11-17_G-017_Seg-1' # data_folder = os.path.join('/groups/mousebrainmicro/home/base/CODE/MOUSELIGHT/navigator/data/',swc_name) inputfolder, h5_name_w_ext = os.path.split(input_h5_file) file_name, _ = h5_name_w_ext.split(os.extsep) segmentation_output_h5_file = os.path.join(output_folder, file_name + '_segmented.h5') swc_output_file = os.path.join(output_folder, file_name + '_segmented.swc') if not os.path.exists(output_folder): os.makedirs(output_folder) # AC_output_tif_file = os.path.join(data_folder,swc_name+'_AC_cropped_segmented.tif') # Frangi_output_tif_file = os.path.join(data_folder,swc_name+'_Frangi_cropped_segmented.tif') # TODO export scale/filt if needed # dset_filter_Frangi_magnitude = f_out.create_dataset("/filter/Frangi/magnitude", volume.shape[:3], dtype='f', chunks=f["volume"].chunks[:3], compression="gzip", compression_opts=9) # dset_filter_Frangi_scale = f_out.create_dataset("/filter/Frangi/scale", volume.shape[:3], dtype='f', chunks=f["volume"].chunks[:3], compression="gzip", compression_opts=9) # for each branch, crop a box, run segmentation based on: # 1) frangi vesselness filter # 2) active countours # 3) stat thresholding: TODO: diffusion is buggy, might be better to switch to a regularized version # figure out signal channel # pattern_strings = ['\xc2d', '\xa0', '\xe7', '\xc3\ufffdd', '\xc2\xa0', '\xc3\xa7', '\xa0\xa0', '\xc2', '\xe9'] if selected_channel == None: pattern_strings = ['_G-', '_G_'] pattern_string = '|'.join(pattern_strings) pattern = re.compile(pattern_string) if re.search(pattern, os.path.split(inputfolder)[1]): # green channel ch = 0 else: ch = 1 else: ch = selected_channel with h5py.File(input_h5_file, "r") as f: recon = f["reconstruction"] volume = f["volume"] output_dims = volume.shape if generate_output: f_out = h5py.File(segmentation_output_h5_file, "w") dset_segmentation_AC = f_out.create_dataset( "/segmentation/AC", volume.shape[:3], dtype='uint8', chunks=f["volume"].chunks[:3], compression="gzip", compression_opts=9) dset_segmentation_Frangi = f_out.create_dataset( "/segmentation/Frangi", volume.shape[:3], dtype='uint8', chunks=f["volume"].chunks[:3], compression="gzip", compression_opts=9) dset_swc_Frangi = f_out.create_dataset("/reconstruction/sparse", data=recon[:], dtype='f') lookup_data = {} # keeps track of indicies, subs and radius # sigmas = np.array([0.75, 1.0, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]) sigmas = np.array([0.75, 1.0, 1.5, 2, 2.5]) linkdata = [] window_size = 3 radius_list = func.getRadiusIndicies(radius=sigmas) for ix, txt in enumerate(recon[:100, :]): # ix = np.argmin(np.sum(np.abs(np.array([757.3, 2327.5, 575.0]) - recon[:, 2:5]), axis=1)) print('{} out of {}'.format(ix, recon.shape[0])) start_node = ix # recon[ix,0] end_node = recon[ix, 6] - 1 # 0 based indexing if end_node < 0: continue start = np.asarray(recon[start_node, 2:5], np.int) end = np.asarray(recon[end_node, 2:5], np.int) # vesselness convolution sigmas # padding needs to be at least window_size/2*sigma padding = np.ceil(np.max(window_size * sigmas / 2)).__int__() bbox_min_wo = np.min((start, end), axis=0) bbox_max_wo = np.max((start, end), axis=0) bbox_min = np.min((start, end), axis=0) - padding bbox_max = np.max( (start, end), axis=0) + padding + 1 # add one to make python inclusive bbox_size = bbox_max - bbox_min start_ = start - bbox_min end_ = end - bbox_min roi = ((padding), ()) # crop crop = volume[bbox_min[0]:bbox_max[0], bbox_min[1]:bbox_max[1], bbox_min[2]:bbox_max[2]] crop[crop == 0] = np.min(crop[ crop > 0]) # overwrites any missing voxel with patch minima ## fix line shifts # st = -9;en = 10;shift, shift_float = lnf.findShift(inputim[:,:inputim.shape[1]//2*2,:], st, en, False) ## inputim = np.log(np.asarray( crop[:, :, :, 1], np.float)) # add 1 to prevent /0 cases for log scaling inputim = (inputim - inputim.min()) / (inputim.max() - inputim.min()) ## FRANGI filtresponse, scaleresponse = frangi.frangi( inputim, sigmas, window_size=window_size, alpha=0.01, beta=1.5, frangi_c=2 * np.std(inputim), black_vessels=False) # min filter to to local noise supression to tune to local center scaleresponse = minimum_filter(scaleresponse, 3) # sitk.Show(sitk.GetImageFromArray(np.swapaxes(inputim,2,0))) # sitk.Show(sitk.GetImageFromArray(np.swapaxes(filtresponse/np.max(filtresponse),2,0))) # sitk.Show(sitk.GetImageFromArray(np.swapaxes(scaleresponse,2,0))) # cost: high intensity low scale # filter out scale response for local radius variations: cost_array = scaleresponse / (0.001 + filtresponse) cost_array[cost_array > 100] = 100 # TODO: smooth/prune path: if s[i-1] has +/-1 access to s[i+1], delete s[i], this is to prevent triangular extensions, i.e. |\ or |/ # shortest path based on cost_array path, cost = skig.route_through_array(cost_array, start=start_, end=end_, fully_connected=True) path_array = np.asarray(path) # sample along path path_array_indicies = np.ravel_multi_index(path_array.T, cost_array.shape) xyz_trace_locations = path_array + bbox_min # index ids for each location inds = np.ravel_multi_index(xyz_trace_locations.T, output_dims[:3]) # plt.figure() # plt.imshow(np.max(filtresponse**.05,axis=2).T) # plt.plot(path_array[:,0],path_array[:,1]) # branch data based on Frangi # frangi radius estimate around tracing radius_estimate_around_trace = scaleresponse.flat[ path_array_indicies] # filter radius to smooth radius_estimate_around_trace = median_filter( radius_estimate_around_trace, 3) # radius as 4th column branch_data = np.concatenate( (xyz_trace_locations, radius_estimate_around_trace[:, None], inds[:, None]), axis=1) # store recon info linkdata.append(branch_data) if generate_output: # paint functions # paint hdf5: for each trace location, generate a ball with the given radius and paint into segmentation output for xyzr in branch_data: xyz = xyzr[:3] r = xyzr[3] # search the nearest key mask = radius_list[sigmas[np.argmin((sigmas - r)**2)]] paintlocs = np.where(mask) - np.floor(r) + xyz[:, None] for locs in paintlocs.transpose(): dset_segmentation_Frangi[tuple(locs)] = 1 ## segmentation based on active contours if 0: segment = seg.volumeSeg(filtresponse, path_array) # working else: # use cost function to initialize segmentation segment = seg.volumeSeg( inputim, path_array, cost_array=np.max(cost_array) - cost_array) # revert cost for positive active contour segment.runSeg() # TODO: ability to export swc for AC radius_estimate_around_trace_AC = segment.estimateRad() # sitk.Show(sitk.GetImageFromArray(np.swapaxes(segment.mask_ActiveContour,2,0))) # sitk.Show(sitk.GetImageFromArray(np.swapaxes(inputim,2,0))) # sitk.Show(sitk.GetImageFromArray(np.swapaxes(filtresponse/np.max(filtresponse),2,0))) # patch wise write is buggy, so location wise painting paintlocs = bbox_min[:, None] + np.where( segment.mask_ActiveContour) for locs in paintlocs.transpose(): dset_segmentation_AC[tuple(locs)] = 1 for ii, ind in enumerate(inds): lookup_data[ind] = branch_data[ii, :4] swc_data = np.array(func.link2pred(linkdata, lookup_data)) func.array2swc(swcfile=swc_output_file, swcdata=swc_data) if generate_output: # dump dense reconstruction f_out.create_dataset("/reconstruction/dense", data=swc_data, dtype='f') # close output data f_out.close()
def median_filter(X, M=8): """Median filter along the first axis of the feature matrix X.""" for i in xrange(X.shape[1]): X[:, i] = filters.median_filter(X[:, i], size=M) return X
print "Took ", time.time() - t , " secs" t = time.time() print Xdev.shape print Ydev.shape print X.shape print Y.shape Y_hat = regression.ann(Xdev, Ydev, X, Y, K=5) print "Took ", time.time() - t , " secs" #plt.figure();plt.imshow(Ktest_dev); #plt.colorbar() #plt.show() # optionnal step: median filtering for smoothing the data: Y_hat = median_filter(Y_hat,(1,10)) #plt.figure() #plt.subplot(211) #plt.imshow(np.log(Y), # origin='lower') #plt.colorbar() #plt.title('Original') #plt.subplot(212) #plt.imshow(np.log(Y_hat), # origin='lower') #plt.colorbar() #plt.title('Estimation from Nadaraya-Watson') #plt.show() sig_orig = Signal(test_audiofilepath, normalize=True, mono=True)