def wrapped_medfilt(x, y, ks=3, axis='y', **kwargs): """Use scipy.signal.medfilt to filter either the x or y data. Also loop the filter around to prevent edge effects. If the data forms a closed loop the medfilt should account for this, otherwise there will be artifacts introduced in points near (less than ks-1 / 2) the edge of the data. This version of medfilt accounts for that by prepending the last ks data points and appending the first data points, running medfilt, and then removing the pre/appended points. Args: ks: and odd number that represents the width of the filter. See medfilt for more detail. axis: either 'x' or 'y'. Indicates which axis medfilt should be called on. """ from scipy.signal import medfilt _verify_axis(axis) x = np.concatenate((x[-ks:], x, x[:ks])) y = np.concatenate((y[-ks:], y, y[:ks])) if axis == 'x': x = medfilt(x, ks) elif axis == 'y': y = medfilt(y, ks) return x[ks:-ks], y[ks:-ks]
def median_filter(self,size, extend=False): """ do a standard median filtering. give size in Angstroms, will be translated to nearest odd number of pixels. """ #check if wavelength calibration is linear: if (self.l[1]-self.l[0])==(self.l[-1]-self.l[-2]): linear=True else: linear=False l=self.l self.linearize() step=self.l[1]-self.l[0] add_dim=int(np.ceil(size/step // 2 * 2 + 1)) if extend==True: f=np.insert(self.f,0,np.ones(add_dim)*self.f[0]) f=np.append(f,np.ones(add_dim)*self.f[-1]) fe=np.insert(self.fe,0,np.ones(add_dim)*self.fe[0]) fe=np.append(fe,np.ones(add_dim)*self.fe[-1]) self.f=signal.medfilt(f,add_dim)[add_dim:-add_dim] self.fe=signal.medfilt(fe,add_dim,)[add_dim:-add_dim] self.fe=self.fe/np.sqrt(add_dim) else: self.f=signal.medfilt(self.f,add_dim) self.fe=signal.medfilt(self.fe,add_dim) self.fe=self.fe/np.sqrt(add_dim) if linear==False: self.interpolate(l)
def segment_digits(img): # la binariza en caso de que sea escala de grises if not img.dtype == 'bool': img = img > 0 min_size = 32 medfilt_k = 5 img0 = morphology.remove_small_objects(img, min_size=min_size) sum1 = np.sum(img0, 1) sum1 = signal.medfilt(sum1, medfilt_k) # Suavizado del perfil acumulado bp1 = sum1 > 0 # Obtener coordenada en y de los puntos inicio y fin de los dígitos (asumiendo una sola línea de dígitos) idx_top = [i for i in range(len(bp1)) if bp1[i]>0] idx_bottom = [len(bp1)-i+1 for i in range(len(bp1)) if bp1[len(bp1)-i-1]>0] if len(idx_top) > 0 and len(idx_bottom) > 0: bp1[idx_top[0]:idx_bottom[0]+1] = True sum0 = np.sum(img0, 0) sum0 = signal.medfilt(sum0, medfilt_k) bp0 = sum0 > 0 # Obtener coordenada en x de los puntos inicio y fin de los dígitos idx_01_transition = [i for i in range(1, len(bp0)) if bp0[i-1]==False and bp0[i]==True] idx_10_transition = [i for i in range(len(bp0)-1) if bp0[i]==True and bp0[i+1]==False] bb=[] if len(idx_01_transition)==len(idx_10_transition): for i in range(len(idx_01_transition)): bb.append([idx_01_transition[i], idx_top[0], idx_10_transition[i], idx_bottom[0]]) return bb
def interpolate(self): pitch = np.zeros((self.nframes)) pitch[:] = self.samp_values pitch2 = medfilt(self.samp_values, self.SMOOTH_FACTOR) # This part in the original code is kind of confused and caused # some problems with the extrapolated points before the first # voiced frame and after the last voiced frame. So, I made some # small modifications in order to make it work better. edges = self.edges_finder(pitch) first_sample = pitch[0] last_sample = pitch[-1] if len(np.nonzero(pitch2)[0]) < 2: pitch[pitch == 0] = self.PTCH_TYP else: nz_pitch = pitch2[pitch2 > 0] pitch2 = scipy_interp.pchip(np.nonzero(pitch2)[0], nz_pitch)(range(self.nframes)) pitch[pitch == 0] = pitch2[pitch == 0] if self.SMOOTH > 0: pitch = medfilt(pitch, self.SMOOTH_FACTOR) try: if first_sample == 0: pitch[:edges[0]-1] = pitch[edges[0]] if last_sample == 0: pitch[edges[-1]+1:] = pitch[edges[-1]] except: pass self.samp_interp = pitch
def FindSlices(label): ''' Locate likely isolated slices in each axis ''' # Integral over volume ii = float(label.ravel().sum()) # Project onto each axis Px = np.sum(np.sum(label,axis=2), axis=1) / ii Py = np.sum(np.sum(label,axis=2), axis=0) / ii Pz = np.sum(np.sum(label,axis=1), axis=0) / ii # Subtract median filtered baseline estimate (k = 3) # Retains only spikes Dx = Px - medfilt(Px) Dy = Py - medfilt(Py) Dz = Pz - medfilt(Pz) # Locate spikes in residual Dmin = 0.01 Sx = np.where(Dx > Dmin) Sy = np.where(Dy > Dmin) Sz = np.where(Dz > Dmin) return Sx, Sy, Sz
def get_spectrogram(self, sig): sig = sig.astype(np.double) # DC if self.remove_dc: sig = self.dc_filter(sig) # median filter if self.medfilt_t > 0: sig = ss.medfilt(sig, self.medfilt_t) # preemph spectral tilt if self.pre_emph > 0: sig = self.pre_emphasis(sig) # gain normalize: sig = sig / np.abs(sig).max() # fft frames = self.stft(sig) # power spectrum frames = frames.real**2 / self.nfft # median filter 2d if self.medfilt_s[0] > 0 and self.medfilt_s[1] > 0: frames = ss.medfilt(frames, kernel_size=self.medfilt_s) return frames
def ReadFiles(self): """Read the data from txt files""" try: import matplotlib.pyplot as plt self.names = tkFileDialog.askopenfilenames(title='Choose acceleration files') self.num = 0 for name in self.names: self.data = genfromtxt(name, delimiter=' ') self.lista.append(self.data) noisy_x = self.lista[self.num].transpose()[0] noisy_y = self.lista[self.num].transpose()[1] noisy_z = self.lista[self.num].transpose()[2] self.noisy_x.append(self.lista[self.num].transpose()[0]) self.noisy_y.append(self.lista[self.num].transpose()[0]) self.noisy_z.append(self.lista[self.num].transpose()[0]) self.num = self.num + 1 n = 3 #order of the smedian filter x_set = medfilt(noisy_x,n) y_set = medfilt(noisy_y,n) z_set = medfilt(noisy_z,n) self.x_set.append(x_set) self.y_set.append(y_set) self.z_set.append(z_set) self.numSamples,m = self.lista[0].shape except: showerror("Error in the files ")
def calc_derived_particle_info(self): if self.species == 'e': self.charge = -1.0 else: self.charge = 1.0 self.t = self.ptl.t * self.pic_info.dtwci / self.pic_info.dtwpe self.nt, = self.t.shape self.px = self.ptl.x / self.smime # Change de to di self.py = self.ptl.y / self.smime self.pz = self.ptl.z / self.smime self.adjust_px() self.adjust_py() self.gama = np.sqrt(self.ptl.ux**2 + self.ptl.uy**2 + self.ptl.uz**2 + 1.0) self.mint = 0 self.maxt = np.max(self.t) self.jdote_x = self.ptl.ux * self.ptl.ex * self.charge / self.gama self.jdote_y = self.ptl.uy * self.ptl.ey * self.charge / self.gama self.jdote_z = self.ptl.uz * self.ptl.ez * self.charge / self.gama self.dt = np.zeros(self.nt) self.dt[0:self.nt - 1] = np.diff(self.t) self.jdote_x_cum = np.cumsum(self.jdote_x) * self.dt self.jdote_y_cum = np.cumsum(self.jdote_y) * self.dt self.jdote_z_cum = np.cumsum(self.jdote_z) * self.dt self.jdote_tot_cum = self.jdote_x_cum + self.jdote_y_cum + self.jdote_z_cum kernel = 9 self.ex = signal.medfilt(self.ptl.ex, kernel_size=(kernel)) self.ey = signal.medfilt(self.ptl.ey, kernel_size=(kernel)) self.ez = signal.medfilt(self.ptl.ez, kernel_size=(kernel)) self.xmin_b = np.min(self.pxb) self.xmax_b = np.max(self.pxb)
def __init__(self,surfaceList,dt,subslice=None,filter_kernel_size=0,stereo=True): ''' Arguments: *surfaceList*: numpy array of Surfaces (N). *dt*: float, timestep *subslice*: numpy index tuples, created wit np.s_ ''' if subslice is None: if stereo: FourD=np.array([[s.data['Ux'],s.data['Uy'],s.data['Uz']] for s in surfaceList]) else: FourD=np.array([[s.data['Ux'],s.data['Uy']] for s in surfaceList]) else: if stereo: FourD=np.array([[s.data['Ux'][subslice],s.data['Uy'][subslice],s.data['Uz'][subslice]] for s in surfaceList]) else: FourD=np.array([[s.data['Ux'][subslice],s.data['Uy'][subslice]] for s in surfaceList]) if filter_kernel_size>0: print 'start smoothing' for s in FourD: s[0]=sp.medfilt(s[0],kernel_size=filter_kernel_size) s[1]=sp.medfilt(s[1],kernel_size=filter_kernel_size) if stereo: s[2]=sp.medfilt(s[2],kernel_size=filter_kernel_size) print 'stop' inputShape=FourD.shape vecs = FourD.reshape((inputShape[0],np.prod(inputShape[1:]))).T super(DMDPiv,self).__init__(vecs,dt) self.inputShape=inputShape
def removeCR(self, fitsFile, sigmaLevel=7): # use a median filter to smooth the region of interest first # if self.rootName == 'ibxy02jpq': # import ipdb;ipdb.set_trace() im = fitsFile['sci', 1].data[5:5 + self.arraySize, 5:5 + self.arraySize] / self.flat * self.dqMask imROI = im[self.ROI[2]:self.ROI[3], self.ROI[0]:self.ROI[1]] # use a 5 pixel sized median filter to remove hot pixels err = fitsFile['err', 1].data[5:5 + self.arraySize, 5:5 + self.arraySize] / self.flat * self.dqMask errROI = err[self.ROI[2]:self.ROI[3], self.ROI[0]:self.ROI[1]] diff1 = np.abs(imROI - medfilt(imROI, [1, 7]))/errROI diff2 = np.abs(imROI - medfilt(imROI, [7, 1]))/errROI diff = np.minimum(diff1, diff2) yCR, xCR = np.where(diff > sigmaLevel) self.dqMask[self.ROI[2]+yCR, self.ROI[0]+xCR] = np.nan dqROI = self.dqMask[self.ROI[2]:self.ROI[3], self.ROI[0]:self.ROI[1]] self.scanRate = np.nanmean((imROI*dqROI)[:, 20:-20], axis=1) /\ np.median(np.nanmean((imROI*dqROI)[:, 20:-20], axis=1)) self.scanDQIndex = np.where(np.abs(self.scanRate - 1) > 0.04)[0] # find scan rate anomaly # for scanDQi in self.scanDQIndex: # self.dqMask[self.ROI[2]+scanDQi-5:self.ROI[2]+scanDQi+6, # self.ROI[0]:self.ROI[1]] = np.nan print("file:{0} {1}/{2} cosmic ray found".format(self.rootName, len(yCR), imROI.size)) print("file:{0} {1} lines large scanrate".format(self.rootName, len(self.scanDQIndex)))
def compute_weight(pov, energy, params): if params.weight_type != 'binary_both': if params.weight_type == 'pov': weight = pov elif params.weight_type == 'energy': weight = energy elif params.weight_type == 'pov_energy_mult': if params.weight_norm_mult is True: pov = (pov - np.min(pov)) / (np.max(pov) - np.min(pov)) energy = (energy - np.min(energy)) / (np.max(energy) - np.min(energy)) weight = pov * energy elif params.weight_type == 'pov_energy_add': alpha = params.weight_alpha weight = alpha*pov + (1-alpha)*energy if params.weight_norm is True: weight = (weight - np.min(weight)) / (np.max(weight) - np.min(weight)) if params.weight_binary is True: # threshold it to binary weight = np.where(weight > params.weight_th, 1, 0) if params.weight_med_filt: weight = signal.medfilt(weight, params.weight_med_filt) else: # binary decision based on both pov = (pov - np.min(pov)) / (np.max(pov) - np.min(pov)) energy = (energy - np.min(energy)) / (np.max(energy) - np.min(energy)) weight = np.where((pov > params.weight_th) & (energy > params.weight_th), 1, 0) if params.weight_med_filt: weight = signal.medfilt(weight, params.weight_med_filt) return weight
def apply_median_filter(self, window_size=3): if self.mean_signal is None: self.x = signal.medfilt(self.x, window_size) self.y = signal.medfilt(self.y, window_size) self.z = signal.medfilt(self.z, window_size) else: self.mean_signal = signal.medfilt(self.mean_signal, window_size)
def redshift_estimate(self,early_type_wave,early_type_flux,wave,Flux_science,gal_prior=None): ''' estimate redshift for object ''' #manage redshift prior self.gal_prior = gal_prior #continuum subtract Flux_sc = Flux_science - signal.medfilt(Flux_science,171) early_type_flux_sc = early_type_flux - signal.medfilt(early_type_flux,171) #handle single redshift prior flag if self.est_pre_z == '1': if self.gal_prior: self.pre_z_est = self.gal_prior else: nospec = raw_input('You said you are either using a spectroscopic or photometric redshift prior. '\ 'You need to specify a prior value! Either enter a number in now or type (q) to exit') if nospec == 'q': sys.exit() elif not nospec: sys.exit() else: self.gal_prior = np.float(nospec) self.pre_z_est = self.gal_prior #handle user prior flag if self.est_pre_z == '2': print 'Take a look at the plotted galaxy spectrum and note, approximately, at what wavelength do you see the '+self.uline_n+' line. '\ 'Then close the plot and enter that wavelength in angstroms.' plt.plot(wave,Flux_science) plt.xlim(self.lower_w,self.upper_w) plt.show() line_init = raw_input(self.uline_n+' approx. wavelength (A): ') self.pre_z_est = np.float(line_init)/self.uline - 1 #handle no prior flag if self.est_pre_z == '3': self.pre_z_est = None redshift_est,cor,ztest,corr_val = self._cross_cor(self.pre_z_est,self.z_prior_width,early_type_wave,early_type_flux_sc,wave,Flux_sc) self.qualityval = 0 self.first_pass = True self._GUI_display(redshift_est,ztest,corr_val,wave,Flux_science) #self.line_est = Estimateline(self.pspec,ax) #plt.show() try: self.pre_lam_est = self.line_est.lam self.pre_z_est = self.pre_lam_est/3950.0 - 1.0 self.first_pass = False redshift_est,cor,ztest,corr_val = self._cross_cor(self.pre_z_est,self.z_prior_width,early_type_wave,early_type_flux_sc,wave,Flux_sc) print 'redshift est:',redshift_est self._GUI_display(redshift_est,ztest,corr_val,wave,Flux_science) total_new_shift = self.spectra2.dx_tot redshift_est = (self.uline*(1+redshift_est) - total_new_shift)/self.uline - 1 except AttributeError: pass return redshift_est,cor,ztest,corr_val,self.qualityval
def medfilt1D(img,medfilt_radius): " 1D median filter using specified radius " if medfilt_radius<=1: return img; if img.ndim==1: return sig.medfilt(img,medfilt_radius); else: lines = [ sig.medfilt(line ,medfilt_radius) for line in img ]; return np.asarray(lines);
def __calc_gm(self, gseq, t_win=5): tmp = np.array([np.mean(x) for x in gseq]) gmean = -1 * medfilt(tmp.real, t_win) + medfilt(tmp.imag, t_win) * 1j pca = PCA(n_components=1) gmean = pca.fit_transform( np.vstack((gmean.imag, gmean.real)).T).flatten() return gmean, pca
def detrend_medfilt(d, K=8): d = np.concatenate([d[K-1::-1],d,d[:-K-1:-1]], axis=0) d = np.concatenate([d[:,K-1::-1],d,d[:,:-K-1:-1]], axis=1) d_sm = medfilt(d, 2*K+1) d_rs = d - d_sm d_sq = np.abs(d_rs)**2 sig = np.sqrt(medfilt(d_sq, 2*K+1) / .456) # puts median on same scale as average f = d_rs / sig return f[K:-K,K:-K]
def analyzeActualWindow(self,window,numSamples): """ function [gravity body] = AnalyzeActualWindow(window,numSamples) % % AnalyzeActualWindow separates the gravity and body acceleration features % contained in the window of real-time acceleration data, by first reducing % the noise on the raw data with a median filter and then discriminating % between the features with a low-pass IIR filter.""" #REDUCE THE NOISE ON THE SIGNALS BY MEDIAN FILTERING n = 3 #order of the median filter x_axis = medfilt(window[:,0],n) y_axis = medfilt(window[:,1],n) z_axis = medfilt(window[:,2],n) #APPLY IIR FILTER TO GET THE GRAVITY COMPONENTS #IIR filter parameters (all frequencies are in Hz) Fs = 32; # sampling frequency Fpass = 0.25; # passband frequency Fstop = 2; # stopband frequency Apass = 0.001; # passband ripple (dB) Astop = 100; # stopband attenuation (dB) match = 'pass'; # band to match exactly #Create the IIR filter # iirdesign agruements Wip = (Fpass)/(Fs/2) Wis = (Fstop+1e6)/(Fs/2) Rp = Apass # passband ripple As = Astop # stopband attenuation # The iirdesign takes passband, stopband, passband ripple, # and stop attenuation. bb, ab = ifd.iirdesign(Wip, Wis, Rp, As, ftype='cheby1') #Gravity components g1 = lfilter(bb,ab,x_axis) g2 = lfilter(bb,ab,y_axis) g3 = lfilter(bb,ab,z_axis) #COMPUTE THE BODY-ACCELERATION COMPONENTS BY SUBTRACTION (PREGUNTA) gravity = zeros((numSamples,3)); body = zeros((numSamples,3)); i=0 while(i < numSamples): gravity[i,0] = g1[i]; gravity[i,1] = g2[i]; gravity[i,2] = g3[i]; body[i,0] = x_axis[i] - g1[i]; body[i,1] = y_axis[i] - g2[i]; body[i,2] = z_axis[i] - g3[i]; i = i + 1 #COMPUTE THE BODY-ACCELERATION COMPONENTS BY SUBTRACTION return gravity, body
def RS_Kalman(lat,lng,l_size=5 ,g_size=5): Lat=signal.medfilt(lat,l_size) Lng=signal.medfilt(lng,g_size) drewgps(Lat,Lng) Lat=KalmanByGroup(Lat) Lng=KalmanByGroup(Lng) drewgps(Lat,Lng) gps_data = [[Lat[i],Lng[i]] for i in range(len(Lat))] return gps_data
def normalize(self, data): if self.verbose: print 'Normalizing data' # Parameters filterWidth = 151 scale = 30 ## nearZero = np.std(data) * 0.05 nearZero = 10 numCols = data.shape[1] maxValue = 400 newData = np.zeros(data.shape) for i in range(numCols): d = data[:,i].copy() # Find envelope if self.verbose: print " - {}: Finding envelope".format(i) upperPeaks = np.zeros(d.shape) lowerPeaks = np.zeros(d.shape) currentUpper = 0.0 currentLower = 0.0 for j in range(1, len(d)-1): if (d[j-1] < d[j] > d[j+1]) and d[j] > 0: currentUpper = d[j] if (d[j-1] > d[j] < d[j+1]) and d[j] < 0: currentLower = d[j] upperPeaks[j] = currentUpper lowerPeaks[j] = currentLower # Decimate peak list dec = 10 decUpperPeaks = upperPeaks[np.arange(1,len(upperPeaks), dec)] decLowerPeaks = lowerPeaks[np.arange(1,len(lowerPeaks), dec)] # Filter shortened peak list if self.verbose: print " - {}: Median filter".format(i) decUpperPeaks = sig.medfilt(decUpperPeaks, filterWidth) decLowerPeaks = sig.medfilt(decLowerPeaks, filterWidth) # Un-decimate peak list for x in range(len(upperPeaks)): upperPeaks[x] = decUpperPeaks[x/dec] lowerPeaks[x] = decLowerPeaks[x/dec] # Normalize using width of envelope width = (upperPeaks - lowerPeaks) / 2.0 width[width < nearZero] = np.inf d = (d / width) * scale # Limit peask d[d > maxValue] = maxValue d[d < -maxValue] = -maxValue newData[:,i] = d return newData
def plot_example(missed, acknowledged): sensor_miss = import_sensorfile(missed) sensor_ack = import_sensorfile(acknowledged) # Window data mag_miss = window_data(process_input(sensor_miss)) mag_ack = window_data(process_input(sensor_ack)) # Window data mag_miss = window_data(process_input(sensor_miss)) mag_ack = window_data(process_input(sensor_ack)) # Filter setup kernel = 15 # apply filter mag_miss_filter = sci.medfilt(mag_miss, kernel) mag_ack_filter = sci.medfilt(mag_ack, kernel) # calibrate data mag_miss_cal = mf.calibrate_median(mag_miss) mag_miss_cal_filter = mf.calibrate_median(mag_miss_filter) mag_ack_cal = mf.calibrate_median(mag_ack) mag_ack_cal_filter = mf.calibrate_median(mag_ack_filter) # PLOT sns.set_style("white") current_palette = sns.color_palette('muted') sns.set_palette(current_palette) plt.figure(0) # Plot RAW missed and acknowledged reminders ax1 = plt.subplot2grid((2, 1), (0, 0)) plt.ylim([-1.5, 1.5]) plt.ylabel('Acceleration (g)') plt.plot(mag_miss_cal, label='Recording 1') plt.legend(loc='lower left') ax2 = plt.subplot2grid((2, 1), (1, 0)) # Plot Missed Reminder RAW plt.ylim([-1.5, 1.5]) plt.ylabel('Acceleration (g)') plt.xlabel('t (ms)') plt.plot(mag_ack_cal, linestyle='-', label='Recording 2') plt.legend(loc='lower left') # CALC AND SAVE STATS stats_one = sp.calc_stats_for_data_stream_as_dictionary(mag_miss_cal) stats_two = sp.calc_stats_for_data_stream_as_dictionary(mag_ack_cal) data = [stats_one, stats_two] write_to_csv(data, 'example_waves') plt.show()
def filter_acceleration(x, y, z): x = medfilt(np.array(x)) y = medfilt(np.array(y)) z = medfilt(np.array(z)) x = butter_bandpass_filter(x, 0, 20, 50, order=3) y = butter_bandpass_filter(y, 0, 20, 50, order=3) z = butter_bandpass_filter(z, 0, 20, 50, order=3) return x, y, z
def filter_gyroscope(x, y, z): x = medfilt(np.array(x)) y = medfilt(np.array(y)) z = medfilt(np.array(z)) x = butter_bandpass_filter(x, 0.3, 20, 50, order=3) y = butter_bandpass_filter(y, 0.3, 20, 50, order=3) z = butter_bandpass_filter(z, 0.3, 20, 50, order=3) return x, y, z
def process_cell(img): # la binariza en caso de que sea escala de grises if not img.dtype == 'bool': img = img > 0 # Binarizar # Calcular máscaras para limpiar lineas largas verticales h_k = 0.8 sum0 = np.sum(img, 0) # Aplastar la matriz a una fila con las sumas de los valores de cada columna. thr0 = sum0 < h_k * img.shape[0] thr0 = thr0.reshape(len(thr0), 1) # Convertirlo a vector de una dimensión # Calcular máscaras para limpiar lineas largas horizontales w_k = 0.5 sum1 = np.sum(img, 1) thr1 = sum1 < w_k * img.shape[1] thr1 = thr1.reshape(len(thr1), 1) mask = thr0.transpose() * thr1 # Generar máscara final para la celda mask_lines = mask.copy() elem = morphology.square(5) mask = morphology.binary_erosion(mask, elem) # Eliminar ruido img1 = np.bitwise_and(mask, img) # Imagen filtrada # segmentación del bloque de números kerw = 5 # Kernel width thr_k = 0.8 # Calcular mascara para marcar inicio y fin de región con dígitos horizontalmente sum0 = np.sum(img1, 0) sum0 = signal.medfilt(sum0, kerw) thr0 = sum0 > thr_k * np.median(sum0) thr0 = np.bitwise_and(thr0.cumsum() > 0, np.flipud(np.flipud(thr0).cumsum() > 0)) thr0 = thr0.reshape(len(thr0), 1) # Calcular mascara para marcar inicio y fin de región con dígitos verticalmente sum1 = np.sum(img1, 1) sum1 = signal.medfilt(sum1, kerw) thr1 = sum1 > thr_k * np.median(sum1) thr1 = np.bitwise_and(thr1.cumsum() > 0, np.flipud(np.flipud(thr1).cumsum() > 0)) thr1 = thr1.reshape(len(thr1), 1) # Mascara final para inicio y fin de caracteres (bounding box of digit region) mask = thr0.transpose() * thr1 mask = morphology.binary_dilation(mask, morphology.square(2)) img = np.bitwise_and(mask_lines.astype(img.dtype), img) # Aplicar máscara para quitar lineas img = morphology.binary_dilation(img, morphology.disk(1)) # Dilatación para unir números quebrados por la máscara anterior img = morphology.binary_erosion(img, morphology.disk(1)) # Volver a la fomorma 'original' con los bordes unidos return np.bitwise_and(mask, img)
def two_column_data(self, z, smooth): self.wave, self.flux = self.spectrum filterSize = int(len(self.wave)/self.nw) * smooth * 2 + 1 preFiltered = medfilt(self.flux, kernel_size=filterSize) wave, flux = self.readSpectrumFile.two_col_input_spectrum(self.wave, preFiltered, z) binnedwave, binnedflux, minindex, maxindex = self.preProcess.log_wavelength(wave, flux) newflux, continuum = self.preProcess.continuum_removal(binnedwave, binnedflux, self.numSplinePoints, minindex, maxindex) meanzero = self.preProcess.mean_zero(binnedwave, newflux, minindex, maxindex) apodized = self.preProcess.apodize(binnedwave, meanzero, minindex, maxindex) #filterSize = smooth * 2 + 1 medianFiltered = medfilt(apodized, kernel_size=1)#filterSize) # from scipy.interpolate import interp1d # # plt.plot(self.flux) # # spline = interp1d(binnedwave[minindex:maxindex], binnedflux[minindex:maxindex], kind='cubic') # waveSpline = np.linspace(binnedwave[minindex],binnedwave[maxindex-1],num=self.numSplinePoints) # print spline # print '###' # print spline(binnedwave[minindex:maxindex]) # plt.figure('1') # plt.plot(waveSpline, spline(waveSpline), '--', label='spline') # # print wave # print binnedwave # print binnedflux # print len(binnedwave) # plt.plot(wave,flux) # plt.figure('2') # plt.plot(binnedwave, binnedflux, label='binned') # plt.plot(binnedwave, newflux, label='continuumSubtract1') # plt.plot(binnedwave, continuum, label='polyfit1') # print len(binnedwave) # print (min(binnedwave), max(binnedwave)) # print len(newflux) # # #newflux2, poly2 = self.preProcess.continuum_removal(binnedwave, binnedflux, 6, minindex, maxindex) # #plt.plot(binnedwave, newflux2, label='continuumSubtract2') # #plt.plot(binnedwave, poly2, label='polyfit2') # plt.plot(binnedwave, apodized, label='taper') # plt.legend() # plt.figure('filtered') # plt.plot(medianFiltered) # plt.figure('3') # plt.plot(medfilt(apodized,kernel_size=3)) # plt.show() return binnedwave, medianFiltered, minindex, maxindex
def get_median_bandpass(self, medlen=21, plot=False): self.median_bandpass_avg = medfilt(self.bandpass_avg, medlen) self.median_bandpass_std = medfilt(self.bandpass_std, medlen) if plot: plotxy(self.median_bandpass_avg, self.freqs, labx="Frequency (MHz)") plotxy(self.median_bandpass_avg+self.median_bandpass_std, self.freqs, color="red") plotxy(self.median_bandpass_avg-self.median_bandpass_std, self.freqs, color="red") closeplot() return self.median_bandpass_avg
def HighPassFilter(t, px, py, moco_kernel, central_fix): """ Slow drift correction by robust high-pass filtering Effectively forces long-term average pupil fixation to be centrally fixated in video space. Arguments ---- t : 1D float array Video soft timestamps in seconds px : 1D float array Video space pupil center x py : 1D float array Video space pupil center y moco_kernel : integer Temporal kernel width in samples [31] central_fix : float tuple (x,y) coordinate in video space of central fixation Returns ---- px_filt : 1D float array Drift corrected video space pupil center x py_filt : 1D float array Drift corrected video space pupil center y """ # Force odd-valued kernel width moco_kernel = utils._forceodd(moco_kernel) print(" Highpass filtering with %d sample kernel" % moco_kernel) # Infill NaN regions # Replace NaNs with unreasonable but finite value for median filtering nan_idx = np.isnan(px) px[nan_idx] = -1e9 py[nan_idx] = -1e9 # Moving median filter to estimate baseline px_bline = medfilt(px, moco_kernel) py_bline = medfilt(py, moco_kernel) # Restore NaNs to vectors px_bline[nan_idx] = np.nan py_bline[nan_idx] = np.nan # Subtract baseline and add central fixation offset px_filt = px - px_bline + central_fix[0] py_filt = py - py_bline + central_fix[1] return px_filt, py_filt, px_bline, py_bline
def __calc_lm(self, lseq, t_win=5): for x in range(len(lseq)): if(len(lseq[x]) == 0): lseq[x] = [np.nan] ltmp = np.array([np.mean(x) for x in lseq]) ltmp = medfilt(ltmp.real, t_win) + medfilt(ltmp.imag, t_win) * 1j lmean = np.abs(ltmp) idx = ~np.isnan(ltmp) size = np.mean([len(x) for x in lseq]) length = np.sum(idx) lstd = nanmean([np.std(x) for x in lseq], axis=0)[0, 0] return lmean, size, length, lstd
def test_none(self): # Ticket #1124. Ensure this does not segfault. try: signal.medfilt(None) except: pass # Expand on this test to avoid a regression with possible contiguous # numpy arrays that have odd strides. The stride value below gets # us into wrong memory if used (but it does not need to be used) dummy = np.arange(10, dtype=np.float64) a = dummy[5:6] a.strides = 16 assert_(signal.medfilt(a, 1) == 5.)
def test_none(self): """Ticket #1124. Ensure this does not segfault.""" try: signal.medfilt(None) except: pass # Expand on this test to avoid a regression with possible contiguous # numpy arrays that have odd strides. The stride value below gets # us into wrong memory if used (but it does not need to be used) a = np.lib.stride_tricks.as_strided(np.ones(1), shape=(1,), strides=(2**31,)) # a may be contiguous (and is certainly for numpy <1.8) because it # has only one element signal.medfilt(a)
def clear_other(): # get some linear data x = np.linspace(0, 1, 101) # add some noisy signal x[3::10] = 1.5 p.plot(x) p.plot(signal.medfilt(x, 3)) p.plot(signal.medfilt(x, 5)) p.legend(['original signal', 'length 3', 'length 5']) p.show()
for h in [7, 5, 4, 3, 2, 1]: # remove exposures del img_hud[h] # remove median img_data = img_hud[1].data img_hud[1].data = img_data - np.median(img_data) # store to array sky_sub_imgs[:, :, i_s] = np.array(img_hud[1].data) i_s += 1 # save img_hud.writeto(new_filename, overwrite=True) img_hud.close() # median combine and save sky image if running_sky_median: print 'Creating running sky median - can take a while' sky_median = medfilt(sky_sub_imgs, kernel_size=(1, 1, median_window)) # replace frames at the beging and at the end of the running median sky n_f = len(image_filename) i_beg_replace = (median_window - 1) / 2 for i_b in range(0, i_beg_replace): sky_median[:, :, i_b] = sky_median[:, :, i_beg_replace] i_end_replace = n_f - i_beg_replace - 1 for i_b in range(i_end_replace + 1, n_f): sky_median[:, :, i_b] = sky_median[:, :, i_end_replace] else: sky_image = np.median(sky_sub_imgs, axis=2) flat_hud[0].data = sky_image flat_hud.writeto('syk_image.fits', overwrite=True) flat_hud.close() # substract sky image and median from image
def callback(data): """ This callback runs each time a LIDAR scan is obtained from the /scan topic in ROS. Returns a topic /detection. If no object is found, /detection = [0,0]. If an object is found, /detection = [angle,distance] to median of scan. """ # Initialize parameters rate = rospy.Rate(10) scan_dist_thresh = 0.1 # Distance threshold to split obj into 2 obj. plot_data = True # Set max/min angle and increment scan_min = data.angle_min scan_max = data.angle_max scan_inc = data.angle_increment # Build angle array x = np.arange(scan_min, scan_max, scan_inc) # Pre-compute trig functions of angles xsin = np.sin(x) xcos = np.cos(x) # Apply a median filter to the range scans y = sg.medfilt(data.ranges, 1) # Calculate the difference between consecutive range values y_diff1 = np.power(np.diff(y), 2) # Convert range and bearing measurement to cartesian coordinates x_coord = y * xsin y_coord = y * xcos # Compute difference between consecutive values in cartesian coordinates x_diff = np.power(np.diff(x_coord), 2) y_diff = np.power(np.diff(y_coord), 2) # Compute physical distance between measurements dist = np.power(x_diff + y_diff, 0.5) # Segment the LIDAR scan based on physical distance between measurements x2 = np.array( np.split(x, np.argwhere(dist > scan_dist_thresh).flatten()[1:])) y2 = np.array( np.split(y, np.argwhere(dist > scan_dist_thresh).flatten()[1:])) dist2 = np.array( np.split(dist, np.argwhere(dist > scan_dist_thresh).flatten()[1:])) x_coord2 = np.array( np.split(x_coord, np.argwhere(dist > scan_dist_thresh).flatten()[1:])) y_coord2 = np.array( np.split(y_coord, np.argwhere(dist > scan_dist_thresh).flatten()[1:])) rate = rospy.Rate(10.0) # Loop through each segmented object for i in range(len(x2)): # Check if there are at least 4 points in an object (reduces noise) xlen = len(x2[i]) - 0 if xlen > 4: # Calculate distance of this object dist2_sum = np.sum(dist2[i][1:xlen - 1]) detectX = 0.0 detectY = 0.0 # Check if this object is too small if dist2_sum > 0.25 and dist2_sum < 3: ang = np.median(x2[i]) dis = np.median(y2[i]) mn = min(y2[i][1:xlen]) mx = max(y2[i][1:xlen]) print "Ang" print ang print "Dis" print dis roboX = rospy.get_param("/currentRobotX") roboY = rospy.get_param("/currentRobotY") arenaPnt1 = rospy.get_param("/arenaPnt1") arenaPnt2 = rospy.get_param("/arenaPnt2") deadZone1 = rospy.get_param("/deadZone1") deadZone2 = rospy.get_param("/deadZone2") roboR = rospy.get_param("/currentRobotR") print "RoboX" print roboX print "RoboY" print roboY print "RoboR" print roboR detectX = roboX + (dis * math.cos(ang + roboR)) detectY = roboY - (dis * math.sin(ang + roboR)) print "DetectX" print detectX print "DetectY" print detectY if ang > -1.4 and ang < 1.4 and detectX > arenaPnt1[ 0] and detectX < arenaPnt2[0] and detectY < arenaPnt1[ 1] and detectY > arenaPnt2[1] and not ( detectX > deadZone1[0] and detectX < deadZone2[0] and detectY < deadZone1[1] and detectY > deadZone2[1]): bearing = np.array([ang, dis], dtype=np.float32) else: print "Bad box" rate.sleep() # Check if bearing exists. Store [0,0] if no object was found if 'bearing' not in locals(): bearing = np.array([0, 0], dtype=np.float32) # Publish bearing to ROS on topic /detection pub = rospy.Publisher("/detection", numpy_msg(Floats), queue_size=1) pub.publish(bearing) # If we want to plot the LIDAR scan, open the plot environment if plot_data: plt.figure(1) for i in range(len(x2)): xlen = len(x2[i]) - 0 if xlen > 4: dist2_sum = np.sum(dist2[i][1:xlen - 1]) if dist2_sum < 0.25: if plot_data: plt.plot(x2[i][1:xlen], y2[i][1:xlen], 'k-', linewidth=2.0) else: if dist2_sum > 3: if plot_data: plt.plot(x2[i][1:xlen], y2[i][1:xlen], 'b-', linewidth=2.0) else: ang = np.median(x2[i]) # - (math.pi / 2.0) dis = np.median(y2[i]) roboX = rospy.get_param("/currentRobotX") roboY = rospy.get_param("/currentRobotY") arenaPnt1 = rospy.get_param("/arenaPnt1") arenaPnt2 = rospy.get_param("/arenaPnt2") deadZone1 = rospy.get_param("/deadZone1") deadZone2 = rospy.get_param("/deadZone2") roboR = rospy.get_param("/currentRobotR") detectX = roboX + (dis * math.cos(ang + roboR)) detectY = roboY - (dis * math.sin(ang + roboR)) if ang > -1.4 and ang < 1.4 and detectX > arenaPnt1[ 0] and detectX < arenaPnt2[ 0] and detectY < arenaPnt1[ 1] and detectY > arenaPnt2[1] and not ( detectX > deadZone1[0] and detectX < deadZone2[0] and detectY < deadZone1[1] and detectY > deadZone2[1]): if plot_data: plt.plot(x2[i][1:xlen], y2[i][1:xlen], 'g-', linewidth=2.0) else: if plot_data: plt.plot(x2[i][1:xlen], y2[i][1:xlen], 'r-', linewidth=2.0) plt.ylim([0, 20]) plt.xlim([-5, 5]) plt.gca().invert_xaxis() plt.xlabel('Left of robot [m] ') plt.ylabel('Front of robot [m]') plt.title('Laser Scan') imgdata = StringIO.StringIO() plt.savefig(imgdata, format='png') imgdata.seek(0) img_array = np.asarray(bytearray(imgdata.read()), dtype=np.uint8) im = cv2.imdecode(img_array, 1) bridge = CvBridge() image_output = rospy.Publisher("/output/keyevent_image", Image, queue_size=1) image_output.publish(bridge.cv2_to_imgmsg(im, "bgr8")) plt.close() pass
def MedianFilter(image): image = signal.medfilt(image, (5, 5)) return image
import rasterio from scipy.signal import medfilt path = "tests/data/RGB.byte.tif" output = "/tmp/filtered.tif" with rasterio.open(path) as src: array = src.read() profile = src.profile # apply a 5x5 median filter to each band filtered = medfilt(array, (1, 5, 5)).astype('uint8') # Write to tif, using the same profile as the source with rasterio.open(output, 'w', **profile) as dst: dst.write(filtered)
i = i + 1 w111 = np.concatenate((w1, w11), axis=0) w1 = w111[100:600] x111 = np.concatenate((x1, x11), axis=0) x1 = x111[100:600] y111 = np.concatenate((y1, y11), axis=0) y1 = y111[100:600] z111 = np.concatenate((z1, z11), axis=0) z1 = z111[100:600] col1 = w1 col2 = x1 col3 = y1 col4 = z1 col1 = signal.medfilt(col1, 7) col2 = signal.medfilt(col2, 7) col3 = signal.medfilt(col3, 7) col4 = signal.medfilt(col4, 7) def butter_lowpass(cutOff, fs, order=1): nyq = 0.5 * fs normalCutoff = cutOff / nyq b, a = butter(order, normalCutoff, btype='low', analog=True) return b, a def butter_lowpass_filter(data, cutOff, fs, order=4): b, a = butter_lowpass(cutOff, fs, order=order) y = lfilter(b, a, data) return y
i = 1 histogram_overlay, all_x, all_y = histogram_lane_detection(img, steps, search_window, h_window, peak_threshold=2000, frame_debug=True) cv2.imshow('OVERLAY', cv2.cvtColor(histogram_overlay, cv2.COLOR_RGB2BGR)) start = masked_img.shape[0] - (i * pixels_per_step) end = start - pixels_per_step histogram = np.sum(masked_img[end:start, :], axis=0) histogram_smooth = signal.medfilt(histogram, h_window) peaks = np.array(signal.find_peaks_cwt(histogram_smooth, np.arange(1, 10))) print(peaks) peaks = peaks[np.nonzero(histogram_smooth[peaks] > peak_threshold)] print(peaks) print(histogram_smooth) print(histogram_smooth / max(histogram_smooth)) histogram_overlay = np.zeros((img.shape[0], img.shape[1], 3), np.uint8) COLOR_BOX_BORDER = np.array([100, 160, 255], np.uint8) COLOR_HIST_PLOT = np.array([225, 150, 50], np.uint8) COLOR_POSITIVE_DETECTION = np.array([0, 150, 0], np.uint8) COLOR_THRESHOLD = np.array([50, 225, 225], np.uint8) COLOR_GRID = np.array([100, 150, 150], np.uint8) print(histogram_smooth.shape) print(start, end) histogram_overlay[start - 2:start, :, :] = COLOR_BOX_BORDER
#Making the CSV file: #visititems() recursively gets every item(i.e group, dataset etc) and its address in the hdf file #and passes it to a specified callable function f.visititems(DatasetCheck) with open('ScriptOutput.csv', 'w') as h: for row in csvTemplate: rowstring = [] for item in row: rowstring.append(str(item)) h.write(",".join(rowstring) + "\n") #Plotting the Image: imgHt = np.array(f['AwakeEventData']['XMPP-STREAK']['StreakImage'].get( 'streakImageHeight'))[0] imgWdth = np.array(f['AwakeEventData']['XMPP-STREAK']['StreakImage'].get( 'streakImageWidth'))[0] img = np.array( f['AwakeEventData']['XMPP-STREAK']['StreakImage'].get('streakImageData')) img = medfilt(np.reshape(img, (imgHt, imgWdth))) plt.imshow(img) plt.savefig('XMPP-STREAK-Image.png') #Creating the Datetime objects: timestamp = round(1541962108935000000 / (10**(9))) date_str = datetime.datetime.fromtimestamp(timestamp) datetime_obj_UTC = datetime.datetime.strptime(str(date_str), "%Y-%m-%d %H:%M:%S") datetime_obj_CERN = datetime_obj_UTC.replace(tzinfo=pytz.utc).astimezone( pytz.timezone('Europe/Zurich'))
def run_epd(lcfile, epdparams=None, lccols=None, lccolnames=None, addcols=None, framecol='rstfc', sigclip=5.0, smooth=21, minndet=200, num_aps=3, outdir=None, outext='.epdlc'): ''' Runs EPD on the given lcfile Args: lcfile (str): Path to lcfile. epdparams (dict or OrderedDict): keys are columns, values are either None or a callable that operates on a lightcurve lccols (list): Columns to read from the lc lccolnames (list): Names of columns read from the lc addcols (DataFrame): Additional external parameters framecol (str): Column to merge lc and addcols on sigclip (float): Sigma-clipping to be done on lc before fitting smooth (int or False): Smoothing parameter to apply to lc minndet (int): Minimum number of observations in lc num_aps (int): Number of apertures outext (str): Extension of output file. Returns: outfile if successful, None otherwise. ''' # Defaults if epdparams is None: epdparams = DEFAULT_EPDPARAMS # Read LC lc = read_lc(lcfile, lccols, lccolnames) # Join with additional columns if addcols is not None: lc = pd.merge(lc, addcols, how='left', on=framecol) # Generate additional columns required for EPD lc = generate_newcols(lc, epdparams) epdcols = list(epdparams.keys()) # Only use rows where all params are finite finite_params = np.ones(len(lc), dtype=bool) for col in epdcols: finite_params &= np.isfinite(lc[col]) # Run for each aperture epcols = [] for ap in range(1, num_aps+1): magcol = 'rm%d' % ap finiteind = np.isfinite(lc[magcol]) & finite_params if np.sum(np.ones_like(finiteind)[finiteind]) < minndet: continue mag_median = np.nanmedian(lc[magcol]) mag_stdev = np.nanstd(lc[magcol]) # Sigma clip if sigclip: excludeind = abs(lc[magcol] - mag_median) < (sigclip * mag_stdev) finalind = finiteind & excludeind else: finalind = finiteind # Smoothing if smooth: mags = medfilt(lc[magcol][finalind], smooth) else: mags = lc[magcol][finalind] ### Perform fitting # Construct the matrix from the EPD columns (and a constant term) epdmatrix = np.c_[lc[epdcols], np.ones(len(lc))] # Use least squares fitting to only chosen rows coeffs, residuals, rank, singulars = lstsq(epdmatrix[finalind], mags) # Now compute full EPD mags lc.loc[:,'ep%d' % ap] = (lc[magcol] - np.dot(coeffs, epdmatrix.T) + mag_median) epcols.append('ep%d' % ap) # Save lcfile if at least one epdcolumn was generated lcid = os.path.splitext(os.path.basename(lcfile))[0] if len(epcols) == 0: print('%sZ: EPD failed for %s.' % (datetime.utcnow().isoformat(), lcid + '.grcollectilc')) return None else: if outdir is None: outdir = os.path.dirname(lcfile) outfile = os.path.join(outdir, lcid + outext) # Proper file output by re-reading lines from input file inf = open(lcfile, 'rb') inflines = inf.readlines() inf.close() outf = open(outfile, 'wb') for line, row in zip(inflines, lc.iterrows()): epmags = [('%.6f' % row[1][col]) for col in epcols] inline = line.decode('utf-8').rstrip('\n') outline = ' '.join([inline] + epmags) + '\n' outf.write(outline.encode('utf-8')) outf.close() print('%sZ: EPD OK for %s.' % (datetime.utcnow().isoformat(), lcid + '.grcollectilc')) return outfile
def median_filter(shared_array, i, **kwargs): shared_array[i] = signal.medfilt(shared_array[i], kwargs['kernel_size'])
NP = dims[0] an_conv = np.copy(an) start = int(sys.argv[4]) end = int(sys.argv[5]) B_start = start B_end = end NB = B_end - B_start if (NB): B_v = an[start:end + 1] * np.cos(an[-1]) B_h = an[start:end + 1] * np.sin(an[-1]) for i in range(0, B_start): an_conv[i] = flt.medfilt(an[i], 11) if (sigma): an_conv[i] = filters.gaussian_filter(an_conv[i], sigma) if (NB): for i in range(B_start, B_end + 1): if (sigma): B_v[i - B_start] = filters.gaussian_filter(B_v[i - B_start], sigma) B_h[i - B_start] = filters.gaussian_filter(B_h[i - B_start], sigma) an_conv[i] = np.sqrt(B_v[i - B_start]**2.0 + B_h[i - B_start]**2.0) if (NB): an_conv[-1] = np.cos(an_conv[-1]) an_conv[-1] = filters.gaussian_filter(an_conv[-1], sigma) an_conv[-1] = np.clip(an_conv[-1], -0.99, 0.99) an_conv[-1] = np.arccos(an_conv[-1])
k = (win - 1) // 2 m = y[i + 1:((i + 1) + k)] n = y[(i-k):i] if i < k: # left boundaries zero1 = np.zeros(k-i, dtype=int) a = np.concatenate((zero1, y[:i+1], m), axis=None) q[i] = np.median(a) elif (i+((win-1)//2)) > (len(x)-1): # right boundaries p = k-((len(x)-1)-i) zero2 = np.zeros(p, dtype=int) b = np.concatenate((n, y[i:], zero2), axis=None) q[i] = np.median(b) else: # middle data c = y[i-((win-1)//2):(i+((win+1)//2))] q[i] = np.median(c) plt.plot(x, q, linewidth=1, label='mujmedian', color='green') # inbuilt np.medfilt() function z = signal.medfilt(y, win) plt.plot(x, z, linewidth=0.8, color='orangered', label='medfilt') plt.legend() plt.subplot(212) plt.autoscale(enable=True, axis='x', tight=bool) plt.plot(x, z-q) plt.title('Error', loc='center') plt.show()
def clean_scan_using_variability(dynamical_spectrum, length, bandwidth, good_mask=None, freqsplat=None, noise_threshold=5., debug=True, nofilt=False, outfile="out", label="", smoothing_window=0.05, debug_file_format='pdf', info_string="Empty info string"): """Clean a spectroscopic scan using the difference of channel variability. From the dynamical spectrum, i.e. the list of spectra obtained in each sample of a scan, we calculate the rms variability of each frequency channel. This forms a sort of rms spectrum. We calculate the baseline of this spectrum, and all channels whose rms is above above noise_threshold times the reference median absolute deviation (:func:`srttools.fit.ref_mad`), calculated with a minimum window of 20 samples, are cut and assigned an interpolated value between the closest valid points. The baseline is calculated with :func:`srttools.fit.baseline_als`, using a lambda value depending on the number of channels, with a formula that has been shown to work in a few standard cases but might be modified in the future. Parameters ---------- dynamical_spectrum : 2-d array Array of shape MxN, with M spectra of N elements each. length : float Duration in seconds of the scan (assumed to have constant sample time) bandwidth : float Bandwidth in MHz Other parameters ---------------- good_mask : boolean array this mask specifies channels that should never be discarded as RFI, for example because they contain spectral lines freqsplat : str List of frequencies to be merged into one. See :func:`srttools.scan.interpret_frequency_range` noise_threshold : float The threshold, in sigmas, over which a given channel is considered noisy debug : bool Print out debugging information nofilt : bool Do not filter noisy channels (set noise_threshold to 1e32) outfile : str Root file name for the diagnostics plots (outfile_label.png) label : str Label to append to the filename (outfile_label.png) smoothing_window : float Width of smoothing window, in fraction of spectral length Returns ------- results : object The attributes of this object are: lc : array-like The cleaned light curve freqmin : float Minimum frequency in MHz, referred to local oscillator freqmax : float Maximum frequency in MHz, referred to local oscillator See Also -------- srttools.fit.baseline_als srttools.fit.ref_mad """ try: bandwidth_unit = bandwidth.unit bandwidth = bandwidth.value except AttributeError: bandwidth_unit = u.MHz if len(dynamical_spectrum.shape) == 1: if not debug or not HAS_MPL: return None lc = dynamical_spectrum times = length * np.arange(lc.size) / lc.size # Now, PLOT IT ALL -------------------------------- # Prepare subplots fig = plt.figure("{}_{}".format(outfile, label), figsize=(15, 15)) plt.plot(times, lc) plt.xlabel('Time') plt.ylabel('Counts') plt.gca().text(0.05, 0.95, info_string, horizontalalignment='left', verticalalignment='top', transform=plt.gca().transAxes, fontsize=20) plt.savefig("{}_{}.{}".format(outfile, label, debug_file_format)) plt.close(fig) return None dynspec_len, nbin = dynamical_spectrum.shape # Calculate first light curve times = length * np.arange(dynspec_len) / dynspec_len lc = np.sum(dynamical_spectrum, axis=1) if len(lc) > 10: lc = baseline_als(times, lc) else: lc -= np.median(lc) lcbins = np.arange(len(lc)) # Calculate spectral variability curve meanspec = np.sum(dynamical_spectrum, axis=0) / dynspec_len spectral_var = \ np.sqrt(np.sum((dynamical_spectrum - meanspec) ** 2, axis=0) / dynspec_len) / meanspec df = bandwidth / len(meanspec) allbins = np.arange(len(meanspec)) * df # Mask frequencies -- avoid those excluded from splat freqmask = np.ones(len(meanspec), dtype=bool) freqmin, freqmax, binmin, binmax = \ interpret_frequency_range(freqsplat, bandwidth, nbin) freqmask[0:binmin] = False freqmask[binmax:] = False # Calculate the variability image varimg = np.sqrt((dynamical_spectrum - meanspec) ** 2) / meanspec # Set up corrected spectral var mod_spectral_var = spectral_var.copy() mod_spectral_var[0:binmin] = spectral_var[binmin] mod_spectral_var[binmax:] = spectral_var[binmax] # Some statistical information on spectral var # median_spectral_var = np.median(mod_spectral_var[freqmask]) stdref = ref_mad(mod_spectral_var[freqmask], 20) # Calculate baseline of spectral var --------------- # Empyrical formula, with no physical meaning smoothing_window_int = int(nbin * smoothing_window) // 2 * 2 + 1 smoothing_window_int = np.max([smoothing_window_int, 11]) baseline = medfilt(mod_spectral_var[binmin:binmax], smoothing_window_int) baseline = \ np.concatenate((np.zeros(binmin) + baseline[0], baseline, np.zeros(nbin - binmax) + baseline[-1] )) # Set threshold if nofilt: wholemask = freqmask else: threshold = baseline + noise_threshold * stdref mask = spectral_var < threshold threshold = baseline - noise_threshold * stdref mask = mask & (spectral_var > threshold) wholemask = freqmask & mask if good_mask is None: good_mask = np.zeros_like(freqmask, dtype=bool) wholemask[good_mask] = 1 # Calculate frequency-masked lc lc_masked = np.sum(dynamical_spectrum[:, freqmask], axis=1) # lc_masked = baseline_als(times, lc_masked, outlier_purging=False) if len(lc_masked) > 10: lc_masked = baseline_als(times, lc_masked, outlier_purging=False) else: lc_masked -= np.median(lc_masked) bad_intervals = contiguous_regions(np.logical_not(wholemask)) # Calculate cleaned dynamical spectrum cleaned_dynamical_spectrum = \ _clean_dyn_spec(dynamical_spectrum, bad_intervals) cleaned_meanspec = \ np.sum(cleaned_dynamical_spectrum, axis=0) / len(cleaned_dynamical_spectrum) cleaned_varimg = \ np.sqrt((cleaned_dynamical_spectrum - cleaned_meanspec) ** 2 / cleaned_meanspec ** 2) cleaned_spectral_var = \ np.sqrt(np.sum((cleaned_dynamical_spectrum - cleaned_meanspec) ** 2, axis=0) / dynspec_len) / cleaned_meanspec mean_varimg = np.mean(cleaned_varimg[:, freqmask]) std_varimg = np.std(cleaned_varimg[:, freqmask]) lc_corr = np.sum(cleaned_dynamical_spectrum[:, freqmask], axis=1) if len(lc_corr) > 10: lc_corr = baseline_als(times, lc_corr, outlier_purging=False) else: lc_corr -= np.median(lc_corr) results = type('test', (), {})() # create empty object results.lc = lc_corr results.freqmin = freqmin * u.MHz results.freqmax = freqmax * u.MHz results.mask = wholemask if not debug or not HAS_MPL: return results # Now, PLOT IT ALL -------------------------------- # Prepare subplots fig = plt.figure("{}_{}".format(outfile, label), figsize=(15, 15)) if len(lc_corr) < 10: for i in dynamical_spectrum: plt.plot(allbins[1:], i[1:]) plt.plot(allbins[1:], meanspec[1:]) plt.xlabel('Time') plt.ylabel('Counts') ax = plt.gca() ax.text(0.05, 0.95, info_string, horizontalalignment='left', verticalalignment='top', transform=ax.transAxes, fontsize=20) plt.savefig("{}_{}.{}".format(outfile, label, debug_file_format)) plt.close(fig) return results gs = GridSpec(4, 3, hspace=0, wspace=0, height_ratios=(1.5, 1.5, 1.5, 1.5), width_ratios=(3, 0., 1.2)) ax_meanspec = plt.subplot(gs[0, 0]) ax_dynspec = plt.subplot(gs[1, 0], sharex=ax_meanspec) ax_cleanspec = plt.subplot(gs[2, 0], sharex=ax_meanspec) ax_lc = plt.subplot(gs[1, 2], sharey=ax_dynspec) ax_cleanlc = plt.subplot(gs[2, 2], sharey=ax_dynspec, sharex=ax_lc) ax_var = plt.subplot(gs[3, 0], sharex=ax_meanspec) ax_text = plt.subplot(gs[0, 2]) ax_meanspec.set_ylabel('Counts') ax_dynspec.set_ylabel('Sample') ax_cleanspec.set_ylabel('Sample') ax_var.set_ylabel('r.m.s.') ax_var.set_xlabel('Frequency from LO ({})'.format(bandwidth_unit)) ax_cleanlc.set_xlabel('Counts') # Plot mean spectrum ax_meanspec.plot(allbins[1:], meanspec[1:], label="Unfiltered") # ax_meanspec.plot(allbins[1:], meanspec[1:], label="Whitelist applied") ax_meanspec.plot(allbins[wholemask], meanspec[wholemask], label="Final mask") ax_meanspec.set_ylim([np.min(cleaned_meanspec), np.max(cleaned_meanspec)]) try: cmap = plt.get_cmap("magma") except Exception: cmap = plt.get_cmap("gnuplot2") ax_dynspec.imshow(varimg, origin="lower", aspect='auto', cmap=cmap, vmin=mean_varimg - 5 * std_varimg, vmax=mean_varimg + 5 * std_varimg, extent=(0, bandwidth, 0, varimg.shape[0]), interpolation='none') ax_cleanspec.imshow(cleaned_varimg, origin="lower", aspect='auto', cmap=cmap, vmin=mean_varimg - 5 * std_varimg, vmax=mean_varimg + 5 * std_varimg, extent=(0, bandwidth, 0, varimg.shape[0]), interpolation='none') # Plot variability ax_var.plot(allbins[1:], spectral_var[1:], label="Spectral rms") ax_var.plot(allbins[mask], spectral_var[mask]) ax_var.plot(allbins, cleaned_spectral_var, zorder=10, color="k") ax_var.plot(allbins[1:], baseline[1:]) ax_var.plot(allbins[1:], baseline[1:] + noise_threshold * stdref, color='r', lw=2) ax_var.plot(allbins[1:], baseline[1:] - noise_threshold * stdref, color='r', lw=2) minb = np.min(baseline[1:]) - 2 * noise_threshold * stdref maxb = np.max(baseline[1:]) + 2 * noise_threshold * stdref ax_var.set_ylim([minb, maxb]) # Plot light curves ax_lc.plot(lc, lcbins, color="grey") ax_lc.plot(lc_masked, lcbins, color="b") ax_cleanlc.plot(lc_masked, lcbins, color="grey") ax_cleanlc.plot(lc_corr, lcbins, color="k") dlc = max(lc_corr) - min(lc_corr) ax_lc.set_xlim([np.min(lc_corr) - dlc / 10, max(lc_corr) + dlc / 10]) # Indicate bad intervals for b in bad_intervals: maxsp = np.max(meanspec) ax_meanspec.plot(b * df, [maxsp] * 2, color='k', lw=2) middleimg = [varimg.shape[0] / 2] ax_dynspec.plot(b * df, [middleimg] * 2, color='k', lw=2) maxsp = np.max(spectral_var) ax_var.plot(b * df, [maxsp] * 2, color='k', lw=2) # Indicate freqmin and freqmax ax_var.set_xlim([0, allbins[-1]]) ax_dynspec.axvline(freqmin) ax_dynspec.axvline(freqmax) ax_cleanspec.axvline(freqmin) ax_cleanspec.axvline(freqmax) ax_var.axvline(freqmin) ax_var.axvline(freqmax) ax_meanspec.axvline(freqmin) ax_meanspec.axvline(freqmax) ax_text.text(0.05, 0.95, info_string, horizontalalignment='left', verticalalignment='top', transform=ax_text.transAxes, fontsize=20) ax_text.axis("off") fig.tight_layout() plt.savefig( "{}_{}.{}".format(outfile, label, debug_file_format)) plt.close(fig) return results
def median_filter(signal, size=5): med_signal = medfilt(signal.flatten(), size) return med_signal
#------APERTURA DE LA SEÑAL------- mat_struct = sio.loadmat('/home/luciasucunza/git_proyecto_ecg/Filtros/TP4_ecg.mat') ecg_one_lead = mat_struct['ecg_lead'] ecg_one_lead = ecg_one_lead.flatten(1) cant_muestras = len(ecg_one_lead) fs = 1000 nyq_frec = fs / 2 t = np.arange(cant_muestras) / fs #------OBTENCION DE LA BASELINE POR MEDIANA MOVIL------- to_todo = time.time() baseline = sig.medfilt (ecg_one_lead, kernel_size = int (np.around(fs*0.5*0.2) *2 +1 )) baseline = sig.medfilt (baseline, kernel_size = int (np.around(fs*0.5*0.6) *2 +1 )) baseline = sig.medfilt (baseline, kernel_size = int (np.around(fs*0.5*(1/50)) *2 +1 )) tf_todo = time.time() #------Parametros de Diezmado------- to_500 = time.time() f_B = 7 fs_old_1 = fs fs_new_1 = 500 nyq_frec_1 = fs_old_1/2 fs_old_3 = fs_new_1
def find_slits_corners_aps_1id( img, method='quadrant+', medfilt2_kernel_size=3, medfilt_kernel_size=23, ): """ Automatically locate the slit box location by its four corners. NOTE: The four slits that form a binding box is the current setup at aps_1id, which reduce the illuminated region on the detector. Since the slits are stationary, they can serve as a reference to check detector drifting during the scan. Technically, the four slits should be used to find the transformation matrix (not necessarily affine) to correct the image. However, since we are dealing with 2D images with very little distortion, affine transformation matrices were used for approximation. Therefore the "four corners" are used instead of all four slits. Parameters ---------- img : np.ndarray 2D images method : str, ['simple', 'quadrant', 'quadrant+'], optional method for auto detecting slit corners - simple :: assume a rectange slit box, fast but less accurate (1 pixel precision) - quadrant :: subdivide the image into four quandrant, then use an explicit method to find the corner (1 pixel precision) - quadrant+ :: similar to quadrant, but use curve_fit (gauss1d) to find the corner (0.1 pixel precision) medfilt2_kernel_size : int, optional 2D median filter kernel size for noise reduction medfilt_kernel_size : int, optional 1D median filter kernel size for noise reduction Returns ------- tuple autodetected slit corners (counter-clockwise order) (upperLeft, lowerLeft, lowerRight, upperRight) """ img = medfilt2d( np.log(img.astype(np.float64)), kernel_size=medfilt2_kernel_size, ) rows, cols = img.shape # simple method is simple, therefore it stands out if method.lower() == 'simple': # assuming a rectangle type slit box col_std = medfilt(np.std(img, axis=0), kernel_size=medfilt_kernel_size) row_std = medfilt(np.std(img, axis=1), kernel_size=medfilt_kernel_size) # NOTE: in the tiff img # x is col index, y is the row index ==> key point here !!! # img slicing is doen with img[row_idx, col_idx] # ==> so the image idx and corner position are FLIPPED! _left = np.argmax(np.gradient(col_std)) _right = np.argmin(np.gradient(col_std)) _top = np.argmax(np.gradient(row_std)) _bottom = np.argmin(np.gradient(row_std)) cnrs = np.array([ [_left, _top], [_left, _bottom], [_right, _bottom], [_right, _top], ]) else: # predefine all quadrants # Here let's assume that the four corners of the slit box are in the # four quadrant defined by the center of the image # i.e. # uppper left quadrant: img[0 :cnt[1], 0 :cnt[0]] => quadarnt origin = (0, 0) # lower left quadrant: img[cnt[1]: , 0 :cnt[0]] => quadarnt origin = (cnt[0], 0) # lower right quadrant: img[cnt[1]: , cnt[0]: ] => quadarnt origin = (cnt[0], cnt[1]) # upper right quadrant: img[0 :cnt[1], cnt[0]: ] => quadarnt # origin = (0, cnt[1]) # center of image that defines FOUR quadrants cnt = [int(cols / 2), int(rows / 2)] Quadrant = namedtuple('Quadrant', 'img col_func, row_func') quadrants = [ Quadrant(img=img[0:cnt[1], 0:cnt[0]], col_func=np.argmax, row_func=np.argmax), # upper left, 1st quadrant # lower left, 2nd quadrant Quadrant(img=img[cnt[1]:, 0:cnt[0]], col_func=np.argmax, row_func=np.argmin), # lower right, 3rd quadrant Quadrant(img=img[cnt[1]:, cnt[0]:], col_func=np.argmin, row_func=np.argmin), # upper right, 4th quadrant Quadrant(img=img[0:cnt[0], cnt[1]:], col_func=np.argmin, row_func=np.argmax), ] # the origin in each quadrants ==> easier to set it here quadrantorigins = np.array([ [0, 0], # upper left, 1st quadrant [0, cnt[1]], # lower left, 2nd quadrant # lower right, 3rd quadrant [cnt[0], cnt[1]], [cnt[1], 0], # upper right, 4th quadrant ]) # init four corners cnrs = np.zeros((4, 2)) if method.lower() == 'quadrant': # the standard quadrant method for i, q in enumerate(quadrants): cnrs[i, :] = np.array([ q.col_func( np.gradient( medfilt(np.std(q.img, axis=0), kernel_size=medfilt_kernel_size)) ), # x is col_idx q.row_func( np.gradient( medfilt(np.std(q.img, axis=1), kernel_size=medfilt_kernel_size))), # y is row_idx ]) # add the origin offset back cnrs = cnrs + quadrantorigins elif method.lower() == 'quadrant+': # use Gaussian curve fitting to achive subpixel precision # TODO: # improve the curve fitting with Lorentz and Voigt fitting function for i, q in enumerate(quadrants): # -- find x subpixel position cnr_x_guess = q.col_func( np.gradient( medfilt(np.std(q.img, axis=0), kernel_size=medfilt_kernel_size))) # isolate the strongest peak to fit tmpx = np.arange(cnr_x_guess - 10, cnr_x_guess + 11) tmpy = np.gradient(np.std(q.img, axis=0))[tmpx] # tmpy[0] is the value from the highest/lowest pixle # tmpx[0] is basically cnr_x_guess # 5.0 is the guessted std, coeff, _ = curve_fit( gauss1d, tmpx, tmpy, p0=[tmpy[0], tmpx[0], 5.0], maxfev=int(1e6), ) cnrs[i, 0] = coeff[1] # x position # -- find y subpixel positoin cnr_y_guess = q.row_func( np.gradient( medfilt(np.std(q.img, axis=1), kernel_size=medfilt_kernel_size))) # isolate the peak (x, y here is only associated with the peak) tmpx = np.arange(cnr_y_guess - 10, cnr_y_guess + 11) tmpy = np.gradient(np.std(q.img, axis=1))[tmpx] coeff, _ = curve_fit( gauss1d, tmpx, tmpy, p0=[tmpy[0], tmpx[0], 5.0], maxfev=int(1e6), ) cnrs[i, 1] = coeff[1] # y posiiton # add the quadrant shift back cnrs = cnrs + quadrantorigins else: raise NotImplementedError( "Available methods are: simple, quadrant, quadrant+") # return the slit corner detected return cnrs
def synthesize( fs, f0s, SPEC, NM=None, wavlen=None, ener_multT0=False, nm_cont=False # If False, force binary state of the noise mask (by thresholding at 0.5) , nm_lowpasswinlen=9, hp_f0coef=0.5 # factor of f0 for the cut-off of the high-pass filter (def. 0.5*f0) , antipreechohwindur=0.001 # [s] Use to damp the signal at the beginning of the signal AND at the end of it # Following options are for post-processing the features, after the generation/transformation and thus before waveform synthesis , pp_f0_rmsteps=False # Removes steps in the f0 curve # (see sigproc.resampling.f0s_rmsteps(.) ) , pp_f0_smooth=None # Smooth the f0 curve using median and FIR filters of given window duration [s] , pp_atten1stharminsilences=None # Typical value is -25 , verbose=1): winnbper = 4 # Number of periods in a synthesis windows. It still contains only one single pulse, but leaves space for the VTF to decay without being cut abruptly. # Copy the inputs to avoid modifying them f0s = f0s.copy() SPEC = SPEC.copy() if not NM is None: NM = NM.copy() else: NM = np.zeros(SPEC.shape) # Check the size of the inputs if f0s.shape[0] != SPEC.shape[0]: raise ValueError( 'F0 size {} and spectrogram size {} do not match'.format( f0s.shape[0], SPEC.shape[0])) # pragma: no cover if not NM is None: if SPEC.shape != NM.shape: raise ValueError( 'spectrogram size {} and NM size {} do not match.'.format( SPEC.shape, NM.shape)) # pragma: no cover if wavlen == None: wavlen = int(np.round(f0s[-1, 0] * fs)) dftlen = (SPEC.shape[1] - 1) * 2 shift = np.median(np.diff(f0s[:, 0])) if verbose > 0: print( 'PML Synthesis (dur={}s, fs={}Hz, f0 in [{:.0f},{:.0f}]Hz, shift={}s, dftlen={})' .format(wavlen / float(fs), fs, np.min(f0s[:, 1]), np.max(f0s[:, 1]), shift, dftlen)) # Prepare the features # Enforce continuous f0 f0s[:, 1] = np.interp(f0s[:, 0], f0s[f0s[:, 1] > 0, 0], f0s[f0s[:, 1] > 0, 1]) # If asked, removes steps in the f0 curve if pp_f0_rmsteps: f0s = sp.f0s_rmsteps(f0s) # If asked, smooth the f0 curve using median and FIR filters if not pp_f0_smooth is None: print(' Smoothing f0 curve using {}[s] window'.format(pp_f0_smooth)) import scipy.signal as sig lf0 = np.log(f0s[:, 1]) bcoefslen = int(0.5 * pp_f0_smooth / shift) * 2 + 1 lf0 = sig.medfilt(lf0, bcoefslen) bcoefs = np.hamming(bcoefslen) bcoefs = bcoefs / sum(bcoefs) lf0 = sig.filtfilt(bcoefs, [1], lf0) f0s[:, 1] = np.exp(lf0) winlenmax = getwinlen(np.min(f0s[:, 1]), fs, winnbper) if winlenmax > dftlen: warnings.warn( '\n\nWARNING: The maximum window length ({}) is bigger than the DFT length ({}). Please, increase the DFT length of your spectral features (the second dimension) or check if the f0 curve has extremly low values and try to clip them to higher values (at least higher than 50Hz). The f0 curve has been clipped to {}Hz.\n\n' .format(winlenmax, dftlen, winnbper * fs / float(dftlen))) # pragma: no cover f0s[:, 1] = np.clip(f0s[:, 1], winnbper * fs / float(dftlen - 2), 1e6) if not NM is None: # Remove noise below f0, as it is supposed to be already the case for n in range(NM.shape[0]): NM[n, :int((float(dftlen) / fs) * 2 * f0s[n, 1])] = 0.0 if not nm_cont: print(' Forcing binary noise mask') NM[NM <= 0.5] = 0.0 # To be sure that voiced segments are not hoarse NM[NM > 0.5] = 1.0 # To be sure the noise segments are fully noisy # Generate the pulse positions [1](2) (i.e. the synthesis instants, the GCIs in voiced segments) ts = [0.0] while ts[-1] < float(wavlen) / fs: cf0 = np.interp(ts[-1], f0s[:, 0], f0s[:, 1]) if cf0 < 50.0: cf0 = 50 ts.append(ts[-1] + (1.0 / cf0)) ts = np.array(ts) f0s = np.vstack((ts, np.interp(ts, f0s[:, 0], f0s[:, 1]))).T # Resample the features to the pulse positions # Spectral envelope uses the nearest, to avoid over-smoothing SPECR = np.zeros((f0s.shape[0], dftlen / 2 + 1)) for n, t in enumerate(f0s[:, 0]): # Nearest: Way better for plosives idx = int(np.round(t / shift)) idx = np.clip(idx, 0, SPEC.shape[0] - 1) SPECR[n, :] = SPEC[idx, :] # Keep trace of the median energy [dB] over the whole signal ener = np.mean(SPECR, axis=1) idxacs = np.where(sp.mag2db(ener) > sp.mag2db(np.max(ener)) - 30)[0] # Get approx active frames # TODO Param enermed = sp.mag2db(np.median(ener[idxacs])) # Median energy [dB] ener = sp.mag2db(ener) # Resample the noise feature to the pulse positions # Smooth the frequency response of the mask in order to avoid Gibbs # (poor Gibbs nobody want to see him) nm_lowpasswin = np.hanning(nm_lowpasswinlen) nm_lowpasswin /= np.sum(nm_lowpasswin) NMR = np.zeros((f0s.shape[0], dftlen / 2 + 1)) for n, t in enumerate(f0s[:, 0]): idx = int(np.round(t / shift)) # Nearest is better for plosives idx = np.clip(idx, 0, NM.shape[0] - 1) NMR[n, :] = NM[idx, :] if nm_lowpasswinlen > 1: NMR[n, :] = scipy.signal.filtfilt(nm_lowpasswin, [1.0], NMR[n, :]) NMR = np.clip(NMR, 0.0, 1.0) # The complete waveform that we will fill with the pulses wav = np.zeros(wavlen) # Half window on the left of the synthesized segment to avoid pre-echo dampinhwin = np.hanning( 1 + 2 * int(np.round(antipreechohwindur * fs))) # 1ms forced dampingwindow dampinhwin = dampinhwin[:(len(dampinhwin) - 1) / 2 + 1] for n, t in enumerate(f0s[:, 0]): f0 = f0s[n, 1] if verbose > 1: print "\rPM Synthesis (python) t={:4.3f}s f0={:3.3f}Hz ".format( t, f0), # Window's length # TODO It should be ensured that the beggining and end of the # noise is within the window. Nothing is doing this currently! winlen = getwinlen(f0, fs, winnbper) # TODO We also assume that the VTF's decay is shorter # than winnbper-1 periods (dangerous with high pitched and tense voice). if winlen > dftlen: raise ValueError( 'The window length ({}) is bigger than the DFT length ({}). Please, increase the dftlen of your spectral features or check if the f0 curve has extremly low values and try to clip them to higher values (at least higher than 50[Hz])' .format(winlen, dftlen)) # pragma: no cover # Set the rough position of the pulse in the window (the closest sample) # We keep a third of the window (1 period) on the left because the # pulse signal is minimum phase. And 2/3rd (remaining 2 periods) # on the right to let the VTF decay. pulseposinwin = int((1.0 / winnbper) * winlen) # The sample indices of the current pulse wrt. the final waveform winidx = int(round(fs * t)) + np.arange(winlen) - pulseposinwin # Build the pulse spectrum # Let start with a Dirac S = np.ones(dftlen / 2 + 1, dtype=np.complex64) # Add the delay to place the Dirac at the "GCI": exp(-j*2*pi*t_i) delay = -pulseposinwin - fs * (t - int(round(fs * t)) / float(fs)) S *= np.exp((delay * 2j * np.pi / dftlen) * np.arange(dftlen / 2 + 1)) # Add the spectral envelope # Both amplitude and phase E = SPECR[n, :] # Take the amplitude from the given one if hp_f0coef != None: # High-pass it to avoid any residual DC component. fcut = hp_f0coef * f0 if not pp_atten1stharminsilences is None and ener[ n] - enermed < pp_atten1stharminsilences: fcut = 1.5 * f0 # Try to cut between first and second harm HP = sp.butter2hspec(fcut, 4, fs, dftlen, high=True) E *= HP # Not necessarily good as it is non-causal, so make it causal... # ... together with the VTF response below. # Build the phase of the envelope from the amplitude E = sp.hspec2minphasehspec(E, replacezero=True) # We spend 2 FFT here! S *= E # Add it to the current pulse # Add energy correction wrt f0. # STRAIGHT and AHOCODER vocoders do it. # (why ? to equalize the energy when changing the pulse's duration ?) if ener_multT0: S *= np.sqrt(fs / f0) # Generate the segment of Gaussian noise # Use mid-points before/after pulse position if n > 0: leftbnd = int(np.round(fs * 0.5 * (f0s[n - 1, 0] + t))) else: leftbnd = int(np.round(fs * (t - 0.5 / f0s[n, 1]))) # int(0) if n < f0s.shape[0] - 1: rightbnd = int(np.round(fs * 0.5 * (t + f0s[n + 1, 0]))) - 1 else: rightbnd = int(np.round( fs * (t + 0.5 / f0s[n, 1]))) #rightbnd=int(wavlen-1) gausswinlen = rightbnd - leftbnd # The length of the noise segment gaussnoise4win = np.random.normal(size=(gausswinlen)) # The noise GN = np.fft.rfft(gaussnoise4win, dftlen) # Move the noise to freq domain # Normalize it by its energy (@Yannis, That's your answer at SSW9!) GN /= np.sqrt(np.mean(np.abs(GN)**2)) # Place the noise within the pulse's window delay = (pulseposinwin - (leftbnd - winidx[0])) GN *= np.exp((delay * 2j * np.pi / dftlen) * np.arange(dftlen / 2 + 1)) # Add it to the pulse spectrum, under the condition of the mask S *= GN**NMR[n, :] # That's it! the pulse spectrum is ready! # Move it to time domain deter = np.fft.irfft(S)[0:winlen] # Add half window on the left of the synthesized segment # to avoid any possible pre-echo deter[:leftbnd - winidx[0] - len(dampinhwin)] = 0.0 deter[leftbnd - winidx[0] - len(dampinhwin):leftbnd - winidx[0]] *= dampinhwin # Add half window on the right # to avoid cutting the VTF response abruptly deter[-len(dampinhwin):] *= dampinhwin[::-1] # Write the synthesized segment in the final waveform if winidx[0] < 0 or winidx[-1] >= wavlen: # The window is partly outside of the waveform ... # ... thus copy only the existing part itouse = np.logical_and(winidx >= 0, winidx < wavlen) wav[winidx[itouse]] += deter[itouse] else: wav[winidx] += deter if verbose > 1: print '\r \r', if verbose > 2: # pragma: no cover import matplotlib.pyplot as plt plt.ion() _, axs = plt.subplots(3, 1, sharex=True, sharey=False) times = np.arange(len(wav)) / float(fs) axs[0].plot(times, wav, 'k') axs[0].set_ylabel('Waveform\nAmplitude') axs[0].grid() axs[1].plot(f0s[:, 0], f0s[:, 1], 'k') axs[1].set_ylabel('F0\nFrequency [Hz]') axs[1].grid() axs[2].imshow(sp.mag2db(SPEC).T, origin='lower', aspect='auto', interpolation='none', extent=(f0s[0, 0], f0s[-1, 0], 0, 0.5 * fs)) axs[2].set_ylabel('Amp. Envelope\nFrequency [Hz]') from IPython.core.debugger import Pdb Pdb().set_trace() return wav
def process_image(self, index=None): if index is None: index = self.ui.file_listview.selectedIndexes() try: logger.info("Processing image {0}, {1}".format(index.row(), index.data())) except AttributeError: il = self.ui.file_listview.selectedIndexes() if len(il) > 0: index = self.ui.file_listview.selectedIndexes()[0] else: self.ui.image_size_label.setText("ERROR") self.ui.moment_1_label.setText("-.--") self.ui.moment_2_label.setText("-.--") return try: # Load selected image pic = np.load("{0}/data/{1}".format(QtCore.QDir.currentPath(), index.data())) except ValueError: self.ui.image_size_label.setText("ERROR") self.ui.moment_1_label.setText("-.--") self.ui.moment_2_label.setText("-.--") return self.ui.image_size_label.setText("{0} x {1}".format(pic.shape[1], pic.shape[0])) # logger.debug("Image size: {0} x {1}".format(pic.shape[0], pic.shape[1])) # Large ROI around spot, the spot should always be inside roi_t = self.ui.roi_top_spinbox.value() roi_h = self.ui.roi_height_spinbox.value() roi_l = self.ui.roi_left_spinbox.value() roi_w = self.ui.roi_width_spinbox.value() kernel = self.ui.medfilt_spinbox.value() if kernel % 2 == 0: kernel += 1 pic_roi = np.double(pic[roi_t:roi_t + roi_h, roi_l:roi_l + roi_w]) pic_proc = medfilt2d(np.double(pic[roi_t:roi_t+roi_h, roi_l:roi_l+roi_w]), kernel) # Background level from first 20 columns, one level for each row (the background structure is banded): bkg_level = pic_proc[:, 0:20].mean(1) self.ui.bkg_label.setText("{0:.1f}".format(bkg_level.mean())) pic_proc = np.maximum(0, pic_proc - 1.1 * bkg_level[:, np.newaxis]) dx = 13e-6 dx = 1 x = dx * np.arange(pic_proc.shape[1]) x0 = ((pic_proc * x[np.newaxis, :]).sum(1) / pic_proc.sum(1)) x1 = (pic_proc * (x[np.newaxis, :] - x0[:, np.newaxis]) ** 2).sum(1) / pic_proc.sum(1) x0_good = x0[~np.isnan(x0)] x1_good = x1[~np.isnan(x1)] # xc = np.median(x0_good) # xs = np.median(x1_good) # Assume spot is in the central +-50 vertical pixels of the ROI. Get the index for the maximum value xc = pic_proc[pic_proc.shape[0]//2 - 50:pic_proc.shape[0]//2 + 50, :].sum(0).argmax() xs = 10 logger.debug("x0: {0}, x1: {1}".format(x0_good.shape, x1_good.shape)) logger.debug("xc: {0}, xs: {1}".format(xc, xs)) # Cut an new ROI around the central pixel column xc, width xs * 10 pic_roi2 = pic_roi[:, np.maximum(0, int(xc - xs * 5)):np.minimum(pic_roi.shape[1]-1, int(xc + xs * 5))] pic_proc2 = medfilt2d(np.double(pic_roi2), kernel) # Background level to cut in the reduced ROI: bkg_cut = self.ui.bkg_spinbox.value() pic_proc2 = np.maximum(0, pic_proc2 - bkg_cut) # Create mask around the signal spot by heavily median filtering in the vertical direction (mask_kern ~ 25) mask_kern = self.ui.mask_spinbox.value() if mask_kern % 2 == 0: mask_kern += 1 mask = medfilt(np.maximum(0, pic_roi2 - bkg_cut), [mask_kern, kernel]) pic_proc3 = pic_proc2 * (mask > 0) xt = dx * np.arange(pic_proc3.shape[1]) xt0 = ((pic_proc3 * xt[np.newaxis, :]).sum(1) / pic_proc3.sum(1)) xt1 = (pic_proc3 * (xt[np.newaxis, :] - xt0[:, np.newaxis]) ** 2).sum(1) / pic_proc3.sum(1) xt0_good = xt0[~np.isnan(xt0)] xt1_good = xt1[~np.isnan(xt1)] ind = ~np.isnan(xt1) charge = pic_proc3.sum() xt1_w = (xt1[ind] * pic_proc3[ind, :].sum(1)).sum() / charge self.ui.moment_1_label.setText("{0:.1f} pixels".format(xt1_w)) self.ui.moment_2_label.setText("{0:.1f} pixels".format(xt1_good.mean())) self.ui.charge_label.setText("{0}".format(charge)) if self.ui.processed_radiobutton.isChecked(): self.ui.image_widget.setImage(pic_proc.transpose(), autoLevels=True) elif self.ui.tight_radiobutton.isChecked(): try: self.ui.image_widget.setImage(pic_proc3.transpose(), autoLevels=True) except ValueError: logger.error("pic_proc3 shape {0}".format(pic_proc3.shape)) self.ui.image_widget.setImage(pic_roi2.transpose(), autoLevels=True) else: self.ui.image_widget.setImage(pic_roi.transpose(), autoLevels=False)
def main(parser): # print(bcolors.HEADER + "*** Conduction Velocity Analyzer ***" + bcolors.ENDC) # on windows cmd colors don't work print('\n***************************************') print('*** Conduction Velocity Analyzer ******') print('\n***************************************') # parse arguments args = parser.parse_args() txt_path = args.input_filepath[0] # extract filename and directory path filename = os.path.basename(txt_path) folderpath = os.path.dirname(txt_path) # extract experiment parameters from filename and print info to console burst, param = extract_info(filename, _print=True) # create excel file wb, sheet, out_path, n_col_sheet_block = prepare_excel_wb( folderpath, filename, burst, param) # read OM tracks values values = np.loadtxt(txt_path, dtype=np.float, usecols=(0, 1, 2)) param[pKeys.exp_duration] = values.shape[0] / param[pKeys.frame_rate] print("- Duration of record : {} ms".format(param[pKeys.exp_duration])) # split timing, roi1 (apex) and roi2 (base) values full_ms, full_apex, full_base = values[:, 0], values[:, 1], values[:, 2] ''' ===== [ START ANALISIS ] ===== - for each burst ''' for (i_burst, b) in enumerate(burst): print('\n\n********* Analyzing Burst at {}Hz *********'.format( b[bKeys.freq_stim])) # extract ramp ms = full_ms[b[bKeys.start_ms]:b[bKeys.stop_ms]] apex = full_apex[b[bKeys.start_ms]:b[bKeys.stop_ms]] base = full_base[b[bKeys.start_ms]:b[bKeys.stop_ms]] # plot_signals(apex, base, title="Ramp at {}Hz".format(b[keys.freq_stim])) # detrend signals apex_flat = sign.detrend(apex, type='linear') base_flat = sign.detrend(base, type='linear') # plot original and flattened # plot_signals(apex, base, title="Original Tracks") # plot_signals(apex_flat, base_flat, title="Detrended Tracks") # if selected, apply median filter and plot results if param[pKeys.rank] is not 0: apex_filt = sign.medfilt(apex_flat, kernel_size=param[pKeys.rank]) base_filt = sign.medfilt(base_flat, kernel_size=param[pKeys.rank]) # plot_signals(apex_filt, base_filt, title="Filtered Tracks (rank = {})".format(param[keys.rank])) # ENSURE TO USE THE RIGHT SIGNAL (filtered or only flattened) (apex_sgn, base_sgn) = (apex_filt, base_filt) if param[pKeys.rank] is not 0 else (apex_flat, base_flat) ''' ===== [ FIND PEAKS, PERIODS and SELECT AP TO ANALYZE ] ===== ''' # find peaks - restituisce (x, y) dei picchi # -> io passo il segnale invertito (perchè usa il 'max') # impongo che i picchi siano distanti fra loro almeno più di 2/3 del periodo # e prendo solo la x (tempo): [0]-> prendi solo la x del picco a_peaks = sign.find_peaks(-apex_sgn, distance=(2 / 3) * b[bKeys.AP_duration])[0] b_peaks = sign.find_peaks(-base_sgn, distance=(2 / 3) * b[bKeys.AP_duration])[0] # plotto i segnali con picchi e durate dei periodi plot_signals_with_peaks([apex_sgn, base_sgn], [a_peaks, b_peaks], ['Apex', 'Base'], "Ramp at {}Hz".format(b[bKeys.freq_stim]), _enum_peaks=True, _plot_intervals=True, frame_rate=param[pKeys.frame_rate]) # stima durata di ogni AP da differenza picchi consecutivi nell'apice print("- Control of stimulation frequency in the apex signal... ") AP_periods = np.diff(a_peaks) / param[pKeys.frame_rate] freq_stim_estimated = 1 / np.mean(AP_periods / 1000) print("-- Stimulation Frequency obtained: {0:0.1f}Hz".format( freq_stim_estimated)) # user can select which potentials use to estimmate mean conduction velocity selection = input( "\n***AP selection to estimate mean current velocity.\n" " Insert AP indexex between spaces. Example:\n" " 0 2 3 5 7 8 9 [<- then press 'Enter'].\n" " Please, enter your selection here and the press 'Enter': \n") # extract selected indexes ap_selected_idx = selection.split(' ') print('- AP selected for Conduction Velocity Estimation: ') for l in ap_selected_idx: print(l, end='°, ') ''' ===== [ ANALYZE EACH ACTION POTENTIALS TO FIND DELAY APEX-BASE ] ===== ''' # USE INTERPOLATION AND CROSS-CORRELATION TO ESTIMATE DELAY # from selected AP potentials # Idea: per ogni picco, prendo l'intorno giusto per selezionare il potenziale d'azione, # interpolo, calcolo delay usando il picco della cross-correlazione e poi medio tutti i delay cv_list = list() # list of conduction velocity extracted for each AP # for each AP selected for (i_spike, spike_num) in enumerate( np.asarray(ap_selected_idx).astype(np.int)): # calculate extremes of selected action potential signal t1, t2 = np.int(b[bKeys.AP_duration] * spike_num), np.int( b[bKeys.AP_duration] * (spike_num + 1)) ms_sel = ms[t1:t2] # time base_sel = base_sgn[t1:t2] # base apex_sel = apex_sgn[t1:t2] # apex # interpolo i due segnali di fattore 0.2 (come su LabView) dt = 0.2 # calcolo funzione di interpolazione delle due tracce f_apex = interpolate.interp1d(ms_sel, apex_sel) f_base = interpolate.interp1d(ms_sel, base_sel) # creo nuovo asse temporale ms_res = np.arange(ms_sel[0], ms_sel[-1], dt) # resample signals using interpolation functions calculated above apex_res = f_apex(ms_res) base_res = f_base(ms_res) # estimate delay by cross-correlation max values delay_ms = lag_finder_in_ms(apex_res, base_res, 1000 / dt) # estimate and save conduction velocity cv = param[pKeys.ROI_distance_mm] / delay_ms cv_list.append(cv) # write spike_num and cv sheet.write(i_spike + 7, (i_burst * (n_col_sheet_block + 1)) + 1, "{}".format(int(spike_num))) sheet.write(i_spike + 7, (i_burst * (n_col_sheet_block + 1)) + 2, "{0:0.3f}".format(cv)) # estimate mean and std. error avg = np.mean(np.asarray(cv_list)) sem = stats.sem(np.asarray(cv_list)) # write mean and std. error into excel file sheet.write(17, (i_burst * (n_col_sheet_block + 1)) + 2, "{0:0.3f}".format(avg)) sheet.write(18, (i_burst * (n_col_sheet_block + 1)) + 2, "{0:0.3f}".format(sem)) # print results and average print("*** RESULTS:") print("- Conduction velocities in m/s:") for cv in cv_list: print("-- {0:0.3f} m/s".format(cv)) print("\n- Average Conduction velocity: ", end='') print("-- {0:0.3f} +- {1:0.3f} m/s".format(avg, sem)) # save excel file wb.save(out_path) print('\nOutput saved in:') print(out_path) print( ' --------- Conduction Velocity Analyzer: Process finished. ---------\n' )
local_file="env_data/msd/flow-2018.csv", url= "http://wdl.water.ca.gov/waterdatalibrary/docs/Hydstra/docs/B95820Q/2018/FLOW_15-MINUTE_DATA_DATA.CSV", skiprows=3, parse_dates=['time'], names=['time', 'flow_cfs', 'quality', 'notes']) ax2.plot(msd_flow.time, msd_flow.flow_cfs, label='MSD Flow (cfs)') ax.axis(xmin=times.min(), xmax=times.max()) msd_turb = cdec.cdec_dataset('MSD', times.min(), times.max(), sensor=27, cache_dir='env_data') msd_turb['turb_lp'] = ('time', ), signal.medfilt(msd_turb.sensor0027.values, 5) msd_tempF = cdec.cdec_dataset('MSD', times.min(), times.max(), sensor=25, cache_dir='env_data') ax_turb = ax2.twinx() ax_turb.plot(msd_turb.time, msd_turb.sensor0027, label='MSD Turb. (NTU)', color='orange', alpha=0.2, lw=0.4) ax_turb.plot(msd_turb.time,
line_count += 1 else: kls.append(row[kli]) recs.append(row[reci]) for n, idx in enumerate(zidx): zs[n].append(row[idx]) line_count += 1 if kls == []: print('{} had no progress, skipping'.format(model_name)) continue kls = medfilt(np.asarray(kls[100:], dtype=np.float32), 19) recs = medfilt(np.asarray(recs[100:], dtype=np.float32), 19) plt.figure(figsize=(10, 10)) plt.subplot(221) data = load_uniform_pendulum() v = VAE(load_from=model_name, network=network) mus, logvars = v.encode(data[:4000])[:2] sigmas = np.mean(np.exp(0.5 * logvars), axis=0) bars(d, sigmas, plt_shape, type='sigma', title=model_name) plt.legend()
def plot_output(x_test, y_test, model, median_filtering=False, kernel_size=31, x_lim=None): score = model.evaluate(x_test, y_test, verbose=2) predictions = model.predict(x_test) # print('Test score:', score[0]) # print('Test accuracy:', score[1]) ########### 2 class predictions ##################################################### positive_predictions = predictions[:, 0][np.where(y_test[:, 0])] negative_predictions = predictions[:, 1][np.where(y_test[:, 1])] if median_filtering: filt_predictions = np.zeros(np.shape(predictions)) predictions[:, 0] = medfilt(predictions[:, 0], kernel_size=kernel_size) predictions[:, 1] = 1 - filt_predictions[:, 0] positive_predictions = predictions[:, 0][np.where(y_test[:, 0])] negative_predictions = predictions[:, 1][np.where(y_test[:, 1])] true_positive_rate = (sum(np.round(positive_predictions))) / sum(y_test[:, 0]) true_negative_rate = sum(np.round(negative_predictions)) / sum(y_test[:, 1]) figs = [] f = plt.figure(figsize=(12, 6)) plt.plot(predictions[:, 0], 'g.', markersize=12, label='y_pred') plt.plot(y_test[:, 0], '--b', linewidth=1, markersize=2, label='y_test') plt.legend(loc=9, ncol=2) plt.ylim([-0.1, 1.4]) plt.yticks([0, 0.2, 0.4, 0.6, 0.8, 1.0]) if x_lim: plt.xlim([x_lim[0], x_lim[1]]) plt.ylabel('Classifier output') plt.xlabel('Signal window number') # if save_fig == 'publication_softmax': # plt.tight_layout() # plt.savefig('../../../TexFiles/Papers/ECML/Images/softmax_' + model_name + '.pdf') # if save_fig_individual: # plt.savefig('Outputs/' + 'solo_softmax_' + model_name + '.pdf') # figs.append(f) print 'True positive rate', true_positive_rate, 'True negative rate', true_negative_rate #plt.savefig('Outputs/' + 'ClassOutput_' + model_name + '.pdf', transparent = True) #print 'saved as', 'ClassOutput_' + model_name + '.pdf' #plt.show() cnf_matrix = confusion_matrix(y_test[:, 1], np.round(predictions[:, 1]).astype(int)) f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 6)) y_true = y_test[:, 0] y_score = predictions[:, 0] roc_score = roc_auc_score(y_true, y_score) fpr, tpr, thresholds = roc_curve(y_true, y_score) #plt.subplot(1,2,2) #plt.figure(figsize=(4,4)) ax1.plot(fpr, tpr, '.-') ax1.plot([0, 1], [0, 1], 'k--') ax1.set_xlim([-0.01, 1.01]) ax1.set_ylim([-0.01, 1.01]) ax1.set_yticks([0.2, 0.4, 0.6, 0.8, 1.0]) ax1.set_xlabel('False positive rate') ax1.set_ylabel('True positive rate') ax1.set_title('ROC, area = %.4f' % roc_score) y_test_pr = y_test[:, 0] precision = dict() recall = dict() average_precision = dict() for i in range(1): precision[i], recall[i], _ = precision_recall_curve(y_test_pr, y_score) average_precision[i] = average_precision_score(y_test_pr, y_score) # Plot Precision-Recall curve #plt.clf() ax2.plot(recall[0], precision[0], color='b', label='Precision-Recall curve') ax2.set_xlabel('Recall') ax2.set_ylabel('Precision') ax2.set_ylim([0.0, 1.05]) ax2.set_xlim([0.0, 1.0]) ax2.set_yticks([0.2, 0.4, 0.6, 0.8, 1.0]) ax2.set_title('Precision-Recall, area = {0:0.3f}'.format( average_precision[0])) #plt.legend(loc="lower left") # if save_fig == 'publication': # plt.tight_layout() # plt.savefig('/Outputs/Papers/ECML/Images/metric_' + model_name + '.pdf') # plt.show() F1 = f1_score(np.round(predictions[:, 0]), y_test[:, 0], average='binary') print 'F1 score', F1 perf_metrics = { "tpr": true_positive_rate, "tnr": true_negative_rate, "f1": F1, "roc": roc_score, "pr": average_precision[0], "conf_matrix": cnf_matrix } return predictions, perf_metrics
def load_signal(DS, winL, winR, do_preprocess): class_ID = [[] for i in range(len(DS))] beat = [[] for i in range(len(DS))] # record, beat, lead R_poses = [np.array([]) for i in range(len(DS))] Original_R_poses = [np.array([]) for i in range(len(DS))] valid_R = [np.array([]) for i in range(len(DS))] my_db = mit_db() patients = [] # Lists # beats = [] # classes = [] # valid_R = np.empty([]) # R_poses = np.empty([]) # Original_R_poses = np.empty([]) size_RR_max = 20 pathDB = '/home/mondejar/dataset/ECG/' DB_name = 'mitdb' fs = 360 jump_lines = 1 # Read files: signal (.csv ) annotations (.txt) fRecords = list() fAnnotations = list() lst = os.listdir(pathDB + DB_name + "/csv") lst.sort() for file in lst: if file.endswith(".csv"): if int(file[0:3]) in DS: fRecords.append(file) elif file.endswith(".txt"): if int(file[0:3]) in DS: fAnnotations.append(file) MITBIH_classes = [ 'N', 'L', 'R', 'e', 'j', 'A', 'a', 'J', 'S', 'V', 'E', 'F' ] #, 'P', '/', 'f', 'u'] AAMI_classes = [] AAMI_classes.append(['N', 'L', 'R']) # N AAMI_classes.append(['A', 'a', 'J', 'S', 'e', 'j']) # SVEB AAMI_classes.append(['V', 'E']) # VEB AAMI_classes.append(['F']) # F #AAMI_classes.append(['P', '/', 'f', 'u']) # Q RAW_signals = [] r_index = 0 #for r, a in zip(fRecords, fAnnotations): for r in range(0, len(fRecords)): print("Processing signal " + str(r) + " / " + str(len(fRecords)) + "...") # 1. Read signalR_poses filename = pathDB + DB_name + "/csv/" + fRecords[r] print(filename) f = open(filename, 'rb') reader = csv.reader(f, delimiter=',') next(reader) # skip first line! MLII_index = 1 V1_index = 2 if int(fRecords[r][0:3]) == 114: MLII_index = 2 V1_index = 1 MLII = [] V1 = [] for row in reader: MLII.append((int(row[MLII_index]))) V1.append((int(row[V1_index]))) f.close() RAW_signals.append( (MLII, V1) ) ## NOTE a copy must be created in order to preserve the original signal # display_signal(MLII) # 2. Read annotations filename = pathDB + DB_name + "/csv/" + fAnnotations[r] print(filename) f = open(filename, 'rb') next(f) # skip first line! annotations = [] for line in f: annotations.append(line) f.close # 3. Preprocessing signal! if do_preprocess: #scipy.signal # median_filter1D baseline = medfilt(MLII, 71) baseline = medfilt(baseline, 215) # Remove Baseline for i in range(0, len(MLII)): MLII[i] = MLII[i] - baseline[i] # TODO Remove High Freqs # median_filter1D baseline = medfilt(V1, 71) baseline = medfilt(baseline, 215) # Remove Baseline for i in range(0, len(V1)): V1[i] = V1[i] - baseline[i] # Extract the R-peaks from annotations for a in annotations: aS = a.split() pos = int(aS[1]) originalPos = int(aS[1]) classAnttd = aS[2] if pos > size_RR_max and pos < (len(MLII) - size_RR_max): index, value = max(enumerate(MLII[pos - size_RR_max:pos + size_RR_max]), key=operator.itemgetter(1)) pos = (pos - size_RR_max) + index peak_type = 0 #pos = pos-1 if classAnttd in MITBIH_classes: if (pos > winL and pos < (len(MLII) - winR)): beat[r].append((MLII[pos - winL:pos + winR], V1[pos - winL:pos + winR])) for i in range(0, len(AAMI_classes)): if classAnttd in AAMI_classes[i]: class_AAMI = i break #exit loop #convert class class_ID[r].append(class_AAMI) valid_R[r] = np.append(valid_R[r], 1) else: valid_R[r] = np.append(valid_R[r], 0) else: valid_R[r] = np.append(valid_R[r], 0) R_poses[r] = np.append(R_poses[r], pos) Original_R_poses[r] = np.append(Original_R_poses[r], originalPos) #R_poses[r] = R_poses[r][(valid_R[r] == 1)] #Original_R_poses[r] = Original_R_poses[r][(valid_R[r] == 1)] # Set the data into a bigger struct that keep all the records! my_db.filename = fRecords my_db.raw_signal = RAW_signals my_db.beat = beat # record, beat, lead my_db.class_ID = class_ID my_db.valid_R = valid_R my_db.R_pos = R_poses my_db.orig_R_pos = Original_R_poses return my_db
# In[13]: # BF_imgs=np.zeros((masterdark.shape[0],masterdark.shape[1],len(base_frames_fullnames))) BF_imgs = [] for i in range(len(base_frames_fullnames)): # sys.stdout.write('\r') # sys.stdout.write("Add base frame "+str(i+1)+"/"+str(len(base_frames_fullnames))) # sys.stdout.flush() bf_fname = base_frames_fullnames[i] hdulist = fits.open(bf_fname, ignore_missing_end=True) # BF_imgs[:,:,i]=hdulist[0].data.astype('float') BF_imgs.append(hdulist[0].data.astype('float')) hdulist.close() BF_imgs[-1] = (BF_imgs[-1] - masterdark.astype('float')) BF_imgs[-1] = ss.medfilt(BF_imgs[-1], kernel_size=avr_width1) # In[14]: # YPIX, XPIX = np.mgrid[1:BF_imgs[:,:,0].shape[0]+1, 1:BF_imgs[:,:,0].shape[1]+1] YPIX, XPIX = np.mgrid[1:BF_imgs[0].shape[0] + 1, 1:BF_imgs[0].shape[1] + 1] # In[15]: err, az0, alt0, a, b, c, d = get_solve_pars(solve_pars_fname) # In[16]: #az_c, alt_c = arc_pix2hor(255,255,az0,alt0,a,b) #print(az_c*180/np.pi,alt_c*180/np.pi) # 292.814270937 83.6690150447
def crop_video(opt, track, cropfile): cap = cv2.VideoCapture( os.path.join(opt.avi_dir, opt.reference, 'video.avi')) total_frames = cap.get(7) cap.set(1, track[0][0]) # CHANGE THIS !!! fourcc = cv2.VideoWriter_fourcc(*'XVID') vOut = cv2.VideoWriter(cropfile + 't.avi', fourcc, cap.get(5), (224, 224)) fw = cap.get(3) fh = cap.get(4) dets = [[], [], []] for det in track[1]: dets[0].append( ((det[3] - det[1]) * fw + (det[2] - det[0]) * fh) / 4) # H+W / 4 dets[1].append((det[1] + det[3]) * fw / 2) # crop center x dets[2].append((det[0] + det[2]) * fh / 2) # crop center y # Smooth detections dets[0] = signal.medfilt(dets[0], kernel_size=5) dets[1] = signal.medfilt(dets[1], kernel_size=5) dets[2] = signal.medfilt(dets[2], kernel_size=7) for det in zip(*dets): cs = opt.crop_scale bs = det[0] # Detection box size bsi = int(bs * (1 + 2 * cs)) # Pad videos by this amount ret, frame = cap.read() frame = np.pad(frame, ((bsi, bsi), (bsi, bsi), (0, 0)), 'constant', constant_values=(0, 0)) my = det[2] + bsi # BBox center Y mx = det[1] + bsi # BBox center X face = frame[int(my - bs):int(my + bs * (1 + 2 * cs)), int(mx - bs * (1 + cs)):int(mx + bs * (1 + cs))] vOut.write(cv2.resize(face, (224, 224))) audiotmp = os.path.join(opt.tmp_dir, opt.reference, 'audio.wav') audiostart = track[0][0] / cap.get(5) audioend = (track[0][-1] + 1) / cap.get(5) cap.release() vOut.release() # ========== CROP AUDIO FILE ========== command = ( "ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 -ss %.3f -to %.3f %s -loglevel quiet" % (os.path.join(opt.avi_dir, opt.reference, 'video.avi'), audiostart, audioend, audiotmp)) # -async 1 output = subprocess.call(command, shell=True, stdout=None) if output != 0: pdb.set_trace() sample_rate, audio = wavfile.read(audiotmp) # ========== COMBINE AUDIO AND VIDEO FILES ========== command = ( "ffmpeg -y -i %st.avi -i %s -c:v copy -c:a copy %s.avi -loglevel quiet" % (cropfile, audiotmp, cropfile)) # -async 1 output = subprocess.call(command, shell=True, stdout=None) if output != 0: pdb.set_trace() print('Written %s' % cropfile) os.remove(cropfile + 't.avi') return [track, dets]
def smooth(flux): return medfilt(flux, kernel_size=7)
usecols=(0, 1), encoding=None, skip_footer=data.size) for entry in pte_entries: get_cachelines(entry[0], int(entry[1])) print "\nSLAT cacheline candidates:" pprint.pprint(set(cachelines)) # Read time values for each cachelines into a dictionary of {cacheline:[timings]} cache_n_timing = {} cache_list = [] for cacheline in range(ncacheline): sample = data[cacheline * nrounds:(cacheline + 1) * nrounds] sample = medfilt(sample) sample = savgol_filter(sample, nrounds - 1, 0) cache_list.append(sample) avg = sum(sample.tolist()) / nrounds cache_n_timing[cacheline] = avg g_index = [s for s in range(ncacheline)] g_cols = [r for r in range(nrounds)] df = DataFrame(cache_list, index=g_index, columns=g_cols) plt.pcolor(df) plt.savefig("results/slatfilter.png") sorted_cache_score = sorted(cache_n_timing.items(), key=operator.itemgetter(1))[::-1]
def dissectData(data, onbodywindow=2, thrsd=0.1, thrrange=0.5, samplerate=50): ''' for a given datastream, identify on-body segments [and maybe pickle them input: pandas dataframe of data with accelerometer in columns named x,y,z window size in minutes returns number of periods dict ''' tt = data.copy() # tt= returndf.copy()[30000:-60000] tt = tt[['x', 'y', 'z']] # tt.columns = ['x','y','z'] if tt.isnull().sum().sum() != 0: print("nulls in data - I can't handle that ") ar = tt[['x', 'y', 'z']].values smoothedx = medfilt(ar[:, 0], 3) smoothedy = medfilt(ar[:, 1], 3) smoothedz = medfilt(ar[:, 2], 3) #tt['sx','sy','sz'] =smoothed[:,0:2] tt['sx'] = smoothedx tt['sy'] = smoothedy tt['sz'] = smoothedz # tt.plot() #np.std(tt.z.values) #tt.describe() tt.drop(['x', 'y', 'z'], axis=1, inplace=True) #tt.plot() #samplerate=50 offBodySegments, allsds = getOffBody( onbodywindow, tt, samplerate, thrsd, thrrange) # x minute mindow, 50 samples/second #plt.plot(np.asarray(list(offBodySegments.values()))) #plt.show() #offBodySegments = offBodyPeriods.copy() offBodyar = np.asarray(list(offBodySegments.values())) # get periods of segments from ' off body' dict #only interested in > 10 minutes of wear time/offtime, so 5 consecutive zeros or # TODO 5 consecutive ones #offBodySegments = offBodyPeriods.copy() runsize = 5 periodcounter = 0 periods = {} # generate dict of periods of on body - runs of 0's switch = np.sign( offBodyar[:runsize].sum()) # initial state 0 if 0 , 1 if > 0 if switch == 0: startrun = 0 for i in np.arange(len(offBodyar) - runsize): runsum = offBodyar[i:i + runsize].sum() # at beginning , record beginning oldswitch = switch switch = np.sign(runsum) # either 0 or > 0 if oldswitch != switch: # state transition if switch == 0: # start of new period print('start new period ', i) startrun = i # all elements must be zero else: # end of period switch from 0 to 1 - on to off body endrun = i + runsize - 2 # the first runsize-1 values are actualy 0 print('end period ', endrun) periods[periodcounter] = [startrun, endrun] periodcounter += 1 if (i + runsize == len(offBodyar) - 1) & (switch == 0): # end of array while in 0 print('end of array processing', i + runsize, len(offBodyar) - 1, switch) endrun = i + runsize periods[periodcounter] = [startrun, endrun] ws = 2 # minutes samplesize = int(ws * samplerate * 60) def picklePeriods(data, periods, samplesize, fn): import pickle import os epoch = {} #os.getcwd() for period in periods: length = periods[period][1] - periods[period][0] + 1 length *= samplesize start = samplesize * periods[period][0] epoch[period] = data[start:start + length] pickle.dump(epoch[period], open(fn + '_' + str(period), 'wb')) return #fn='userid-date-device' #picklePeriods(data, periods, samplesize,fn) # import pdb #pdb.set_trace() return periods, offBodySegments
for j in range(0, len(start_lable)): lens = 0.0 distance = [] print(start_lable[j]) print(end_lable[j]) print('s', end_lable[j] - start_lable[j]) for i in range(start_lable[j], end_lable[j]): x = data_x[i + 1] - data_x[i] y = data_y[i + 1] - data_y[i] z = data_z[i + 1] - data_z[i] # 用math.sqrt()求平方根 lens = math.sqrt((x**2) + (y**2) + (z**2)) # print('累加距离为',lens)1 print(lens) distance.append(lens) distance_1 = signal.medfilt(distance, 3) np.savetxt('/Users/lipengzhi/Desktop/duibi_modle/' + str(j) + '.csv', distance_1, delimiter=',') middle = int(len(distance_1) / 2) distance_fir = distance_1[:middle] distance_las = distance_1[middle:] print(distance_fir, distance_las) max_distance_fir = np.max(distance_fir) max_distance_las = np.max(distance_las) print('前一个波峰最大值', max_distance_fir) print('后一个波峰最大值', max_distance_las) print('两个峰值的差距', max_distance_fir - max_distance_las)
def getPTdetail(vacc, sintheta, PTlog, smph=0.35, smpd=40): ''' get the type of postural transition : 0 - none: not a PT - don't know what it is ! 1- sist 2 - stsi 3 - bumpy sist (double and or twist) 4 - bumpy stsi and add duration input: VERTICAL accelerometer data as array, log of start end of PTS (PTlog) in sample times, smph: minimum peak height for sintheta smpd: minimum peak distance for sintheta - in samples ''' minlength = 20 # samples = /50 seconds * 50Hz - minimum duration of PT from BMC.functions import detect_peaks Vv = getVertVelocity(vacc) sinfilt = medfilt(sintheta, 25) # this may be too high sinfilt = movingAv(sintheta, 12) #plt.plot(Vv,label='Vv') PTdetail = {} # len(sintheta) for i, pt in PTlog.items(): start, end, dur = pt #start,end,dur = PTlog[33] # testing if dur < minlength: continue showgr = False #plt.plot(sinfilt,label='sinfilt') # addGrid() ptsintheta = sinfilt[start:end] plt.plot(ptsintheta) # now we have one PT. get peaks for sintheta, then peaks for arb sinthpeaks = detect_peaks.detect_peaks(ptsintheta, mph=smph, mpd=smpd, edge='both', kpsh=True, show=showgr) # if 2 take the mean, else the median ? # TODO - if there are multiple peaks x distance apart, raise hte threshold to # separate them if len(sinthpeaks) == 0: print(i, 'no sin theta peaks found, rejecting PT. mph,mpd =', smph, smpd) continue try: sinthpeak = np.mean(sinthpeaks) sinthy = ptsintheta[int(sinthpeak)] except ValueError: print('sinthpeaks: ', sinthpeak, 'PTlog: ', i) continue #plt.plot(ptvv) #addGrid() #get 1 secs before and after peak sinpeak = int(start + sinthpeak) offset = sinpeak - 50 # sample no. of start of data ptvv = Vv[offset:sinpeak + 50] Vvpeaks = detect_peaks.detect_peaks(ptvv, mph=0.25, mpd=40, show=showgr) if len(Vvpeaks) == 0: print( i, ' no Vertical velocity peaks found, rejecting PT. mph=', ) continue Vvvalleys = detect_peaks.detect_peaks(ptvv, mph=None, mpd=20, valley=True, show=showgr) if len(Vvvalleys) == 0: print(i, ' no valleys found, rejecting PT') continue # we want the valley before the peak Vpeak = np.mean( Vvpeaks ) # assuming max of 2? may need the one closest to sintheta peak # find smallest negative value to get closest valley before peak Vvv = Vvvalleys - Vpeak # this is a float , so we can: idx = (Vvv**-1).argmin() Vvalley = Vvvalleys[idx] # how far apart are peak and valley ... min 80, max 220 (by eye) #TODO reject if samples too far apart #note that the velocity zero can drift around , so if the valley is above zero, # adjust to -0.1 and make same adjust to vpeak if ptvv[Vvalley] >= 0: valley = -0.1 diff = ptvv[Vvalley] - valley peak = ptvv[int(Vpeak)] - diff else: valley = ptvv[Vvalley] peak = ptvv[int(Vpeak)] ratio = abs(peak / valley) if ratio > 1.9: result = 1 #sist ratio arrived at empirically #TODO derive the ratio from the data else: result = 2 # stsi possintheta = [sinpeak, sinthy] pospeak = [Vpeak + offset, peak] posvalley = [Vvalley + offset, valley] #returnval = [possinthteta,pospeak,posvalley] PTdetail[i] = [result, possintheta, pospeak, posvalley, ratio] # will add the duration later return PTdetail
def getVectors(data, start=0, end=-1, gvalue=9.81): ''' from accelerometer data passed in as dataframe , extract 1. gravity vector 2. body vector 3. vertical acceleration 4. realign original vectors with lag induced in the claculated vectors by the butterworth filters 5. realigned horizontal vectors input: dataframe of x, y , z ; start pos; end pos ''' #data=walk15.copy() from scipy.signal import medfilt #gvalue=9.82 # note g can change over time dependent on accelerometer #gvalue=9.74 tt = data[start:end].copy() tt = data[['x', 'y', 'z']] tt.isnull().sum() ar = tt[['x', 'y', 'z']].values sx = medfilt(ar[:, 0], 3) sy = medfilt(ar[:, 1], 3) sz = medfilt(ar[:, 2], 3) # now butterworth with recommended filter params : # filter order 2, cut off frequency 1.6Hz delay 0.141 seconds xg, yg, zg = lpFilter(sx, sy, sz, 1.6, 50, 2) # delay signal by 0.141 s = 7 slots - chop 7 off beginning of true, 7 off the end of filtered xg = xg[:-7] yg = yg[:-7] zg = zg[:-7] sx = sx[7:] sy = sy[7:] sz = sz[7:] # do the same with the original data ar = ar[7:, :] #and adjust the dataframe newdf = data.iloc[7:, :].copy() # calculate inertial signals - i.e. body movements, by subtracting gravity xb = sx - xg yb = sy - yg zb = sz - zg arb = np.asarray([xb, yb, zb]).transpose() argr = np.asarray([xg, yg, zg]).transpose() ars = np.asarray([sx, sy, sz]).transpose() # normalise and get get dot product for vertical acc normg = (xg**2 + yg**2 + zg**2)**0.5 # equiv to gdotg vacc = np.array([np.dot(a, b) for a, b in zip(argr, ars)]) vacc = vacc / normg vacc = vacc - gvalue # just to be explicit - body only # calculate in a different way using body acc only # this way introduces a variation in G that seems to destabilise the # signal slightly - it seems more valid to use the fixed gravity constant # calculated for the device # vacc2 = np.array([np.dot(a,b) for a,b in zip(arb,argr)]) # vacc2 = vacc2/normg # vacc2 = np.array([vacc2]).transpose() # #vacc2.transpose().shape # # argr.shape vacc2d = np.array([vacc]).transpose() Vvacc = vacc2d * argr # vertical acceleration expressed as a vector Vhacc = arb - Vvacc return ar, arb, argr, ars, vacc, Vhacc, newdf