def calc_we(cdata,basedir,log_target_rate): # WE parameters we_dir = os.path.join(basedir,cdata['name'],'analysis') we_dt = cdata['tau'] we_nbins = cdata['nbins'] we_target_count = cdata['target_count'] winsize_flux = cdata['analysis']['winsize_flux'] winsize_err = cdata['analysis']['winsize_err'] last_n = cdata['analysis']['last_n'] we_nframes = 0 we_data_files = glob(os.path.join(we_dir,'*/rate.h5')) for fname in we_data_files: f = h5py.File(fname,'r') we_nframes = max(we_nframes,f.attrs['last_completed_iter']-2) f.close() we_err = np.empty((len(we_data_files),2,we_nframes)) we_err.fill(np.nan) for k,fname in enumerate(we_data_files): print 'we: {}'.format(fname) f = h5py.File(fname,'r') s = f['data'][:] dget = min(we_nframes,s.shape[0]) s = s[:dget,:] ss = np.empty_like(s) for i in xrange(4): ss[:,i] = smooth(s[:,i],winsize_flux,'flat') sm = s[-last_n:,:].sum(0) rAB = (1.0*ss[:,1]) / (we_dt*ss[:,2]) rBA = (1.0*ss[:,0]) / (we_dt*ss[:,3]) rABm = (1.0*sm[1]) / (we_dt*sm[2]) rBAm = (1.0*sm[0]) / (we_dt*sm[3]) print 'we_{} -- kAB: {}, kBA: {}'.format(k,rABm,rBAm) we_err[k,0,:rAB.shape[0]] = logfunc(rAB) - log_target_rate[0] we_err[k,1,:rBA.shape[0]] = logfunc(rBA) - log_target_rate[1] f.close() we_err = np.abs(we_err) #we_err_avg = np.sqrt(np.mean(we_err**2,0)) we_err_avg = np.sqrt(bn.nanmean(we_err**2,0)) for i in xrange(2): we_err_avg[i,:] = smooth(we_err_avg[i,:],winsize_err,'flat') we_t = we_dt * we_nbins * we_target_count * np.arange(we_nframes) return we_err_avg, we_t
def smooth(self,smooth,downsample=True,**kwargs): """ Smooth the spectrum by factor `smooth`. Documentation from the :mod:`smooth` module: Parameters ---------- downsample: bool Downsample the spectrum by the smoothing factor? """ smooth = round(smooth) self.data = sm.smooth(self.data,smooth,downsample=downsample,**kwargs) if downsample: self.xarr = self.xarr[::smooth] if len(self.xarr) != len(self.data): raise ValueError("Convolution resulted in different X and Y array lengths. Convmode should be 'same'.") if self.error is not None: self.error = sm.smooth(self.error,smooth,**kwargs) self.baseline.downsample(smooth) self.specfit.downsample(smooth) self._smooth_header(smooth)
def kinematic_params(mech_x_l, time_list, smooth_window): mech_time_mat = np.matrix(time_list) tstep_size = .03333333333 num_el = mech_time_mat.shape[1] uniform_time2 = np.cumsum(np.round((mech_time_mat[0,1:] - mech_time_mat[0,0:-1]) / tstep_size) * tstep_size) uniform_time2 = np.column_stack((np.matrix([0]), uniform_time2)) mech_x_mat = np.matrix(mech_x_l) if uniform_time2.shape[1] != mech_x_mat.shape[1]: pdb.set_trace() mech_x_mat = np.matrix(smooth.smooth(mech_x_mat.A1, smooth_window, 'blackman')) uniform_time2 = uniform_time2[:,smooth_window-1:-smooth_window+1] vel = gradient(uniform_time2, mech_x_mat) uniform_time2 = uniform_time2[:,1:-1] vel = np.matrix(smooth.smooth(vel.A1, smooth_window, 'blackman')) mech_x_mat = mech_x_mat[:,smooth_window-1:-smooth_window+1] uniform_time2 = uniform_time2[:,smooth_window-1:-smooth_window+1] acc = gradient(uniform_time2, vel) uniform_time2 = uniform_time2[:,1:-1] vel = vel[:,1:-1] mech_x_mat = mech_x_mat[:,2:-2] acc = np.matrix(smooth.smooth(acc.A1, smooth_window, 'blackman')) vel = vel[:,smooth_window-1:-smooth_window+1] mech_x_mat = mech_x_mat[:,smooth_window-1:-smooth_window+1] uniform_time2 = uniform_time2[:,smooth_window-1:-smooth_window+1] return mech_x_mat.A1.tolist(), vel.A1.tolist(), acc.A1.tolist(), uniform_time2.A1.tolist()
def get_matrix_for_textfile(data, img_size_crop_x, img_size_crop_y, stim, zz, time_start, time_end,\ f_f_flag, dff_start, dff_end,stim_start,stim_end,filename, pp): #Cropping unwanted pixels as specified by user if img_size_crop_x!= 0 and img_size_crop_y!=0: print "Cropping x and y pixels.." data1 = data[img_size_crop_y:-img_size_crop_y, img_size_crop_x:-img_size_crop_x] elif img_size_crop_x==0 and img_size_crop_y!=0: print "Cropping only y pixels.." data1 = data[img_size_crop_y:-img_size_crop_y, :] elif img_size_crop_x!=0 and img_size_crop_y==0: print "Cropping only x pixels.." data1 = data[:, img_size_crop_x:-img_size_crop_x] else: data1 = data print 'Creating array from stack for Stim ' + stim + ' Z='+ str(filename) temp_matfile_for_thunder = np.zeros([np.size(data1, axis=0)*np.size(data1, axis=1),3+(time_end-time_start+1)+smooth_window-2], dtype=np.int) count = 0 for yy in xrange(0,np.size(data1, axis=1)): for xx in xrange(0,np.size(data1, axis=0)): temp_matfile_for_thunder[count,0] = xx+1; temp_matfile_for_thunder[count,1] = yy+1; temp_matfile_for_thunder[count,2] = zz; # Create delta f/f values if necessary if f_f_flag==0: temp_matfile_for_thunder[count,3:] = smooth(data1[xx,yy,time_start:time_end],smooth_window,'hanning') else: temp_matfile_for_thunder[count,3:] = smooth(((data1[xx,yy,time_start:time_end]-np.mean(data1[xx,yy,dff_start:dff_end]))/np.std(data1[xx,yy,dff_start:dff_end])),smooth_window,'hanning') count = count+1 #Plot heatmap for validation with sns.axes_style("white"): A = temp_matfile_for_thunder[:,3:] B = np.argsort(np.mean(A, axis=1)) C = A[B,:] if f_f_flag == 1: #Plot with correct climif dff is true fig2 = plt.imshow(C[-1000:,:],aspect='auto', cmap='jet',vmin=-5, vmax=5) else: fig2 = plt.imshow(C[-1000:,:],aspect='auto', cmap='jet') plot_vertical_lines(stim_start-time_start,stim_end-time_start) labels, locs = plt.xticks() labels1 = [int(item) for item in labels] labels2 = [str(int(item)+time_start) for item in labels] plt.xticks((labels1),(labels2)) plt.xlim(0,(time_end-time_start)) plt.title('Sorted Heatmap Z='+str(filename)) plt.colorbar() fig2 = plt.gcf() pp.savefig(fig2) plt.close() A = None return temp_matfile_for_thunder
def get_matrix_for_textfile(data_mat, name_for_saving_files, num_z_planes, time_start,time_end, f_f_flag, dff_start, dff_end, stimulus_on_time, stimulus_off_time, smooth_window, pp): #Save as numpy array print 'Creating array from stack for ' + name_for_saving_files if smooth_window!=0: temp_numpy_array_for_thunder = np.zeros([np.size(data_mat, axis=0)*np.size(data_mat, axis=1)*np.size(data_mat,axis=2),3+(time_end-time_start+1)+smooth_window-2], dtype=np.int) else: temp_numpy_array_for_thunder = np.zeros([np.size(data_mat, axis=0)*np.size(data_mat, axis=1)*np.size(data_mat,axis=2),3+(time_end-time_start)], dtype=np.int) print np.shape(temp_numpy_array_for_thunder) count = 0 count1 = 0 for zz in xrange(0,np.size(num_z_planes,axis=0)): for yy in xrange(0,np.size(data_mat, axis=1)): for xx in xrange(0,np.size(data_mat, axis=0)): temp_numpy_array_for_thunder[count,0] = xx+1; temp_numpy_array_for_thunder[count,1] = yy+1; temp_numpy_array_for_thunder[count,2] = num_z_planes[zz]; # Create delta f/f values if necessary if smooth_window!=0: if f_f_flag==0: temp_numpy_array_for_thunder[count,3:] = smooth(data_mat[xx,yy,zz,time_start:time_end],smooth_window,'hanning') else: temp_numpy_array_for_thunder[count,3:] = smooth(((data_mat[xx,yy,zz,time_start:time_end]-np.mean(data_mat[xx,yy,zz,dff_start:dff_end]))/np.std(data_mat[xx,yy,zz,dff_start:dff_end])),smooth_window,'hanning') else: if f_f_flag==0: temp_numpy_array_for_thunder[count,3:] = data_mat[xx,yy,zz,time_start:time_end] else: temp_numpy_array_for_thunder[count,3:] = ((data_mat[xx,yy,zz,time_start:time_end]-np.mean(data_mat[xx,yy,zz,dff_start:dff_end]))/np.std(data_mat[xx,yy,zz,dff_start:dff_end])) count = count+1 #Plot heatmap for validation with sns.axes_style("white"): A = temp_numpy_array_for_thunder[count1:count-1,3:] count1 = count-1 B = np.argsort(np.mean(A, axis=1)) C = A[B,:] if f_f_flag == 1: #Plot with correct clim if dff is true fig2 = plt.imshow(C[-1000:,:],aspect='auto', cmap='jet',vmin=-5, vmax=5) else: fig2 = plt.imshow(C[-1000:,:],aspect='auto', cmap='jet') plot_vertical_lines_onset(stimulus_on_time) plot_vertical_lines_offset(stimulus_off_time) plt.title(name_for_saving_files +' Z='+ str(zz+1)) plt.colorbar() fig2 = plt.gcf() pp.savefig(fig2) plt.close() return temp_numpy_array_for_thunder
def smooth(self, smooth, **kwargs): """ Smooth the spectrum by factor "smooth". Options are defined in sm.smooth because 'Spectra' does not have a header attribute, don't do anything to it... """ smooth = round(smooth) self.data = sm.smooth(self.data, smooth, **kwargs) self.xarr = self.xarr[::smooth] if len(self.xarr) != len(self.data): raise ValueError("Convolution resulted in different X and Y array lengths. Convmode should be 'same'.") self.error = sm.smooth(self.error, smooth, **kwargs) self.baseline.downsample(smooth) self.specfit.downsample(smooth)
def pop_spectra(self,row,col): self.laserFlashFile.switchOffHotPixTimeMask() dataDict = self.laserFlashFile.getTimedPacketList(row,col,timeSpacingCut=0.001) peakHeights=np.asarray(dataDict['peakHeights'])*1.0 baselines=np.asarray(dataDict['baselines'])*1.0 peakHeights-=baselines biggest_photon = int(min(peakHeights)) n_inbin,phase_bins=np.histogram(peakHeights,bins=np.abs(biggest_photon),range=(biggest_photon,0)) phase_bins=(phase_bins+(phase_bins[1]-phase_bins[0])/2.0)[:-1] try: last_ind = np.where(n_inbin>5)[0][-1] except IndexError: last_ind=len(n_inbin)-1 expTime = self.laserFlashFile.hotPixTimeMask.expTime self.axes.plot(phase_bins,n_inbin*1.0/expTime, 'k.',alpha=0.5,label="raw") self.axes.set_xlim(phase_bins[(np.where(n_inbin >= 3))[0][0]],phase_bins[last_ind]) self.laserFlashFile.switchOnHotPixTimeMask(reasons=['laser not on','hot pixel']) dataDict = self.laserFlashFile.getTimedPacketList(row,col,timeSpacingCut=0.001) peakHeights=np.asarray(dataDict['peakHeights'])*1.0 baselines=np.asarray(dataDict['baselines'])*1.0 peakHeights-=baselines biggest_photon = int(min(peakHeights)) n_inbin,phase_bins=np.histogram(peakHeights,bins=np.abs(biggest_photon),range=(biggest_photon,0)) phase_bins_1=(phase_bins+(phase_bins[1]-phase_bins[0])/2.0)[:-1] intTime_1 = self.laserFlashFile.hotPixTimeMask.getEffIntTime(row,col) smoothed_1 = np.asarray(smooth.smooth(n_inbin, 30, 'hanning')) self.axes.plot(phase_bins_1,n_inbin*1.0/intTime_1, 'b.',label="laser on, hotpix masked") self.laserFlashFile.switchOnHotPixTimeMask(reasons=['laser not off','hot pixel']) dataDict = self.laserFlashFile.getTimedPacketList(row,col,timeSpacingCut=0.001) peakHeights=np.asarray(dataDict['peakHeights'])*1.0 baselines=np.asarray(dataDict['baselines'])*1.0 peakHeights-=baselines biggest_photon = int(min(peakHeights)) n_inbin,phase_bins=np.histogram(peakHeights,bins=np.abs(biggest_photon),range=(biggest_photon,0)) phase_bins_2=(phase_bins+(phase_bins[1]-phase_bins[0])/2.0)[:-1] intTime_2 = self.laserFlashFile.hotPixTimeMask.getEffIntTime(row,col) smoothed_2 = np.asarray(smooth.smooth(n_inbin, 30, 'hanning')) self.axes.plot(phase_bins_2,n_inbin*1.0/intTime_2, 'g.',label="laser off, hotpix masked") self.axes.legend(loc=2) self.axes.plot(phase_bins_1, smoothed_1*1.0/intTime_1,'b-') self.axes.plot(phase_bins_2,smoothed_2*1.0/intTime_2,'g-') self.axes.set_xlabel('phase [ADC/DAC units]') self.axes.set_ylabel('Exposure time adjusted count rate [photons/s]')
def smooth_traces(browser): """ Smooth traces Options: 1) Window type 2) Window length """ # Get options window = str(browser.ui.toolStackedWidget.smoothComboBox.currentText()) window_len = float(browser.ui.toolStackedWidget.smoothLength.text()) # Get data and widgets plotWidget = browser.ui.dataPlotsWidget toolsWidget = browser.ui.toolStackedWidget # Smooth data results = [] for item in plotWidget.plotDataItems: # Copy attributes and add some new ones attrs = item.attrs attrs['smooth_window_type'] = window attrs['smooth_window_length'] = window_len # Smooth traceSmooth = smooth.smooth(item.data, window_len=window_len, window=window) results.append([item.text(0), traceSmooth, attrs]) # Plot smoothed trace x = np.arange(0, len(traceSmooth)*item.attrs['dt'], item.attrs['dt']) plotWidget.plot(x, traceSmooth, pen=pg.mkPen('#F2EF44', width=1)) # Store results parentText = plotWidget.plotDataItems[0].parent().text(0) # Assumes all plotted data have the same parent aux.save_results(browser, parentText+'_smooth', results)
def load(self,name): """ Returns a two dimensional numpy array where a[:,0] is wavelength in Angstroms and a[:,1] is flux in counts/sec/angstrom/cm^2 Noisy spectra are smoothed with window_len in the .txt file. Ergs and AB Mag units are automatically converted to counts. """ fname = self.objects[name]['dataFile'] fullFileName = os.path.join(self.this_dir,"data",fname[0]) if (string.count(fullFileName,"fit")): a = self.loadSdssSpecFits(fullFileName) else: a = numpy.loadtxt(fullFileName) len = int(self.objects[name]['window_len'][0]) if len > 1: a[:,1] = smooth.smooth(a[:,1], window_len=len)[len/2:-(len/2)] try: fluxUnit = self.objects[name]['fluxUnit'][0] scale = float(fluxUnit.split()[0]) a[:,1] *= scale except ValueError: print "error" ergs = string.count(self.objects[name]['fluxUnit'][0],"ergs") if ergs: a[:,1] *= (a[:,0] * self.k) mag = string.count(self.objects[name]['fluxUnit'][0],"mag") if mag: a[:,1] = (10**(-2.406/2.5))*(10**(-0.4*a[:,1]))/(a[:,0]**2) * (a[:,0] * self.k) return a
def speedBias(self, bias_type='normal', debug=False): ''' Calculates the unsigned speed bias quickly without having to calculate everything else. ''' if debug: print 'Calculating bias on unsigned speed...' # grab important variables mod_u = self.Variables.struct['mod_timeseries']['ua'] mod_v = self.Variables.struct['mod_timeseries']['va'] mod_spd = np.sqrt(mod_u**2 + mod_v**2) obs_u = self.Variables.struct['obs_timeseries']['ua'] obs_v = self.Variables.struct['obs_timeseries']['va'] obs_spd = np.sqrt(obs_u**2 + obs_v**2) # change times to datetime times obs_time = self.Variables.struct['obs_time'] mod_time = self.Variables.struct['mod_time'] obs_dt, mod_dt = [], [] for i in np.arange(obs_time.size): obs_dt.append(dn2dt(obs_time[i])) for i in np.arange(mod_time.size): mod_dt.append(dn2dt(mod_time[i])) # perform interpolation and grab bias (mod_sp_int, obs_sp_int, step_sp_int, start_sp_int) = \ smooth(mod_spd, mod_dt, obs_spd, obs_dt, debug=debug) stats = TidalStats(mod_sp_int, obs_sp_int, step_sp_int, start_sp_int, type='speed', debug=debug) bias = stats.getBias(bias_type=bias_type) return bias
def powerRMSE(self, debug=False): ''' Calculates the RMSE quickly without having to calculate everything else. ''' # grab important variables mod_u = self.Variables.struct['mod_timeseries']['ua'] mod_v = self.Variables.struct['mod_timeseries']['va'] mod_spd = np.sqrt(mod_u**2 + mod_v**2) mod_pow = 0.5 * rho**3 * mod_spd**3 obs_u = self.Variables.struct['obs_timeseries']['ua'] obs_v = self.Variables.struct['obs_timeseries']['va'] obs_spd = np.sqrt(obs_u**2 + obs_v**2) obs_pow = 0.5 * rho**3 * obs_spd**3 # change times to datetime times obs_time = self.Variables.struct['obs_time'] mod_time = self.Variables.struct['mod_time'] obs_dt, mod_dt = [], [] for i in np.arange(obs_time.size): obs_dt.append(dn2dt(obs_time[i])) for i in np.arange(mod_time.size): mod_dt.append(dn2dt(mod_time[i])) # perform interpolation and grab RMSE (mod_pw_int, obs_pw_int, step_pw_int, start_pw_int) = \ smooth(mod_pow, mod_dt, obs_pow, obs_dt, debug=debug) stats = TidalStats(mod_pw_int, obs_pw_int, step_pw_int, start_pw_int, type='power', debug=debug) RMSE = stats.getRMSE() return RMSE
def edgedetect(image): global pixelsum, pixelcount oldimage=image if image[0]=="P3": #converts file to grayscale and smoothes it import smooth image=smooth.smooth(image) file2=oldimage[:] xyarray=image[2].split(" ") width=int(xyarray[0]) height=int(xyarray[1]) count=0 #loop through all non-edge pixels for x in range(1,width-1): for y in range(1,height-1): #calculate the horizontal gradient of the current pixel hg=int(-1*getpixel(x-1,y-1,image)+0*getpixel(x,y-1,image)+1*getpixel(x+1,y-1,image)-2*getpixel(x-1,y,image)+0*getpixel(x,y,image)+2*getpixel(x+1,y,image)-1*getpixel(x-1,y+1,image)+0*getpixel(x,y+1,image)+1*getpixel(x+1,y+1,image)) #calculate the vertical gradient of the current pixel vg=int(1*getpixel(x-1,y-1,image)+2*getpixel(x,y-1,image)+1*getpixel(x+1,y-1,image)+0*getpixel(x-1,y,image)+0*getpixel(x,y,image)+0*getpixel(x+1,y,image)-1*getpixel(x-1,y+1,image)-2*getpixel(x,y+1,image)-1*getpixel(x+1,y+1,image)) #if the threshold is reached, mark the pixel if(abs(hg)+abs(vg)>threshold): #count the number of pixels marked as edges pixelcount+=1 if oldimage[0]=="P3": #image is color, mark as red file2[(width*y+x)*3+4]=255 file2[(width*y+x)*3+5]=0 file2[(width*y+x)*3+5]=0 else: #image is grayscale, mark as white file2[width*y+x+4]=255 pixelsum=width*height return file2
def loadsimdata(filename): # load data and apply efficiency itof = hh.load(filename, 'I(tof)') eff = hh.load('mon1-eff.h5') i = itof.I * eff.I itof.I[:] = i # clean up itof[(0.019,None)].I = 0 # x = itof.tof y = itof.I # convert to counts/10 mus # counts * 10, bins / 10 from smooth import smooth y = smooth(y, window_len=10, window='flat') # y = y[:len(x)] # y *= 10. indexes = range(5, len(x), 10) x = x[indexes] y = y[indexes] * 10 # convert to arcs run #5 # according to ARCS_runinfo.xml of run #5 # beam power 110kW # total run time is 22590/30 seconds # the mc simulated was 2MW, 60Hz y *= 22590/30*110e3/(2e6/60) # extra scaling factor, why? y *= 0.83 return x,y
def findSegments(self, rawData, minSegSize=40, smoothData=False, window_len=5, window='blackman', peakFunc=argrelmax): principal_component_of_left_foot = self.PCABySensor(rawData)['LTIO'][:,0] if smoothData: principal_component_of_left_foot = smooth(principal_component_of_left_foot, window_len=window_len,window=window) segPoints = peakFunc(principal_component_of_left_foot,order=minSegSize)[0] return segPoints
def get_cutoff(data, w_len=50, m=30, guess=None): if guess is None: approx = get_threshold(data, m) else: approx = guess smoothData = smooth(data) return get_first_in_range(smoothData[approx - w_len:approx + w_len], approx, w_len)
def compareTG(data, plot=False, save_csv=False, debug=False, debug_plot=False): """ Does a comprehensive comparison between tide gauge height data and modeled data. Input: - data = dictionary containing all necessary tide gauge and model data. Outputs - elev_suite = dictionary of useful statistics Options: - plot = boolean flag for plotting results - save_csv = boolean flag for saving statistical benchmarks in csv file """ if debug: print "CompareTG..." # load data mod_elev = data['mod_timeseries']['elev'] obs_elev = data['obs_timeseries']['elev'] obs_datenums = data['obs_time'] mod_datenums = data['mod_time'] gear = data['type'] # Type of measurement gear (drifter, adcp,...) #TR: comment out #mod_harm = data['elev_mod_harmonics'] # Save path & create folder name = data['name'] save_path = name.split('/')[-1].split('.')[0]+'/' while exists(save_path): save_path = save_path[:-1] + '_bis/' mkdir(save_path) # convert times and grab values obs_time, mod_time = [], [] for i, v in enumerate(obs_datenums): obs_time.append(dn2dt(v)) for j, w in enumerate(mod_datenums): mod_time.append(dn2dt(w)) if debug: print "...check if they line up in the time domain..." if (mod_time[-1] < obs_time[0] or obs_time[-1] < mod_time[0]): raise PyseidonError("---time periods do not match up---") else: if debug: print "...interpolate timeseries onto a common timestep..." (mod_elev_int, obs_elev_int, step_int, start_int) = \ smooth(mod_elev, mod_time, obs_elev, obs_time, debug=debug, debug_plot=debug_plot) elev_suite = tidalSuite(gear, mod_elev_int, obs_elev_int, step_int, start_int, [], [], [], [], [], [], kind='elevation', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) if debug: print "...CompareTG done." return elev_suite
def edgedetect(image): print image[:10] tvals = image[:] hg = image[:] vg = image[:] theta = image[:] global pixelsum, pixelcount oldimage = image if image[0] == "P3": # converts file to grayscale and smoothes it import smooth image = smooth.smooth(image) print image[:10] file2 = oldimage[:] width = int(image[1]) height = int(image[2]) count = 0 # loop through all non-edge pixels for x in range(1, width - 1): for y in range(1, height - 1): # calculate the horizontal gradient of the current pixel hg[width * y + x + 4] = int( -1 * getpixel(x - 1, y - 1, image) + 0 * getpixel(x, y - 1, image) + 1 * getpixel(x + 1, y - 1, image) - 2 * getpixel(x - 1, y, image) + 0 * getpixel(x, y, image) + 2 * getpixel(x + 1, y, image) - 1 * getpixel(x - 1, y + 1, image) + 0 * getpixel(x, y + 1, image) + 1 * getpixel(x + 1, y + 1, image) ) # calculate the vertical gradient of the current pixel vg[width * y + x + 4] = int( 1 * getpixel(x - 1, y - 1, image) + 2 * getpixel(x, y - 1, image) + 1 * getpixel(x + 1, y - 1, image) + 0 * getpixel(x - 1, y, image) + 0 * getpixel(x, y, image) + 0 * getpixel(x + 1, y, image) - 1 * getpixel(x - 1, y + 1, image) - 2 * getpixel(x, y + 1, image) - 1 * getpixel(x + 1, y + 1, image) ) # store the threshold value tvals[width * y + x + 4] = abs(hg[width * y + x + 4]) + abs(vg[width * y + x + 4]) theta[width * y + x + 4] = float(atan2(-vg[width * y + x + 4], hg[width * y + x + 4])) # if the threshold is reached, mark the pixel for x in range(1, width - 1): for y in range(1, height - 1): if int(tvals[width * y + x + 4]) >= hthreshold: fill(x, y, file2, tvals, theta, lthreshold) pixelsum = width * height return file2
def plot(self,smoothing=0,derivative=False,normalize=False): y=smooth(self.countsPerSec,smoothing) x=self.E if derivative: y=(y[1:]-y[:-1])/self.step x=x[0:-1]+self.step if normalize: y/=np.sqrt(sum(y**2)) elif normalize: y*=len(y)/sum(y) pl.plot(x,y,label=self.fileName+':'+str(self.part))
def testSmoothZCosmos(self): std = MKIDStd.MKIDStd() raw = std.load("zcosmos841948") plt.plot(raw[:,0],raw[:,1], label="raw") len = 31 smoothed = smooth.smooth(raw[:,1],window_len=len) plt.plot(raw[:,0],smoothed[len/2:-(len/2)], label="smoothed") plt.legend() plt.show()
def get_guess(spec, window_len=11, n=2, m=20, bins=2048): sdata = smooth(spec[::-1], window_len) # find values which are higher than neighbors test = r_[True, sdata[1:] > sdata[:-1]] & r_[sdata[:-1] > sdata[1:], True] # find values which are higher than the next m neighbors newInds = where(test)[0] values = [i - 5 for i in newInds[newInds > m] if sdata[i] == max(sdata[i - m:i + m + 1]) and sdata[i] > 1.0] mu_values = [bins - i for i in values[n - 2:n]] return mean(mu_values)
def testSmoothDelta(self): nIn = 100; x = np.zeros(nIn) x[nIn/2] = 1 len = 21 xs = smooth.smooth(x, window_len=len) plt.clf() plt.plot(x) plt.plot(xs[len/2:-(len/2)]) plt.savefit("testSmoothDelta.png")
def velocity(kin_info, smooth_window): mech_time_mat = np.matrix(kin_info['mech_time_arr']) tstep_size = .03333333333 num_el = mech_time_mat.shape[1] uniform_time2 = np.cumsum(np.round((mech_time_mat[0,1:] - mech_time_mat[0,0:-1]) / tstep_size) * tstep_size) uniform_time2 = np.column_stack((np.matrix([0]), uniform_time2)) mech_intrinsic_poses = kin_info['disp_mech_coord_arr'] if uniform_time2.shape[1] != mech_intrinsic_poses.shape[0]: pdb.set_trace() vel = gradient(uniform_time2, np.matrix(mech_intrinsic_poses)) return smooth.smooth(vel.A1, smooth_window, 'blackman')
def loadexpdata(filename, n, binsize=10): # load data and apply efficiency itof = hh.load(filename) x = itof.tof y = itof.I # convert to counts/10 mus # counts * 10, bins / 10 from smooth import smooth y = smooth(y, window_len=binsize, window='flat') indexes = range(binsize/2, binsize/2+binsize*n, binsize) x = x[indexes] y = y[indexes] * binsize return x,y
def extrema(a,bPlot=False,nBins=300,smoothWindowSize=50): hist,binEdges = np.histogram(a,bins=nBins,density=True) histSmooth = smooth.smooth(hist,smoothWindowSize,'hanning') binCenters = binEdges[0:-1]+np.diff(binEdges)/2. extremeIdxs = np.where(np.diff(np.sign(np.diff(histSmooth))))[0]+1 bMaximums = np.diff(np.sign(np.diff(histSmooth)))<0 bMaximums = bMaximums[np.where(np.diff(np.sign(np.diff(histSmooth))))] if bPlot: fig = plt.figure() ax = fig.add_subplot(111) ax.plot(binCenters,histSmooth) ax.plot(binCenters,hist) ax.plot(binCenters[extremeIdxs],histSmooth[extremeIdxs],'.') return {'extremeX':binCenters[extremeIdxs],'extremeY':hist[extremeIdxs],'bMaximums':bMaximums}
def summaryStatistics(data, std_window=10): gradients = [] for col in range(np.shape(data)[1]): gradients.append(np.gradient(smooth(data[:,col]))) gradients = np.column_stack(gradients) standardDeviations = [] for col in range(np.shape(data)[1]): stdev = [] for i in range(len(data)): stdev.append(np.std(data[:,col][max(0,i-(std_window/2)):i+(std_window/2)])) standardDeviations.append(stdev) standardDeviations = np.column_stack(standardDeviations) return (gradients,standardDeviations)
def compareTG(data, plot=False, save_csv=False, debug=False, debug_plot=False): ''' Does a comprehensive comparison between tide gauge height data and modeled data, much like the above function. Input is a dictionary containing all necessary tide gauge and model data. Outputs a dictionary of useful statistics. ''' if debug: print "CompareTG..." # load data mod_elev = data['mod_timeseries']['elev'] obs_elev = data['obs_timeseries']['elev'] obs_datenums = data['obs_time'] mod_datenums = data['mod_time'] #TR: comment out #mod_harm = data['elev_mod_harmonics'] # convert times and grab values obs_time, mod_time = [], [] for i, v in enumerate(obs_datenums): obs_time.append(dn2dt(v)) for j, w in enumerate(mod_datenums): mod_time.append(dn2dt(w)) if debug: print "...check if they line up in the time domain..." if (mod_time[-1] < obs_time[0] or obs_time[-1] < mod_time[0]): print "---time periods do not match up---" sys.exit() else: if debug: print "...interpolate timeseries onto a common timestep..." (mod_elev_int, obs_elev_int, step_int, start_int) = \ smooth(mod_elev, mod_time, obs_elev, obs_time, debug=debug, debug_plot=debug_plot) if debug: print "...get validation statistics..." stats = TidalStats(mod_elev_int, obs_elev_int, step_int, start_int, type='elevation', debug=debug, debug_plot=debug_plot) elev_suite = tidalSuite(mod_elev_int, obs_elev_int, step_int, start_int, type='elevation', plot=plot, save_csv=save_csv, debug=debug, debug_plot=debug_plot) if debug: print "...CompareTG done." return elev_suite
def compareTG(data): ''' Does a comprehensive comparison between tide gauge height data and modeled data, much like the above function. Input is a dictionary containing all necessary tide gauge and model data. Outputs a dictionary of useful statistics. ''' # load data mod_elev = data['mod_timeseries']['elev'] obs_elev = data['obs_timeseries']['elev'] obs_datenums = data['obs_time'] mod_datenums = data['mod_time'] mod_harm = data['elev_mod_harmonics'] print data['name'] # convert times and grab values obs_time, mod_time = [], [] for i, v in enumerate(obs_datenums): obs_time.append(dn2dt(v)) for j, w in enumerate(mod_datenums): mod_time.append(dn2dt(w)) # check if they line up in the time domain if (mod_time[-1] < obs_time[0] or obs_time[-1] < mod_time[0]): # use ut_reconstr to create a new timeseries mod_elev_int = ut_reconstr(obs_datenums, mod_harm)[0] obs_elev_int = obs_elev step_int = obs_time[1] - obs_time[0] start_int = obs_time[0] else: # interpolate timeseries onto a common timestep (obs_elev_int, mod_elev_int, step_int, start_int) = \ smooth(mod_elev, mod_time, obs_elev, obs_time) # get validation statistics stats = TidalStats(mod_elev_int, obs_elev_int, step_int, start_int, debug=True, type='height') elev_suite = stats.getStats() elev_suite['r_squared'] = stats.linReg()['r_2'] elev_suite['phase'] = stats.getPhase(debug=False) return elev_suite
def findSigmaThresh(data, initSigmaThresh=2., tailSlack=0., isPlot=False): ''' Finds the optimal photon trigger threshold by cutting out the noise tail in the pulse height histogram. INPUTS: data - filtered phase timestream data (positive pulses) initSigmaThresh - sigma threshold to use when constructing initial pulse height histogram tailSlack - amount (in same units as data) to relax trigger threshold isPlot - make peak height histograms if true OUTPUTS: threshold - trigger threshold in same units as data sigmaThresh - trigger threshold in units sigma from median ''' peakdict = sigmaTrigger(data, nSigmaTrig=initSigmaThresh) peaksHist, peaksHistBins = np.histogram(peakdict['peakHeights'], bins='auto') if(isPlot): plt.plot(peaksHistBins[:-1], peaksHist) plt.title('Unsmoothed Plot') plt.show() print 'peaksHistLen:', len(peaksHist) peaksHist = smooth.smooth(peaksHist,(len(peaksHistBins)/20)*2+1) print 'peaksHistSmoothLen:', len(peaksHist) if(isPlot): plt.plot(peaksHistBins[0:len(peaksHist)], peaksHist) plt.title('smoothed plot') plt.show() minima=np.ones(len(peaksHist)) #keeps track of minima locations; element is 1 if minimum exists at that index minimaCount = 1 #while there are multiple local minima, look for the deepest one while(np.count_nonzero(minima)>1): minima = np.logical_and(minima, np.logical_and((peaksHist<=np.roll(peaksHist,minimaCount)),(peaksHist<=np.roll(peaksHist,-minimaCount)))) #print 'minima array:', minima minima[minimaCount-1]=0 minima[len(minima)-minimaCount]=0 #get rid of boundary effects minimaCount += 1 thresholdInd = np.where(minima)[0][0] threshold = peaksHistBins[thresholdInd]-tailSlack sigmaThresh = (threshold-np.median(data))/np.std(data) return threshold, sigmaThresh
def plotTrackingData(trackingData, props): colors = ['Crimson', 'CornflowerBlue', 'DarkOliveGreen', 'DarkOrange', 'LightSlateGray'] nData = len(trackingData) # Prepare plots fig = plt.figure(figsize=(20, 4*nData)) gs = gridspec.GridSpec(nData,3, width_ratios=[1,6,1]) ax, dataOut = [], [] for d in np.arange(0, len(trackingData)): aviProps = props[d] trackData = trackingData[d] dataOut.append(trackData[:,1]) ax1, ax2, ax3 = d*3+0, d*3+1, d*3+2 ax.append(fig.add_subplot(gs[ax1])) ax.append(fig.add_subplot(gs[ax2])) ax.append(fig.add_subplot(gs[ax3])) # Plot tracking ax[ax1].plot(trackData[:,0],trackData[:,1],colors[d]) # Plot Y over time timePerFrame = 1./aviProps[4] timePerFrame = timePerFrame/60. # convert from seconds to minutes tAxis = np.arange(0, len(trackData)*timePerFrame, timePerFrame) if len(tAxis)>len(trackData): tAxis = np.delete(tAxis, -1) if len(tAxis)<len(trackData): trackData = np.delete(trackData, -1) smooth_trackData = smooth.smooth(trackData[:,1], window_len=50) ax[ax2].plot(tAxis, trackData[:,1],'k',lw=1) ax[ax2].plot(tAxis, smooth_trackData, colors[d], lw=1.5) # Plot histogram ax[ax3].hist(trackData[:,1], bins=30, orientation='horizontal', histtype='stepfilled', normed=True, color=colors[d]) ax[ax3].set_xlim([0,0.008]) # get and plot looms #loomOnsets = da.getLoomOnsets(trackData, aviProps) #for l in loomOnsets: ax[ax2].plot(l, 1150, marker='o', markerfacecolor='k', markeredgecolor='k') plt.show() return dataOut
def segmentationPoints(X, xIsFilename=False, windowSize=100, smoothing='blackman'): if xIsFilename == True: with open(X,'r') as fin: X = np.loadtxt(fin,delimiter=",") smoothX = sm.smooth(X,window_len=windowSize,window=smoothing) #(unsmoothedMins,unsmoothedMaxs) = mm.find_mins_and_maxs1D(X) (smoothedMins,smoothedMaxs) = mm.find_mins_and_maxs1D(smoothX) # Using Dynamic Time Warping to map the smoothed curve onto the original, noisy curve. alignmentOfSmoothedCurveAndOriginal = r.dtw(smoothX, X) warpIndexes=r.warp(alignmentOfSmoothedCurveAndOriginal,True) #pl.plot(warpIndexes,smoothX) #pl.plot(X) #pl.show() minsMappedToOriginal = sorted(list(set([warpIndexes[smoothedMins[i]] for i in range(len(smoothedMins))]))) maxsMappedToOriginal = sorted(list(set([warpIndexes[smoothedMaxs[i]] for i in range(len(smoothedMaxs))]))) #minsMappedToOriginal = mapToOriginalRecursive(X,smoothedMins,unsmoothedMins,"mins") #maxsMappedToOriginal = mapToOriginalRecursive(X,smoothedMaxs,unsmoothedMaxs,"maxs") return (minsMappedToOriginal,maxsMappedToOriginal)
zffs[zffs > 0] = 1 # To find zero crossings, place value 1 at all locations zffs>0 zffs[zffs < 0] = -1 # Place value -1 at all locations zffs<0 gci = np.where(np.diff(zffs) == 2)[ 0] # Take difference and look for the positions of value 2 es = np.abs( zf1[gci + 1] - zf1[gci - 1]) # Positions of value 2 are the instants of zero crossings T0 = np.diff(gci) # Finding period interms of sample number T0 = T0 / float(fs) # Period in seconds f0 = 1.0 / T0 # Frequency in Hz f0 = np.append(f0, f0[-1], f0[-1]) # Filling holes created by two difference operation # Smoothing the melody to remove high frequency contents f0Smooth = smooth.smooth(f0, 10, 'flat') # ------------------------------------------------------------------------------- # # Remove unvoiced portions from F0 contour tempF0 = np.zeros(lenSig) tempF0[gci] = f0Smooth # Get F0 at only voiced regions voicPitch = np.zeros(np.shape(x)[0]) if (np.shape(timeBegVoicSamp)[0] == np.shape(timeEndVoicSamp)[0] ): # Check if number of instants are same for i in range(np.shape(timeBegVoicSamp)[0]): begInst = timeBegVoicSamp[i] # Get beg instant endInst = timeEndVoicSamp[i] # Get end instant voicPitch[begInst:endInst] = tempF0[begInst:
def loaddata2(input_dir, ref_dir, sigmoid=5, do_normalize=True, denoise=False, fs=360): X = [] Y = [] QRS = [] datapaths = glob.glob(os.path.join(input_dir, '*.mat')) datapaths.sort() for data_path in datapaths: #print(data_path) # load data data_mat = sio.loadmat(data_path) data = data_mat['ecg'] data = np.array(data, dtype=np.float32) # print(data.shape) for i in range(data.shape[1]): smoothed_signal = smooth.smooth(data[:, i], window_len=int(fs), window='flat') data[:, i] = data[:, i] - smoothed_signal # denoise ECG for i in range(data.shape[1]): # DWT coeffs = pywt.wavedec(data[:, i], 'db4', level=3) # compute threshold noiseSigma = 0.01 threshold = noiseSigma * math.sqrt(2 * math.log2(data[:, i].size)) # apply threshold newcoeffs = coeffs for j in range(len(newcoeffs)): newcoeffs[j] = pywt.threshold(coeffs[j], threshold, mode='soft') # IDWT data[:, i] = pywt.waverec(newcoeffs, 'db4')[0:len(data)] # normalize the data # data = scale(data).astype(np.float32) # normalize the data if do_normalize: scaler = StandardScaler() scaler.fit(data) data = scaler.transform(data).astype(np.float32) X.append(data) X = np.array(X, dtype=np.float32) print('X.shape', X.shape) refpaths = glob.glob(os.path.join(ref_dir, '*.mat')) refpaths.sort() for ref_path in refpaths: # load reference ref_mat = sio.loadmat(ref_path) R_peaks = ref_mat['R_peak'] R_peaks = np.array(R_peaks, dtype=np.int32).squeeze() ref = smooth_ref(len(data), R_peaks, sigmoid=sigmoid) ref = np.expand_dims(ref, axis=-1) Y.append(ref) QRS.append(R_peaks) Y = np.array(Y, dtype=np.float32) print('Y.shape', Y.shape) # remove duplicate rows #X,indexes = np.unique(X, return_index=True, axis=0) #print(X.shape) # Y = Y[indexes] # QRS = itemgetter(*indexes)(QRS) # shuffle # X, Y, QRS = shuffle(X, Y, QRS, random_state=0) return X, Y, QRS
def compareTG(data, plot=False, save_csv=False, debug=False, debug_plot=False): """ Does a comprehensive comparison between tide gauge height data and modeled data. Input: - data = dictionary containing all necessary tide gauge and model data. Outputs - elev_suite = dictionary of useful statistics Options: - plot = boolean flag for plotting results - save_csv = boolean flag for saving statistical benchmarks in csv file """ if debug: print "CompareTG..." # load data mod_elev = data['mod_timeseries']['elev'] obs_elev = data['obs_timeseries']['elev'] obs_datenums = data['obs_time'] mod_datenums = data['mod_time'] gear = data['type'] # Type of measurement gear (drifter, adcp,...) #TR: comment out #mod_harm = data['elev_mod_harmonics'] # Save path & create folder name = data['name'] save_path = name.split('/')[-1].split('.')[0] + '/' while exists(save_path): save_path = save_path[:-1] + '_bis/' mkdir(save_path) # convert times and grab values obs_time, mod_time = [], [] for i, v in enumerate(obs_datenums): obs_time.append(dn2dt(v)) for j, w in enumerate(mod_datenums): mod_time.append(dn2dt(w)) if debug: print "...check if they line up in the time domain..." if (mod_time[-1] < obs_time[0] or obs_time[-1] < mod_time[0]): raise PyseidonError("---time periods do not match up---") else: if debug: print "...interpolate timeseries onto a common timestep..." (mod_elev_int, obs_elev_int, step_int, start_int) = \ smooth(mod_elev, mod_time, obs_elev, obs_time, debug=debug, debug_plot=debug_plot) elev_suite = tidalSuite(gear, mod_elev_int, obs_elev_int, step_int, start_int, [], [], [], [], [], [], kind='elevation', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) if debug: print "...CompareTG done." return elev_suite
print 'x,y', x[0], x[1] u = self.u(x[0], x[1]) return u != 0. subdomains = CellFunction("size_t", mesh) subdomains.set_all(0) validdata = ValidData(u) validdata.mark(subdomains, 1) dx = Measure("dx")[subdomains] # Smooth the surface and find flow directions based on # Smoothed surface gradients rho = 911 g = 9.81 Nx = smooth(Q, H, -rho * g * H * dSdx, kappa_i) Ny = smooth(Q, H, -rho * g * H * dSdy, kappa_i) # Replace smoothed surface slope direction with data where ice is fast enough # to lower errors: uhat = project(u / Unorm, Q) vhat = project(v / Unorm, Q) # Smooth these directions out too, but over fewer ice thicknesses uhat = smooth(Q, H, rho * g * H * uhat, kappa_v) vhat = smooth(Q, H, rho * g * H * vhat, kappa_v) Nx.vector()[Unorm.vector() > goodU] = uhat.vector()[Unorm.vector() > goodU] Ny.vector()[Unorm.vector() > goodU] = vhat.vector()[Unorm.vector() > goodU] # Finally, smooth again to easy the transitions between data and surface slope Nx = smooth(Q, H, rho * g * H * Nx, kappa_v)
def conductGeneration(generation, corpus, previous_output): ''' Conducts a generation of learning and testing on the input data Inputs generation (int) --- the number of the generation corpus (array) --- the lemmas and their info from reading the corpus file previous_output (dict) --- the output phonology of the previous generation Returns the output of the current generation--the expected outputs for the following generation ''' # Build the right size network net = buildNetwork(input_nodes, constants.hidden_nodes, constants.output_nodes) # Build the right size training set emptytraining_set = SupervisedDataSet(input_nodes, constants.output_nodes) # Initialize corpus object and expected output dictionary training_corpus = objects.Corpus(emptytraining_set) # Iterate through tokens and convert to binary for lemma in corpus: # Iterate through cases for case, form in lemma.cases.iteritems(): # Add words according to their frequencies training_corpus.addByFreq(constants.token_freq, form, previous_output[form.lemmacase]) # Construct the training set print '''--------Generation %d--------''' % generation print '''----------Trial %s----------- Training on %d Epochs Vectors: %s Number of Input Nodes: %d Number of Hidden Nodes: %d Number of Output Nodes: %d Token Frequency taken into account: %s Language to Model: %s Secondary Sound Change (for Italian and Romanian) Implemented: %s\n''' % ( constants.trial, constants.epochs, constants.vectors, input_nodes, constants.hidden_nodes, constants.output_nodes, constants.token_freq, constants.language, bool(constants.secondsoundchange) ) print "Constructing the training set" training_set = training_corpus.constructTrainingSet() # Construct the trainer trainer = BackpropTrainer(net, training_set) # Train print "Training the model" error = trainer.trainEpochs(constants.epochs) print "Number of Tokens in Training Set: %s" % len(training_set) results = {} # For each word in the test set, calculate output tuple print "Running the test set" # Counter to count correct ncorrect = 0 tot_phon = 0 for (form, input_tuple, expected_output) in training_corpus.test: # Activate the net, and smooth the output result = smooth(tuple(net.activate(input_tuple))) # Append output tuple to result results[form.lemmacase] = result # Hash the output tuple to get the phonological form result new_phonology = '' # Divide tuple into chunks (each 11 units, representing one phoneme) chunked_list = list(chunks(list(result), constants.n_feat)) # Divide previous output tuple into chunks chunked_prev = list(chunks(list(previous_output[form.lemmacase]), constants.n_feat)) for phon_index in range(len(chunked_list)): phoneme = chunked_list[phon_index] prev_phoneme = chunked_prev[phon_index] new_phonology += constants.feat_to_phon[tuple(phoneme)] # If phoneme matches, add to number correct if prev_phoneme != [0.0]*constants.n_feat: tot_phon += 1 if phoneme == prev_phoneme: ncorrect += 1 # Output for this generation is new suffix new_suf = ''.join(new_phonology) for seq in functions.to_revert.keys(): if seq in new_suf: new_suf = new_suf.replace(seq, functions.to_revert[seq]) new_suf = new_suf.replace('-', '') if new_suf == '': new_suf = '-' form.output_change[generation] = new_suf print form.lemmacase, form.root+form.suffix, form.changed_suf, form.parent.declension, form.parent.gender, form.parent.totfreq, form.root+new_suf, new_suf print "Results have been determined" try: print "Percentage correct in test run: %f" % round(float(ncorrect)/float(tot_phon)*100, 2) except ZeroDivisionError: print "Percentage correct in test run: 0.00" return results
if name == line[0:9].replace(' ', ''): fluxline = [ float(line[i:i + 10]) * 1.e13 for i in range(10, len(line) - 1, 10) ] flux += fluxline data[name] = flux # Get list of star names starnames = [] fn = open('jacoby-list.rasort', 'r') lines = fn.readlines() for line in lines: starname = line.split()[1] starnames.append(starname) fn.close() # Set initial wavelength lambda1 = 351.0 - 1.3 # Determined from Balmer lines # Create plot and CSV files for star in starnames: if star in data: y = np.array(data[star]) y = smooth.smooth(y, window_len=21, window='hanning') x = 0.14 * np.arange(len(y)) x += lambda1 plot_star(x, y) write_csv(x, y)
xmin=xmin, xmax=xmax) test = prepare_data(filename_format, cases_test, channels, qctimes_test, xmin=xmin, xmax=xmax) else: train = prepare_data(filename_format, cases_train, channels, qctimes_train) test = prepare_data(filename_format, cases_test, channels, qctimes_test) if data_suffix in ['ctc', 'ctc2']: for ibatch in range(train['nbatches']): train['Xdata'][ibatch,:,:,4] = smooth(\ train['Xdata'][ibatch,:,:,4],\ np.isfinite(train['Xdata'][ibatch,:,:,4]), (3,3)) for ibatch in range(test['nbatches']): test['Xdata'][ibatch,:,:,4] = smooth(\ test['Xdata'][ibatch,:,:,4],\ np.isfinite(test['Xdata'][ibatch,:,:,4]), (3,3)) if data_suffix == 'C13': print('zero out C07, C09, GLM') train['Xdata'][:, :, :, 0] = 0. #zero out C07 train['Xdata'][:, :, :, 1] = 0. #zero out C09 train['Xdata'][:, :, :, 3] = 0. #zero out GLM test['Xdata'][:, :, :, 0] = 0. #zero out C07 test['Xdata'][:, :, :, 1] = 0. #zero out C09 test['Xdata'][:, :, :, 3] = 0. #zero out GLM
def conductGeneration(generation, corpus, previous_output): ''' Conducts a generation of learning and testing on the input data inputs generation (int) --- the number of the generation corpus (array) --- the lemmas and their info from reading the corpus file previous_output (dict) --- the output (gender, declension, case, number) of the previous generation Returns the output of the current generation--the expected outputs for the following generation ''' # Build the right size network net = buildNetwork(constants.input_nodes, constants.hidden_nodes, constants.output_nodes) # Build the right size training set emptytraining_set = SupervisedDataSet(constants.input_nodes, constants.output_nodes) # Initialize corpus object and expected output dictionary training_corpus = objects.Corpus(emptytraining_set) # Iterate through tokens and convert to binary for lemma in corpus: # JUST SKIP THOSE THAT ARE DEFECTIVE if len(lemma.cases.keys()) < 6: continue # Iterate through cases for case, form in lemma.cases.iteritems(): # Get new input from previous output new_gender, new_dec, new_case, new_num, prev_output = form.output_change[ generation - 1] # Use new input as new syllables new_syllables = lemma.cases[new_case + new_num].syllables # Append to input change form.input_change[generation] = (new_case + new_num, ''.join(new_syllables).replace( '-', '')) # print form.lemmacase, form.input_change[generation] # Create the input tuple form.createInputTuple(new_syllables) # Add words according to their frequencies training_corpus.addByFreq(constants.token_freq, form, expected_outputs[form.lemmacase]) # Print information print "--------Generation %s--------" % generation if generation >= constants.gnvdrop_generation: print "Genitive Case Dropped" # Construct the training set print "Constructing the training set" training_set = training_corpus.constructTrainingSet() # Construct the trainer trainer = BackpropTrainer(net, training_set) # Train print "Training the model" if constants.epochs == 1: error = trainer.train() else: error = trainer.trainEpochs(constants.epochs) print "Number of Tokens in Training Set: %s" % len(training_set) print "Training Error: %s" % error results = {} # For each word in the test set print "Running the test set" for (form, input_tuple, expected_output) in training_corpus.test: # Determine if we should drop the genitive drop_gen = generation >= constants.gnvdrop_generation # Activate the net, and smooth the output result = smooth(tuple(net.activate(input_tuple)), gendrop=drop_gen, hierarchy=constants.hierarchy) # Append output tuple to result results[form.lemmacase] = result # Hash the output tuple to get the result gender = constants.tup_to_gen[tuple( result[constants.gen_b:constants.dec_b])] dec = constants.tup_to_dec[tuple( result[constants.dec_b:constants.case_b])] case = constants.tup_to_case[tuple( result[constants.case_b:constants.num_b])] num = constants.tup_to_num[tuple(result[constants.num_b:])] output = form.parent_lemma.cases[case + num].phonology # Set input change once we figure out how to deal with the phonology form.output_change[generation] = (gender, dec, case, num, output) print "Results have been determined" return results
error = trainer.trainEpochs(constants.epochs) print "Number of Tokens in Training Set: %s" % len(training_set) results = {} # For each word in the test set, calculate output tuple print "Running the test set" # Counter to count correct ncorrect = 0 for (form, input_tuple, expected_output) in training_corpus.test: # Activate the net, and smooth the output result = smooth(tuple(net.activate(input_tuple)), suf_to_tup) # Append output tuple to result results[form.lemmacase] = result # Add to ncorrect if matches previous if result == previous_output[form.lemmacase]: ncorrect += 1 # Hash the output tuple to get the suffix result new_suffix = suffix_dict[result] print form.lemmacase, form.root + form.suffix, form.parent.declension, form.parent.gender, form.parent.totfreq, form.root + new_suffix, new_suffix # Output for this generation and input for next # Combine new suffix with root to get new phonological form for next generation form.input_phon[generation + 1] = form.root + new_suffix
tss.add(HTSeq.GenomicPosition(chr, pos, strand)) ifp2.close() profile = numpy.zeros(2 * tssdis + 1, dtype='i') for p in tss: promoter = HTSeq.GenomicInterval(p.chrom, p.pos - tssdis, p.pos + tssdis + 1, '.') if promoter.start < 0 or promoter.end > genomesize[p.chrom]: continue wincvg = numpy.fromiter(coverage[promoter], dtype='i', count=2 * tssdis + 1) if p.strand == '+': profile += wincvg else: profile += wincvg[::-1] # mydata.append(profile/1e6) y = smooth.smooth(profile / float(myitem[-1]) * 1e6, 1000) pyplot.plot(x, y, myitem[2] + '-', lw=2, label=myitem[3]) ifp1.close() pyplot.legend(prop={'size': 8}) pyplot.xlim(-1510, 1510) pyplot.xticks([-1500, -1000, -500, 0, 500, 1000, 1500], ('-1500', '-1000', '-500', '0', '500', '1000', '1500')) pyplot.xlabel('Distance form center (bp)') pyplot.ylabel('Average coverage (reads per million)') pyplot.title(sys.argv[6]) #pyplot.show() pyplot.savefig(sys.argv[6] + '.' + sys.argv[5] + '.smooth.png') pyplot.savefig(sys.argv[6] + '.' + sys.argv[5] + '.smooth.eps')
y_sample_used = torch.from_numpy(y_sample_used).float().to(device) y_shuffle_used = torch.from_numpy(y_shuffle_used).float().to(device) pred_xy_de = model_de(x_sample_used, y_sample_used) pred_x_y_de = model_de(x_sample_used, y_shuffle_used) ret_de = torch.mean(pred_xy_de) - torch.log( torch.mean(torch.exp(pred_x_y_de))) loss_de = -ret_de # maximize plot_loss_de.append(loss_de.to('cpu').item()) model_de.zero_grad() loss_de.backward() optimizer_de.step() plot_y_de = np.array(plot_loss_de).reshape(-1, ) smooth_mi_de = smooth(plot_y_de) trans_all = np.load(trans_data_dir + item[:-4] + '.npz') x_sample_trans = trans_all['eeg'] y_sample_trans = trans_all['eye'] model_trans = Net().to(device) optimizer_trans = torch.optim.Adam(model_trans.parameters(), lr=0.001) plot_loss_trans = [] for epoch in tqdm(range(n_epoch)): number_perm = torch.randperm(1823) idx = number_perm[:500] x_sample_used_trans = x_sample_trans[idx] y_sample_used_trans = y_sample_trans[idx] y_shuffle_used_trans = np.random.permutation(y_sample_used_trans)
def conductGeneration(generation, corpus, previousOutput): ''' Conducts a generation of learning and testing on the input data inputs generation (int) --- the number of the generation corpus (array) --- the output of reading the corpus file previousOutput (dict) --- the output of the previous generation outputs ''' print "Trial %s" % str(constants.trial) input_size = constants.inputNodes # if we're using slavic data, modify the expected size of the input vector. if constants.includeSlavic and generation >= constants.generationToIntroduceSlavic: input_size = constants.inputNodesSlav # build the right size network net = buildNetwork(input_size, constants.hiddenNodes, constants.outputNodes) # build the right size training set emptyTrainingSet = SupervisedDataSet(input_size, constants.outputNodes) # initialize corpus object trainingCorpus = objects.Corpus(emptyTrainingSet) # iterate through tokens passed to the function for token in corpus: # iterate through cases for (case, word) in token.cases: # set its syllables, based on the generation (i.e. account for sound changes) word.setSyllables(generation, word.syllables) # extract the gender from the previous generation print previousOutput (placeholder, previousResult) = previousOutput[[ wordinfo for (wordinfo, gender) in previousOutput ].index(word.description)] # print previousResult # print placeholder # we already know the word # adds words according to their frequencies trainingCorpus.configure(word, previousResult, generation) # construct the training set trainingSet = trainingCorpus.constructTrainingSet() # construct the trainer trainer = BackpropTrainer(net, trainingSet) # train if constants.epochs == 1: error = trainer.train() else: error = trainer.trainEpochs(constants.epochs) print "--------Generation: %s--------" % generation if generation >= constants.generationToDropGen: print "Genitive Case Dropped" if constants.includeSlavic and generation >= constants.generationToIntroduceSlavic: print "Slavic Information Introduced" print "Number of Training Epochs: %s" % constants.epochs print "Number of Training Tokens: %s" % len(trainingSet) print "Training Error: %s" % error results = [] # Dictionary of changes changes = { 'total': 0, 'gen_change': defaultdict(lambda: 0), 'dec_change': defaultdict(lambda: 0), 'gencase_change': defaultdict(lambda: 0), 'gennum_change': defaultdict(lambda: 0), 'deccase_change': defaultdict(lambda: 0), 'decnum_change': defaultdict(lambda: 0), 'gencasenum_change': defaultdict(lambda: 0), 'deccasenum_change': defaultdict(lambda: 0) } # for each work in the input for (word, inputTuple, expectedOutput, trueLatinGender, trueRomanianGender) in trainingCorpus.test: # Count how many tokens are in the test set counterBag.totalCounter.increment() # determine if we should drop the genetive should_drop_gen = generation >= constants.generationToDropGen # activate the net, and smooth the output result = smooth(tuple(net.activate(inputTuple)), gendrop=should_drop_gen, equalcase=True) # append output tuple to result results.append((word.description, result)) # If this is the first generation if counterBag.generationCounter.value == 1: # add genchange[word.description].append( (0, word.parentToken.latinGender[0])) # Change index depending if gen has been dropped or not (gen_b, gen_e, dec_b, dec_e, case_b, case_e, num_b, num_e) = (0, 3, 3, 8, 8, 11, 11, 13) # hash the output tuple to get the result gender = constants.tup_to_gen[tuple(result[gen_b:gen_e])] declension = constants.tup_to_dec[tuple(result[dec_b:dec_e])] case = constants.tup_to_case[tuple(result[case_b:case_e])] num = constants.tup_to_num[tuple(result[num_b:num_e])] to_add = (counterBag.generationCounter.value, gender + declension + case + num, word.parentToken.latinGender[0], word.parentToken.declension, word.case, word.num) genchange[word.description].append(to_add) word.genchange[counterBag.generationCounter.value] = (gender, declension, case, num) return results
data, airmass, Object, jd, parallactic_angle, mean_seeing = fromObs(file) pwv = 0. if (inputype=='simu'): data, airmass, Object, jd, parallactic_angle, pwv = fromSimu(file) wmin = 360. wmax = 1000. data = data[(data[:,0]>= wmin) & (data[:,0]<= wmax)] ''' Smoothing the atmospheric curve to match the resolution of the observation ''' if ((inputype=='simu') and (seeing is not None)): sm = smooth.smooth(data[:,0], data[:,1], seeing_x, seeing_y, pix2wgth = pix2wght, plot = plot) data[:,0], data[:,1] = sm.smoothCurve() if seeing is None and inputype is 'simu': mean_seeing = 0. ''' --> Fitting a polynom on edges --> measurement of the EW --> +/- 10 nm sliding --> picking the max --> fitting a parabola on data-continuum and retrieving the minimum ''' for i in EW:
def conductGeneration(generation, corpus, previous_output): ''' Conducts a generation of learning and testing on the input data Inputs generation (int) --- the number of the generation corpus (array) --- the lemmas and their info from reading the corpus file previous_output (dict) --- the output phonology of the previous generation Returns the output of the current generation--the expected outputs for the following generation ''' # Build the right size network net = buildNetwork(input_nodes, constants.hidden_nodes, constants.output_nodes) # Build the right size training set emptytraining_set = SupervisedDataSet(input_nodes, constants.output_nodes) # Initialize corpus object and expected output dictionary training_corpus = objects.Corpus(emptytraining_set) # Iterate through tokens and convert to binary for lemma in corpus: # Iterate through cases for case, form in lemma.cases.iteritems(): # Add words according to their frequencies training_corpus.addByFreq(constants.token_freq, form, previous_output[form.lemmacase]) # Construct the training set print "--------Generation %s--------" % generation print "Constructing the training set" training_set = training_corpus.constructTrainingSet() # Construct the trainer trainer = BackpropTrainer(net, training_set) # Train print "Training the model" error = trainer.trainEpochs(constants.epochs) print "Number of Tokens in Training Set: %s" % len(training_set) results = {} # For each word in the test set, calculate output tuple print "Running the test set" # Counter to count correct ncorrect = 0 for (form, input_tuple, expected_output) in training_corpus.test: # Activate the net, and smooth the output result = smooth(tuple(net.activate(input_tuple)), suf_dict) # Append output tuple to result results[form.lemmacase] = result # Add to ncorrect if matches previous if result == previous_output[form.lemmacase]: ncorrect += 1 # Hash the output tuple to get the suffix result new_suf = inv_suf[result] form.output_change[generation] = new_suf print form.lemmacase, form.root + form.suffix, form.parent.declension, form.parent.gender, form.parent.totfreq, form.root + new_suf, new_suf print "Results have been determined" print "Percentage correct in test run: %f" % round( float(ncorrect) / float(len(previous_output)) * 100, 2) return results
#!/usr/bin/env python import numpy as np from smooth import smooth import pylab from scipy.optimize import curve_fit # hist0 = np.load("anglespectrum0.npy") hist180 = np.load("anglespectrum180.npy") hist0 = smooth(hist0)[:hist0.size] hist180 = smooth(hist180)[:hist180.size] # fft iq0 = np.fft.fft(hist0) iq180 = np.fft.fft(hist180) # corr = iq0 * np.conjugate(iq180) corr = iq180 * np.conjugate(iq0) corr /= np.abs(corr) r = np.fft.ifft(corr) # r = np.real(r) # the argmax of r should be what we want. # - only data within a few degrees are useful r[10:350] = 0 index = np.argmax(r[1:]) + 1 width = 2 peak = r[index - width:index + width + 1]
def videoStabilizationMovement(data_path, blockSize, areaOfSearch, Backward, ID): print '------Video Stabilization--------' # List all the files in the sequence dataset frames_list = sorted(os.listdir(data_path)) nFrames = len(frames_list) frame_dir_0 = os.path.join(data_path, frames_list[0]) first_frame = cv2.imread(frame_dir_0, 0) motionVectors = np.zeros( [nFrames, first_frame.shape[0], first_frame.shape[1], 2]) print 'Motion estimation step' for idx in range(0, nFrames - 1): print ' --> Analyzing frame ', frames_list[idx], '...' frame_dir_curr = os.path.join(data_path, frames_list[idx]) frame_dir_next = os.path.join(data_path, frames_list[idx + 1]) if Backward: toExplore_img = cv2.imread(frame_dir_curr, 0) curr_img = cv2.imread(frame_dir_next, 0) else: curr_img = cv2.imread(frame_dir_curr, 0) toExplore_img = cv2.imread(frame_dir_next, 0) #OF_image = calcOpticalFlowBM(curr_img, toExplore_img, blockSize, areaOfSearch) OF_image = cv2.calcOpticalFlowFarneback(curr_img, toExplore_img, pyr_scale=0.5, levels=3, winsize=30, iterations=5, poly_n=5, poly_sigma=1.2, flags=0) # showOpticalFlowHSVBlockMatching(OF_image, frames_list[idx], saveResult=True, visualization = 'HS') motionVectors[idx, :, :, 0:2] = OF_image point = [138, 83] # Reference point from which the sequence is stabilized plotMotionEstimation(motionVectors, point, name='beforeSmoothing') motionVectors[:, 138, 83, 0] = smooth(motionVectors[:, 138, 83, 0], window_len=1) motionVectors[:, 138, 83, 1] = smooth(motionVectors[:, 138, 83, 1], window_len=1) plotMotionEstimation(motionVectors, point, name='afterSmoothing') print ' -->Motion estimation completed!' print 'Motion compensation step' for idx in range(0, nFrames): print ' --> Compensating frame ', frames_list[idx], '...' frame_dir = os.path.join(data_path, frames_list[idx]) toCompensate = cv2.imread(frame_dir, 1) M = np.float32([[1, 0, -motionVectors[idx][128, 83, 0]], [0, 1, -motionVectors[idx][138, 83, 1]]]) compensated = cv2.warpAffine( toCompensate, M, (first_frame.shape[1], first_frame.shape[0])) cv2.imwrite('results/Stabilized/' + ID + '/input/' + frames_list[idx], compensated) np.save('results/Stabilized/' + ID + '/motionVectors', motionVectors) np.save('results/Stabilized/' + ID + '/point', point) return motionVectors, point
def conductGeneration(generation, corpus, previous_output): ''' Conducts a generation of learning and testing on the input data Inputs generation (int) --- the number of the generation corpus (array) --- the lemmas and their info from reading the corpus file previous_output (dict) --- the output phonology of the previous generation Returns the output of the current generation--the expected outputs for the following generation ''' # Build the right size network net = buildNetwork(input_nodes, constants.hidden_nodes, constants.output_nodes) # Build the right size training set emptytraining_set = SupervisedDataSet(input_nodes, constants.output_nodes) # Initialize corpus object and expected output dictionary training_corpus = objects.Corpus(emptytraining_set) # Iterate through tokens and convert to binary for lemma in corpus: # Iterate through cases for case, form in lemma.cases.iteritems(): # Create the input tuple form.createInputTuple(input_nodes, root_size) # Add words according to their frequencies training_corpus.addByFreq(constants.token_freq, form, previous_output[form.lemmacase]) # Print information print "--------Generation %s--------" % generation # if generation >= constants.gnvdrop_generation: # print "Genitive Case Dropped" # Construct the training set print "Constructing the training set" training_set = training_corpus.constructTrainingSet() # Construct the trainer trainer = BackpropTrainer(net, training_set) # Train print "Training the model" error = trainer.trainEpochs(constants.epochs) print "Number of Tokens in Training Set: %s" % len(training_set) print "Training Error: %s" % error results = {} # For each word in the test set, calculate output tuple print "Running the test set" # Counter to count correct. Exclude -'s from total phonemes ncorrect = 0 tot_phon = 0 for (form, input_tuple, expected_output) in training_corpus.test: # # Determine if we should drop the genitive drop_gen = generation >= constants.gnvdrop_generation # Activate the net, and smooth the output result = smooth(tuple(net.activate(input_tuple)), gendrop=drop_gen, hierarchy=constants.hierarchy) # Append output tuple to result results[form.lemmacase] = result # Hash the output tuple to get the phonological form result new_phonology = '' # Divide tuple into chunks (each 12 units, representing one phoneme) chunked_list = list(chunks(list(result), 12)) # Divide previous output tuple into chunks chunked_prev = list(chunks(list(previous_output[form.lemmacase]), 12)) for phon_index in range(len(chunked_list)): phoneme = chunked_list[phon_index] prev_phoneme = chunked_prev[phon_index] new_phonology += constants.feat_to_phon[tuple(phoneme)] # If phoneme matches, add to number correct if prev_phoneme != [0.5]*12: tot_phon += 1 if phoneme == prev_phoneme: ncorrect += 1 print form.lemmacase, form.parent.declension, form.parent.gender, new_phonology # Set input change once we figure out how to deal with the phonology form.output_change[generation] = new_phonology.replace('-', '') print "Results have been determined" print "Percentage correct in test run: {:.2f}".format(float(ncorrect)/float(tot_phon)*100) return results
def iter_smooth(array, loops=6, window_len=3): for l in np.arange(loops): array = sm.smooth(array, window_len=window_len) return array
d = d/max(d) + 0.1*noise ''' #==================================================================================== d = dMat[count, :] nbin = np.size(d) print 'size(d):', np.size(d) ''' plt.figure(20) plt.plot(x,d,'-') plt.figure(21) plt.plot(x,model,'-') plt.show() ''' # shifting the peak to center peak = np.argmax(smooth(d, 5)) d = np.roll(d, (int(nbin * 0.1) - peak)) peak = np.argmax(smooth(d, 5)) #ON_Off Pulse regions #j3 = peak - int(0.25*nbin)#/((count+1)**0.5)) j2 = peak + int(0.25 * nbin) #/((count+1)**0.5)) j3 = 1 print 'Bin Selection Range', j2, j3 # REMOVING Pedestal mean0 = np.median(d) d = d - mean0 d = d / np.max(d) #==================Off and On Pulse region==============================' offpulse2 = d[j2:(nbin - 1)]
def prep_beam(binfile, matfile, stations_model, ndays=1, nhours=1, fsamp=10., threshold_std=0.5, onebit=False, tempfilter=False, specwhite=True, timenorm=False, fact=10, new=False, fftpower=7, freq_int=(0.02, 0.4)): """ Prepare the raw data before the beamforming. This comprises bandpass filtering, cutting the data into smaller chunks, removing the mean and down-weighting strong transient signals from for example earthquakes. One can chose between several methods to remove transients: 1-bit normalization, time domain normalization, which computes a smoothed traces of the absolute amplitude and down-weights the original trace by this smoothed trace, and a threshold based method that clips the trace at a predefined factor of the traces standard deviation. Note that currently no instrument correction is applied so the input traces either have to be already corrected for the instrument response, they need to be filtered within a frequency band for which all instrument responses are flat or they need to be all from the same instruments. :param files: Day long vertical component SAC files. :param matfile: Name of the file to which the pre-processed traces are written. This saves the time of repeatedly running the pre-processing for beamformer runs with different parameters. The output file is a *.mat file that can also be read with Matlab. :param statons_model: :param nhours: Input data is cut into chunks of length of nhours. :param fsamp: Sampling frequency of the input data. :param threshold_std: Clipping factor; values greater than treshold_std * std(trace) are set to threshold_std. :param onebit: Turn on/off 1-bit normalization :param tempfilter: Turn on/off threshold based clipping. :param specwhite: Turn on/off spectral whitening (only retain spectral phase and set spectral amplitude to one. :param timenomr: Turn on/off time domain normalization. :param fact: Decimation factor. :param new: If set to false it will try to load all return values from matfile. If true it will compute all return values from scratch. :param fftpower: Length of data chunks cut before the FFT. :param freq_int: Frequency interval for the band-pass filter. :return fseis: Pre-processed traces in the frequency domain. :return freqs: Frequency array corresponding to fseis :return slats: Station latitudes. :return slons: Station longitudes. :return dt: Sampling interval after decimation. :return seissmall: Pre-processed traces in the time domain. """ if new: print (' >> Prepare raw data before beamforming...') ntimes = int(ndays) * int(round(24 / nhours)) step = int( nhours * 3600 * fsamp / fact ) stations = [] slons = array([]) slats = array([]) nfiles = np.shape(binfile)[0] seisband = zeros((nfiles, ntimes, step)) freqs = fftfreq(2 ** fftpower, 1. / (fsamp / fact)) for i, (t, s) in enumerate(zip(binfile,stations_model)): data = t print (i, s.station) slons = append(slons, s.lon) slats = append(slats, s.lat) stations.append(s.station) data -= data.mean() data = bandpass(data, freqmin=freq_int[0], freqmax=freq_int[1], df=fsamp, corners=4, zerophase=True) if fact != 1: data = integer_decimation(data, fact) npts = len(data) df = fsamp / fact dt = 1./df seis0 = zeros(ndays * 24 * 3600 * int(df)) istart = int(round(((UTCDateTime(0).hour * 60 + UTCDateTime(0).minute) * 60\ + UTCDateTime(0).second) * df)) if timenorm: smoothdata = smooth(abs(data), window_len=257, window='flat') data /= smoothdata if np.isnan(data).any(): data = zeros(ndays * 24 * 3600 * int(df)) try: seis0[istart:(istart + npts)] = data except (ValueError, e): print ('Problem with %s'%s.station ) raise ValueError seis0 -= seis0.mean() # iterate over nhours for j in np.arange(ntimes): ilow = j * step iup = (j + 1) * step seisband[i, j, :] = seis0[ilow:iup] seisband[i, j, :] -= seisband[i, j, :].mean() if onebit: seisband[i, j, :] = sign(seisband[i, j, :]) if tempfilter: sigmas = seisband[i, :, :].std(axis=1, ddof=1) sgm = ma.masked_equal(array(sigmas), 0.).compressed() sigma = sqrt(sum(sgm ** 2) / sgm.size) threshold = threshold_std * sigma seisband[i] = where(abs(seisband[i]) > threshold, threshold * sign(seisband[i]), seisband[i]) seisband[i] = apply_along_axis(lambda e: e - e.mean(), 1, seisband[i]) ismall = 2 **fftpower ipick = arange(ismall) taper = cosine_taper(len(ipick)) n = ndays * nhours * 3600 * df nsub = int(np.floor(n / ismall)) # Number of time pieces -20 mins long each seissmall = zeros((nfiles, ntimes, nsub, len(ipick))) for ii in np.arange(nfiles): for jj in np.arange(ntimes): for kk in np.arange(nsub): seissmall[ii, jj, kk, :] = seisband[ii, jj, kk * ismall + ipick] * taper fseis = fft(seissmall, n=2**fftpower, axis=3) if np.isnan(fseis).any(): print ("NaN found") return ind = np.where((freqs > freq_int[0]) & (freqs < freq_int[1]))[0] fseis = fseis[:, :, :, ind] if specwhite: fseis = exp(angle(fseis) * 1j) sio.savemat(matfile, {'fseis':fseis, 'slats':slats, 'slons':slons, 'dt':dt, 'freqs':freqs[ind]}) return fseis, freqs[ind], slats, slons, dt else: print (' >> Reading %s and passing preprocess...'%matfile) a = sio.loadmat(matfile) fseis = a['fseis'] freqs = np.squeeze(a['freqs']) slats = np.squeeze(a['slats']) slons = np.squeeze(a['slons']) dt = a['dt'][0][0] return fseis, freqs, slats, slons, dt
def compareUV(data, threeDim, depth=5, plot=False, save_csv=False, debug=False, debug_plot=False): """ Does a comprehensive validation process between modeled and observed. Outputs a list of important statistics for each variable, calculated using the TidalStats class Inputs: - data = dictionary containing all necessary observed and model data - threeDim = boolean flag, 3D or not Outputs: - elev_suite = dictionary of useful statistics for sea elevation - speed_suite = dictionary of useful statistics for flow speed - dir_suite = dictionary of useful statistics for flow direction - u_suite = dictionary of useful statistics for u velocity component - v_suite = dictionary of useful statistics for v velocity component - vel_suite = dictionary of useful statistics for signed flow velocity - csp_suite = dictionary of useful statistics for cubic flow speed Options: - depth = interpolation depth from surface, float - plot = boolean flag for plotting results - save_csv = boolean flag for saving statistical benchmarks in csv file """ if debug: print "CompareUV..." # take data from input dictionary mod_time = data['mod_time'] if not data['type'] == 'Drifter': obs_time = data['obs_time'] mod_el = data['mod_timeseries']['elev'] obs_el = data['obs_timeseries']['elev'] else: obs_time = data['mod_time'] # Save path & create folder name = data['name'] save_path = name.split('/')[-1].split('.')[0] + '/' while exists(save_path): save_path = save_path[:-1] + '_bis/' mkdir(save_path) # Check if 3D simulation if threeDim: obs_u_all = data['obs_timeseries']['u'] obs_v_all = data['obs_timeseries']['v'] mod_u_all = data['mod_timeseries']['u'] mod_v_all = data['mod_timeseries']['v'] bins = data['obs_timeseries']['bins'] siglay = data['mod_timeseries']['siglay'] # use depth interpolation to get a single timeseries mod_depth = mod_el + np.mean(obs_el[~np.isnan(obs_el)]) (mod_u, obs_u) = depthFromSurf(mod_u_all, mod_depth, siglay, obs_u_all, obs_el, bins, depth=depth, debug=debug, debug_plot=debug_plot) (mod_v, obs_v) = depthFromSurf(mod_v_all, mod_depth, siglay, obs_v_all, obs_el, bins, depth=depth, debug=debug, debug_plot=debug_plot) else: if not data['type'] == 'Drifter': obs_u = data['obs_timeseries']['ua'] obs_v = data['obs_timeseries']['va'] mod_u = data['mod_timeseries']['ua'] mod_v = data['mod_timeseries']['va'] else: obs_u = data['obs_timeseries']['u'] obs_v = data['obs_timeseries']['v'] mod_u = data['mod_timeseries']['u'] mod_v = data['mod_timeseries']['v'] if debug: print "...convert times to datetime..." mod_dt, obs_dt = [], [] for i in mod_time: mod_dt.append(dn2dt(i)) for j in obs_time: obs_dt.append(dn2dt(j)) if debug: print "...put data into a useful format..." mod_spd = np.sqrt(mod_u**2.0 + mod_v**2.0) obs_spd = np.sqrt(obs_u**2.0 + obs_v**2.0) mod_dir = np.arctan2(mod_v, mod_u) * 180.0 / np.pi obs_dir = np.arctan2(obs_v, obs_u) * 180.0 / np.pi if not data['type'] == 'Drifter': obs_el = obs_el - np.mean(obs_el[~np.isnan(obs_el)]) # Chose the component with the biggest variance as sign reference if np.var(mod_v) > np.var(mod_u): mod_signed = np.sign(mod_v) obs_signed = np.sign(obs_v) else: mod_signed = np.sign(mod_u) obs_signed = np.sign(obs_u) if debug: print "...check if the modeled data lines up with the observed data..." if (mod_time[-1] < obs_time[0] or obs_time[-1] < mod_time[0]): raise PyseidonError("---time periods do not match up---") else: if debug: print "...interpolate the data onto a common time step for each data type..." if not data['type'] == 'Drifter': # elevation (mod_el_int, obs_el_int, step_el_int, start_el_int) = smooth(mod_el, mod_dt, obs_el, obs_dt, debug=debug, debug_plot=debug_plot) # speed (mod_sp_int, obs_sp_int, step_sp_int, start_sp_int) = smooth(mod_spd, mod_dt, obs_spd, obs_dt, debug=debug, debug_plot=debug_plot) # direction (mod_dr_int, obs_dr_int, step_dr_int, start_dr_int) = smooth(mod_dir, mod_dt, obs_dir, obs_dt, debug=debug, debug_plot=debug_plot) # u velocity (mod_u_int, obs_u_int, step_u_int, start_u_int) = smooth(mod_u, mod_dt, obs_u, obs_dt, debug=debug, debug_plot=debug_plot) # v velocity (mod_v_int, obs_v_int, step_v_int, start_v_int) = smooth(mod_v, mod_dt, obs_v, obs_dt, debug=debug, debug_plot=debug_plot) # velocity i.e. signed speed (mod_ve_int, obs_ve_int, step_ve_int, start_ve_int) = smooth(mod_spd * mod_signed, mod_dt, obs_spd * obs_signed, obs_dt, debug=debug, debug_plot=debug_plot) # cubic signed speed #mod_cspd = mod_spd**3.0 #obs_cspd = obs_spd**3.0 mod_cspd = mod_signed * mod_spd**3.0 obs_cspd = obs_signed * obs_spd**3.0 (mod_cspd_int, obs_cspd_int, step_cspd_int, start_cspd_int) = smooth(mod_cspd, mod_dt, obs_cspd, obs_dt, debug=debug, debug_plot=debug_plot) else: # Time steps step = mod_time[1] - mod_time[0] start = mod_time[0] # Already interpolated, so no need to use smooth... # speed (mod_sp_int, obs_sp_int, step_sp_int, start_sp_int) = (mod_spd, obs_spd, step, start) # direction (mod_dr_int, obs_dr_int, step_dr_int, start_dr_int) = (mod_dir, obs_dir, step, start) # u velocity (mod_u_int, obs_u_int, step_u_int, start_u_int) = (mod_u, obs_u, step, start) # v velocity (mod_v_int, obs_v_int, step_v_int, start_v_int) = (mod_v, obs_v, step, start) # velocity i.e. signed speed (mod_ve_int, obs_ve_int, step_ve_int, start_ve_int) = (mod_spd, obs_spd, step, start) # cubic signed speed #mod_cspd = mod_spd**3.0 #obs_cspd = obs_spd**3.0 mod_cspd = mod_signed * mod_spd**3.0 obs_cspd = obs_signed * obs_spd**3.0 (mod_cspd_int, obs_cspd_int, step_cspd_int, start_cspd_int) = (mod_cspd, obs_cspd, step, start) if debug: print "...remove directions where velocities are small..." MIN_VEL = 0.1 indexMin = np.where(obs_sp_int < MIN_VEL) obs_dr_int[indexMin] = np.nan obs_u_int[indexMin] = np.nan obs_v_int[indexMin] = np.nan obs_ve_int[indexMin] = np.nan obs_cspd_int[indexMin] = np.nan indexMin = np.where(mod_sp_int < MIN_VEL) mod_dr_int[indexMin] = np.nan mod_u_int[indexMin] = np.nan mod_v_int[indexMin] = np.nan mod_ve_int[indexMin] = np.nan mod_cspd_int[indexMin] = np.nan if debug: print "...get stats for each tidal variable..." gear = data['type'] # Type of measurement gear (drifter, adcp,...) if not gear == 'Drifter': elev_suite = tidalSuite(gear, mod_el_int, obs_el_int, step_el_int, start_el_int, [], [], [], [], [], [], kind='elevation', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) else: elev_suite = [] speed_suite = tidalSuite(gear, mod_sp_int, obs_sp_int, step_sp_int, start_sp_int, [], [], [], [], [], [], kind='speed', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) dir_suite = tidalSuite(gear, mod_dr_int, obs_dr_int, step_dr_int, start_dr_int, mod_u, obs_u, mod_v, obs_v, mod_dt, obs_dt, kind='direction', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) u_suite = tidalSuite(gear, mod_u_int, obs_u_int, step_u_int, start_u_int, [], [], [], [], [], [], kind='u velocity', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) v_suite = tidalSuite(gear, mod_v_int, obs_v_int, step_v_int, start_v_int, [], [], [], [], [], [], kind='v velocity', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) # TR: requires special treatments from here on vel_suite = tidalSuite(gear, mod_ve_int, obs_ve_int, step_ve_int, start_ve_int, mod_u, obs_u, mod_v, obs_v, mod_dt, obs_dt, kind='velocity', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) csp_suite = tidalSuite(gear, mod_cspd_int, obs_cspd_int, step_cspd_int, start_cspd_int, mod_u, obs_u, mod_v, obs_v, mod_dt, obs_dt, kind='cubic speed', plot=plot, save_csv=save_csv, save_path=save_path, debug=debug, debug_plot=debug_plot) # output statistics in useful format if debug: print "...CompareUV done." return (elev_suite, speed_suite, dir_suite, u_suite, v_suite, vel_suite, csp_suite)
def pipeline(params, nimage, roiA, roiB, wm): if params.get('pipeline')=='STOCHASTIC': #print "Pipeline : STOCHASTIC" ppserversL=("localhost",) #ppserversL=("*",) data = nimage.getImage() ####! swap x and z for volume coming from Slicer - do not forget tp apply the inverse before to send them back data = data.swapaxes(2,0) print 'pipeline data type : %s' % str(data.dtype) #### shpD = data.shape print 'pipeline data shape : %s' % str(shpD) orgS = nimage.get('origin') org = [float(orgS[0]), float(orgS[1]), float(orgS[2])] print 'Origin : %s:%s:%s' % (str(org[0]), str(org[1]), str(org[2])) spaS = nimage.get('spacing') spa = [float(spaS[0]), float(spaS[1]), float(spaS[2])] print 'Spacing : %s:%s:%s' % (str(spa[0]), str(spa[1]), str(spa[2])) G = nimage.get('grads') b = nimage.get('bval') i2r = nimage.get('ijk2ras') i2rd = nimage.get('ijk2rasd') mu = nimage.get('mu') dims = nimage.get('dimensions') #print 'DWI : ', params.get('dwi') isInRoiA = False if params.hasKey('roiA'): roiAR = numpy.fromstring(roiA.getImage(), 'uint16') roiAR = roiAR.reshape(shpD[2], shpD[1], shpD[0]) # because come from Slicer - will not send them back so swap them one for all roiAR = roiAR.swapaxes(2,0) roiAR[roiAR>0]=1 roiA.setImage(roiAR) isInRoiA = True #print "RoiA : %s" % str(roiAR.shape) isDir = os.access('paths0', os.F_OK) if not isDir: os.mkdir('paths0') isInRoiB = False if params.hasKey('roiB'): if params.get('roiB') != params.get('roiA'): roiBR = numpy.fromstring(roiB.getImage(), 'uint16') roiBR = roiBR.reshape(shpD[2], shpD[1], shpD[0]) roiBR = roiBR.swapaxes(2,0) roiBR[roiBR>0]=1 roiB.setImage(roiBR) isInRoiB = True #print "RoiB : %s" % str(roiBR.shape) isDir = os.access('paths1', os.F_OK) if not isDir: os.mkdir('paths1') isInWM = False if params.hasKey('wm'): wmR = numpy.fromstring(wm.getImage(), 'uint16') wmR = wmR.reshape(shpD[2], shpD[1], shpD[0]) wmR = wmR.swapaxes(2,0) wm.setImage(wmR) isInWM = True #print "WM : %s" % str(wmR.shape) isInTensor = False print "Input volumes loaded!" # values per default smoothEnabled = False wmEnabled = True infWMThres = 300 supWMThres = 900 tensEnabled =True bLine = 0 stEnabled = True totalTracts = 500 maxLength = 200 stepSize = 0.5 stopEnabled = True fa = 0.0 cmEnabled = False probMode = 0 # got from client # special handling for bools if params.hasKey('smoothEnabled'): smoothEnabled = bool(int(params.get('smoothEnabled'))) if params.hasKey('wmEnabled'): wmEnabled = bool(int(params.get('wmEnabled'))) if params.hasKey('tensEnabled'): tensEnabled = bool(int(params.get('tensEnabled'))) if params.hasKey('stEnabled'): stEnabled = bool(int(params.get('stEnabled'))) if params.hasKey('cmEnabled'): cmEnabled = bool(int(params.get('cmEnabled'))) if params.hasKey('spaceEnabled'): spaceEnabled = bool(int(params.get('spaceEnabled'))) if params.hasKey('stopEnabled'): stopEnabled = bool(int(params.get('stopEnabled'))) if params.hasKey('faEnabled'): faEnabled = bool(int(params.get('faEnabled'))) if params.hasKey('traceEnabled'): traceEnabled = bool(int(params.get('traceEnabled'))) if params.hasKey('modeEnabled'): modeEnabled = bool(int(params.get('modeEnabled'))) # can handle normally FWHM = numpy.ones((3), 'float') if params.hasKey('stdDev'): FWHM[0] = float(params.get('stdDev')[0]) FWHM[1] = float(params.get('stdDev')[1]) FWHM[2] = float(params.get('stdDev')[2]) print "FWHM: %s:%s:%s" % (str(FWHM[0]), str(FWHM[1]), str(FWHM[2])) if params.hasKey('infWMThres'): infWMThres = int(params.get('infWMThres')) print "infWMThres: %s" % str(infWMThres) if params.hasKey('supWMThres'): supWMThres = int(params.get('supWMThres')) print "supWMThres: %s" % str(supWMThres) if params.hasKey('bLine'): bLine = int(params.get('bLine')) print "bLine: %s" % str(bLine) if params.hasKey('tensMode'): tensMode = params.get('tensMode') print "tensMode: %s" % str(tensMode) if params.hasKey('totalTracts'): totalTracts = int(params.get('totalTracts')) print "totalTracts: %s" % str(totalTracts) if params.hasKey('maxLength'): maxLength = int(params.get('maxLength')) print "maxLength: %s" % str(maxLength) if params.hasKey('stepSize'): stepSize = float(params.get('stepSize')) print "stepSize: %s" % str(stepSize) if params.hasKey('fa'): fa = float(params.get('fa')) print "fa: %s" % str(fa) if params.hasKey('probMode'): probMode = params.get('probMode') print "probMode: %s" % str(probMode) if params.hasKey('lengthEnabled'): lengthEnabled = params.get('lengthEnabled') print "lengthEnabled: %s" % str(lengthEnabled) if params.hasKey('lengthClass'): lengthClass = params.get('lengthClass') print "lengthClass: %s" % str(lengthClass) ngrads = shpD[3] #b.shape[0] print "Number of gradients : %s" % str(ngrads) G = G.reshape(ngrads,3) b = b.reshape(ngrads,1) i2r = i2r.reshape(4,4) i2rd = i2rd.reshape(4,4) mu = mu.reshape(4,4) r2i = numpy.linalg.inv(i2r) r2id = numpy.linalg.inv(i2rd) mu2 = numpy.dot(r2id[:3, :3], mu[:3, :3]) G2 = numpy.dot(G, mu2[:3, :3].T) vts = vects.vectors print "Tensor flag : %s" % str(tensEnabled) cm = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') cm2 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') cm3 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') cm4 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') if smoothEnabled: for k in range(shpD[3]): timeSM0 = time.time() data[...,k] = sm.smooth(data[...,k], FWHM, numpy.array([ spa[0], spa[1], spa[2] ],'float')) print "Smoothing DWI volume %i in %s sec" % (k, str(time.time()-timeSM0)) if wmEnabled: wm = tensPP.EvaluateWM0(data, bLine, infWMThres, supWMThres) if isInRoiA: # correcting brain mask with roi A print "Correcting mask based on roiA" tmpA = roiA.getImage() wm[tmpA>0]=1 if isInRoiB: # correcting brain mask with roi A & B print "Correcting mask based on roiB" tmpB = roiB.getImage() wm[tmpB>0]=1 else: # avoid singularities in data minVData = 10 wm = tensPP.EvaluateWM0(data, bLine, minVData, data[..., bLine].max()) wmEnabled = True # fix if isInWM or wmEnabled: isDir = os.access('masks', os.F_OK) if not isDir: os.mkdir('masks') tmpF = './masks/' numpy.save(tmpF + 'wm.npy', wm) indx = numpy.transpose(wm.nonzero()) print 'Total Number of voxels : ', shpD[0]*shpD[1]*shpD[2]*ngrads print 'Masked voxels : ', indx.shape[0]*ngrads print 'Index shape : ', indx.shape dataf = data.flatten() cId = numpy.zeros((indx.shape[0]), 'uint32') mdata = numpy.zeros((indx.shape[0], ngrads), data.dtype) for i in range(indx.shape[0]): cId[i] = indx[i][0]*shpD[1]*shpD[2]*shpD[3] + indx[i][1]*shpD[2]*shpD[3] + indx[i][2]*shpD[3] mdata[i,:]= dataf[cId[i]:cId[i]+ngrads] numpy.save(tmpF + 'index.npy', cId) numpy.save(tmpF + 'mdata.npy', mdata) if cmEnabled: print "Compute tensor" timeS1 = time.time() roiFilterOnly = False monoP = False # multiprocessing support dataBlocks = [] wmBlocks = [] nCpu = 2 # could be set to the number of available cores nParts = 1 if shpD[2]>0 and nCpu>0 : job_server = pp.Server(ppservers=ppserversL) ncpusL = job_server.get_ncpus() print "Number of cores on local machine : %s" % str(ncpusL) print "Number of active computing nodes : %s" % str(nCpu) if shpD[2] >= nCpu: nParts = nCpu else: nParts = shpD[2] for i in range(nParts): datax = data[:, :, i*shpD[2]/nParts:(i+1)*shpD[2]/nParts, :] print "data block %i dimension : %s" % (i, str(datax.shape)) dataBlocks.append(datax) if isInWM or wmEnabled: wmx = wm[:, :, i*shpD[2]/nParts:(i+1)*shpD[2]/nParts] wmBlocks.append(wmx) else: monoP = True if not monoP: jobs = [] job_server.set_ncpus(ncpus = ncpusL) for i in range(nParts): jobs.append(job_server.submit(tensPP.EvaluateTensorX1, (dataBlocks[i], G2.T, b.T, wmBlocks[i],),(tensPP.ComputeAFunctional, tensPP.ComputeTensorFunctional,), ("numpy","time",) )) tBlocks = [] for i in range(nParts): tBlocks.append(jobs[i]()) lV = numpy.zeros((shpD[0], shpD[1], shpD[2], 3) , 'float') EV = numpy.zeros((shpD[0], shpD[1], shpD[2], 3, 3), 'float' ) xVTensor = numpy.zeros((shpD[0], shpD[1], shpD[2], 7), 'float') xYTensor = numpy.zeros((shpD[0], shpD[1], shpD[2], 9), 'float') xTensor0 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'float') for i in range(nParts): EV[:, :, i*shpD[2]/nParts:(i+1)*shpD[2]/nParts, ...]= tBlocks[i][0] lV[:, :, i*shpD[2]/nParts:(i+1)*shpD[2]/nParts, :]= tBlocks[i][1] xVTensor[:, :, i*shpD[2]/nParts:(i+1)*shpD[2]/nParts, ...]= tBlocks[i][2] xYTensor[:, :, i*shpD[2]/nParts:(i+1)*shpD[2]/nParts, ...]= tBlocks[i][3] xTensor0[...]= xVTensor[..., 0] lVType = lV.dtype EVType = EV.dtype xVTensorType = xVTensor.dtype wmType = wm.dtype # computation of alpha, beta, logmu0 and principal eigenvector tdata = numpy.zeros((indx.shape[0], 6), 'float') for i in range(indx.shape[0]): cId[i] = indx[i][0]*shpD[1]*shpD[2]*shpD[3] + indx[i][1]*shpD[2]*shpD[3] + indx[i][2]*shpD[3] l = lV[indx[i][0], indx[i][1], indx[i][2], :] index = numpy.argsort(abs(l))[::-1] l =l[index,:] # Set point estimates in the Constrained model E = EV[indx[i][0], indx[i][1], indx[i][2], ...] alpha = (l[1]+l[2])/2 beta = l[0] - alpha logmu0 = xTensor0[indx[i][0], indx[i][1], indx[i][2]] e = E[:, index[0]] tdata[i,0]= alpha tdata[i,1]= beta tdata[i,2]= logmu0 tdata[i,3:]= e numpy.save(tmpF + 'tdata.npy', tdata) isDir = os.access('tensors', os.F_OK) if not isDir: os.mkdir('tensors') tmpF = './tensors/' numpy.save(tmpF + 'eigenv.npy', EV) numpy.save(tmpF + 'lambda.npy', lV) numpy.save(tmpF + 'tensor.npy', xVTensor) numpy.save(tmpF + 'tensor0.npy', xTensor0) numpy.save(tmpF + 'vectors.npy', vts.T) dateT = str(int(round(time.time()))) isDir = os.access('outputs', os.F_OK) if not isDir: os.mkdir('outputs') tmpF = './outputs/' i2r.tofile(tmpF + 'trafo_' + dateT + '.ijk') if smoothEnabled: ga = data[..., bLine] ga = ga.swapaxes(2,0) tmp= 'smooth_' + dateT ga.tofile(tmpF + tmp + '.data') createParams(ga, tmpF + tmp) if wmEnabled: wm = wm.swapaxes(2,0) tmp= 'brain_' + dateT wm.tofile(tmpF + tmp + '.data') createParams(wm, tmpF + tmp) if cmEnabled: xVTensor = xVTensor.swapaxes(2,0) xVTensor = xVTensor.astype('float32') # slicerd do not support double type yet xYTensor = xYTensor.swapaxes(2,0) xYTensor = xYTensor.astype('float32') # slicerd do not support double type yet tmp= 'tensor_' + dateT xYTensor.tofile(tmpF + tmp + '.data') createParams(xYTensor, tmpF + tmp, True) if faEnabled: faMap = tensPP.CalculateFA0(lV) faMap = faMap.swapaxes(2,0) tmp= 'fa_' + dateT faMap.tofile(tmpF + tmp + '.data') createParams(faMap, tmpF + tmp) if traceEnabled: trMap = tensPP.CalculateTrace0(lV) trMap = trMap.swapaxes(2,0) tmp= 'trace_' + dateT trMap.tofile(tmpF + tmp + '.data') createParams(trMap, tmpF + tmp) if modeEnabled: moMap = tensPP.CalculateMode0(lV) moMap = moMap.swapaxes(2,0) tmp= 'mode_' + dateT moMap.tofile(tmpF + tmp + '.data') createParams(moMap, tmpF + tmp) del tBlocks del dataBlocks del wmBlocks del lV del EV del xVTensor del xYTensor else: pass print "Compute tensor in %s sec" % str(time.time()-timeS1) print "Track fibers" if not stopEnabled: fa = 0.0 if isInRoiA: # ROI A print "Search ROI A" roiP = cmpV.march0InVolume(roiA.getImage()) shpR = roiP.shape print "ROI A dimension : %s" % str(shpR) blocksize = totalTracts IJKstartpoints = [] monoP = False nParts = 1 if shpR[0]>0 and nCpu>0 : if shpR[0] >= nCpu: nParts = nCpu else: nParts = shpR[0] for i in range(nParts): roiPx = roiP[i*shpR[0]/nParts:(i+1)*shpR[0]/nParts, :] print "ROI A %i dimension : %s" % (i, str(roiPx.shape)) IJKstartpoints.append(numpy.tile(roiPx,( blocksize, 1))) else: IJKstartpoints.append(numpy.tile(roiP,( blocksize, 1))) monoP = True timeS2 = time.time() # multiprocessing print "Data type : %s" % str(data.dtype) if not monoP: jobs = [] for i in range(nParts): jobs.append(job_server.submit(trackPP.TrackFiberYFM40, (i, 0, params.get('location'), nimage.get('fullname'), nimage.get('type'), shpD, b.T, G2.T, IJKstartpoints[i].T, r2i, i2r, r2id, i2rd, spa,\ 'vectors.npy', vects.vectors.dtype, vects.vectors.T.shape, 'lambda.npy', lVType, 'eigenv.npy', EVType, 'tensor0.npy', xVTensorType, 'wm.npy' ,\ wmType, stepSize, maxLength, fa, spaceEnabled,),(), ("numpy","time",) )) res = jobs[0]() for i in range(nParts-1): res = jobs[i+1]() print "Track fibers in %s sec" % str(time.time()-timeS2) if not roiFilterOnly: print "Connect tract" jobs = [] cm = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') for j in range(nParts): print "Number of paths : %s" % str(IJKstartpoints[j].shape[0]) pathsIJKId = './paths0/' + 'unit_' + str(j) + '_IJK.npy' pathsLENId = './paths0/' + 'unit_' + str(j) + '_LEN.npy' if probMode=='binary': jobs.append(job_server.submit(trackPP.ConnectFibersPZM0, ( params.get('location'), pathsIJKId, pathsLENId, shpD, lengthEnabled),\ (trackPP.ComputeConnectFibersFunctionalP0, trackPP.ComputeConnectFibersFunctionalP1, trackPP.ComputeConnectFibersFunctionalP2,),\ ("numpy","time",) )) elif probMode=='cumulative': jobs.append(job_server.submit(trackPP.ConnectFibersPZM1, ( params.get('location'), pathsIJKId, pathsLENId, shpD, lengthEnabled),\ (trackPP.ComputeConnectFibersFunctionalP0, trackPP.ComputeConnectFibersFunctionalP1, trackPP.ComputeConnectFibersFunctionalP2,),\ ("numpy","time",) )) else: jobs.append(job_server.submit(trackPP.ConnectFibersPZM2, ( params.get('location'), pathsIJKId, pathsLENId, shpD, lengthEnabled),\ (trackPP.ComputeConnectFibersFunctionalP0, trackPP.ComputeConnectFibersFunctionalP1, trackPP.ComputeConnectFibersFunctionalP2,),\ ("numpy","time",) )) cm = jobs[0]() for j in range(nParts-1): cm += jobs[j+1]() else: pass if isInRoiB: # ROI B print "Search ROI B" roiP2 = cmpV.march0InVolume(roiB.getImage()) shpR2 = roiP2.shape print "ROI B dimension : %s" % str(shpR2) blocksize = totalTracts IJKstartpoints2 = [] monoP = False nParts2 = 1 if shpR2[0]>0 and nCpu>0 : if shpR2[0] >= nCpu: nParts2 = nCpu else: nParts2 = shpR2[0] for i in range(nParts2): roiPx = roiP2[i*shpR2[0]/nParts2:(i+1)*shpR2[0]/nParts2, :] print "ROI B %i dimension : %s" % (i, str(roiPx.shape)) IJKstartpoints2.append(numpy.tile(roiPx,( blocksize, 1))) else: IJKstartpoints2.append(numpy.tile(roiP2,( blocksize, 1))) monoP = True timeS3 = time.time() # multiprocessing print "Data type : %s" % str(data.dtype) if not monoP: jobs = [] for i in range(nParts2): jobs.append(job_server.submit(trackPP.TrackFiberYFM40, (i, 1, params.get('location'), nimage.get('fullname'), nimage.get('type'), shpD, b.T, G2.T, IJKstartpoints2[i].T, r2i, i2r, r2id, i2rd, spa,\ 'vectors.npy', vects.vectors.dtype, vects.vectors.T.shape, 'lambda.npy', lVType, 'eigenv.npy', EVType, 'tensor0.npy', xVTensorType, 'wm.npy' ,\ wmType, stepSize, maxLength, fa, spaceEnabled,),(), ("numpy","time",) )) res = jobs[0]() for i in range(nParts2-1): res = jobs[i+1]() print "Track fibers in %s sec" % str(time.time()-timeS3) if not roiFilterOnly: print "Connect tract" jobs = [] cm2 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') for j in range(nParts2): print "Number of paths : %s" % str(IJKstartpoints2[j].shape[0]) pathsIJKId = './paths1/' + 'unit_' + str(j) + '_IJK.npy' pathsLENId = './paths1/' + 'unit_' + str(j) + '_LEN.npy' if probMode=='binary': jobs.append(job_server.submit(trackPP.ConnectFibersPZM0, ( params.get('location'), pathsIJKId, pathsLENId, shpD, lengthEnabled),\ (trackPP.ComputeConnectFibersFunctionalP0, trackPP.ComputeConnectFibersFunctionalP1, trackPP.ComputeConnectFibersFunctionalP2,),\ ("numpy","time",) )) elif probMode=='cumulative': jobs.append(job_server.submit(trackPP.ConnectFibersPZM1, ( params.get('location'), pathsIJKId, pathsLENId, shpD, lengthEnabled),\ (trackPP.ComputeConnectFibersFunctionalP0, trackPP.ComputeConnectFibersFunctionalP1, trackPP.ComputeConnectFibersFunctionalP2,),\ ("numpy","time",) )) else: jobs.append(job_server.submit(trackPP.ConnectFibersPZM2, ( params.get('location'), pathsIJKId, pathsLENId, shpD, lengthEnabled),\ (trackPP.ComputeConnectFibersFunctionalP0, trackPP.ComputeConnectFibersFunctionalP1, trackPP.ComputeConnectFibersFunctionalP2,),\ ("numpy","time",) )) cm2 = jobs[0]() for j in range(nParts2-1): cm2 += jobs[j+1]() else: pass if isInRoiA and isInRoiB: if not monoP: vicinity= 1 threshold = 0.1 minLength = 4 print "Try out connecting" jobs = [] cm3 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') counter1 = 0 counter2 = 0 counterA1 = 0 counterB1 = 0 counterA2 = 0 counterB2 = 0 Pr = 0.0 Fa = 0.0 Wa = 0.0 for i in range(nParts): print "Number of paths : %s" % str(IJKstartpoints[i].shape[0]) #pathsRASId = './paths0/' + 'unit_' + str(i) + '_RAS.npy' pathsIJKId = './paths0/' + 'unit_' + str(i) + '_IJK.npy' #pathsANISId = './paths0/' + 'unit_' + str(i) + '_ANIS.npy' pathsLOGPId = './paths0/' + 'unit_' + str(i) + '_LOGP.npy' pathsLENId = './paths0/' + 'unit_' + str(i) + '_LEN.npy' jobs.append(job_server.submit(trackPP.FilterFibersZM0, (params.get('location'), pathsIJKId, pathsLOGPId, pathsLENId, roiA.getImage(), roiB.getImage(), shpD,\ counter1, counter2, counterA1, counterB1, counterA2, counterB2, Pr, threshold, vicinity), (), ("numpy","time",) )) cm3 = jobs[0]() for i in range(nParts-1): cm3 += jobs[i+1]() print "Filtering of fibers done from region A to region B" if counter1>0: print "Number of curves connecting : %s" % str(counter1) print "Mean probability : %s" % str(Pr/float(counter1)) jobs = [] cm4 = numpy.zeros((shpD[0], shpD[1], shpD[2]), 'uint32') counter1 = 0 counter2 = 0 counterA1 = 0 counterB1 = 0 counterA2 = 0 counterB2 = 0 Pr = 0.0 Fa = 0.0 Wa = 0.0 for i in range(nParts2): print "Number of paths : ", str(IJKstartpoints2[i].shape[0]) #pathsRASId = './paths1/' + 'unit_' + str(i) + '_RAS.npy' pathsIJKId = './paths1/' + 'unit_' + str(i) + '_IJK.npy' #pathsANISId = './paths1/' + 'unit_' + str(i) + '_ANIS.npy' pathsLOGPId = './paths1/' + 'unit_' + str(i) + '_LOGP.npy' pathsLENId = './paths1/' + 'unit_' + str(i) + '_LEN.npy' jobs.append(job_server.submit(trackPP.FilterFibersZM0, (params.get('location'), pathsIJKId, pathsLOGPId, pathsLENId, roiB.getImage(), roiA.getImage(), shpD,\ counter1, counter2, counterA1, counterB1, counterA2, counterB2, Pr, threshold, vicinity), (), ("numpy","time",) )) cm4 = jobs[0]() for i in range(nParts2-1): cm4 += jobs[i+1]() print "Filtering of fibers done from region B to region A" if counter1>0: print "Number of curves connecting : %s" % str(counter1) print "Mean probability : %s" % str(Pr/float(counter1)) else: pass else: print "No tractography to execute!" if cmEnabled: if isInRoiA and not roiFilterOnly: if not (cm == 0).all(): cm = cm.swapaxes(2,0) tmp= 'cmA_' + dateT cm.tofile(tmpF + tmp + '.data') createParams(cm, tmpF + tmp) if isInRoiB and not roiFilterOnly: if not (cm2 == 0).all(): cm2 = cm2.swapaxes(2,0) tmp= 'cmB_' + dateT cm2.tofile(tmpF + tmp + '.data') createParams(cm2, tmpF + tmp) if isInRoiA and isInRoiB and not roiFilterOnly: cm1a2 = cm[...]*cm2[...]/2.0 cm1a2 = cm1a2.astype('uint32') if not (cm1a2 == 0).all(): tmp= 'cmAandB_' + dateT cm1a2.tofile(tmpF + tmp + '.data') createParams(cm1a2, tmpF + tmp) tmp= 'cmFAandB_' + dateT cm1a2f = cm1a2/float(cm1a2.max()) cm1a2f.astype('float32') cm1a2f.tofile(tmpF + tmp + '.data') createParams(cm1a2f, tmpF + tmp) cm1o2 = (cm[...]+cm2[...])/2.0 cm1o2 = cm1o2.astype('uint32') if not (cm1o2 == 0).all(): tmp= 'cmAorB_' + dateT cm1o2.tofile(tmpF + tmp + '.data') createParams(cm1o2, tmpF + tmp) tmp= 'cmFAorB_' + dateT cm1o2f = cm1o2/float(cm1o2.max()) cm1o2f.astype('float32') cm1o2f.tofile(tmpF + tmp + '.data') createParams(cm1o2f, tmpF + tmp) if isInRoiA and isInRoiB: if not (cm3 == 0).all(): tmp= 'cmA2B_' + dateT cm3 = cm3.swapaxes(2,0) cm3.tofile(tmpF + tmp + '.data') createParams(cm3, tmpF + tmp) tmp= 'cmFA2B_' + dateT cm3f = cm3/float(cm3.max()) cm3f.astype('float32') cm3f.tofile(tmpF + tmp + '.data') createParams(cm3f, tmpF + tmp) if not (cm4 == 0).all(): tmp= 'cmB2A_' + dateT cm4 = cm4.swapaxes(2,0) cm4.tofile(tmpF + tmp + '.data') createParams(cm4, tmpF + tmp) tmp= 'cmFB2A_' + dateT cm4f = cm4/float(cm4.max()) cm4f.astype('float32') cm4f.tofile(tmpF + tmp + '.data') createParams(cm4f, tmpF + tmp) print "pipeline STOCHASTIC, data shape end : %s" % str(shpD)
def loaddata_denoise(input_dir, ref_dir, signal_length=5000, sigmoid=5, fs=320): X = [] Y = [] QRS = [] datapaths = glob.glob(os.path.join(input_dir, '*.mat')) datapaths.sort() for data_path in datapaths: # print(data_path) # load data data_mat = sio.loadmat(data_path) data = data_mat['ecg'] data = np.array(data, dtype=np.float32) signal = data # plt.subplot(2,1,1) # plt.plot(signal[0:2000]) # normalize the data # data = scale(data).astype(np.float32) # normalize the data for i in range(signal.shape[1]): smoothed_signal = smooth.smooth(signal[:, i], window_len=int(fs * 0.886), window='flat') signal[:, i] = signal[:, i] - smoothed_signal # denoise ECG for i in range(signal.shape[1]): # DWT coeffs = pywt.wavedec(signal[:, i], 'db4', level=3) # compute threshold noiseSigma = 0.01 threshold = noiseSigma * math.sqrt( 2 * math.log2(signal[:, i].size)) # apply threshold newcoeffs = coeffs for j in range(len(newcoeffs)): newcoeffs[j] = pywt.threshold(coeffs[j], threshold, mode='soft') # IDWT signal[:, i] = pywt.waverec(newcoeffs, 'db4')[0:len(signal)] # resample signal_new = np.zeros((signal_length, signal.shape[1])) for i in range(signal.shape[1]): signal_new[:, i] = resample(signal[:, i], signal_length) signal = signal_new # scaler = StandardScaler() # scaler.fit(signal) # signal = scaler.transform(signal).astype(np.float32) # plt.subplot(2,1,2) # plt.plot(signal[0:2000]) # plt.show() X.append(signal) X = np.array(X, dtype=np.float32) print('X.shape', X.shape) refpaths = glob.glob(os.path.join(ref_dir, '*.mat')) refpaths.sort() for ref_path in refpaths: # load reference ref_mat = sio.loadmat(ref_path) R_peaks = ref_mat['R_peak'] R_peaks = np.array(R_peaks, dtype=np.int32).squeeze() R_peaks = (R_peaks * (signal_length / len(data))).astype(np.int32) ref = smooth_ref(signal_length, R_peaks, sigmoid=sigmoid) ref = np.expand_dims(ref, axis=-1) Y.append(ref) QRS.append(R_peaks) Y = np.array(Y, dtype=np.float32) print('Y.shape', Y.shape) # remove duplicate rows #X,indexes = np.unique(X, return_index=True, axis=0) #print(X.shape) # Y = Y[indexes] # QRS = itemgetter(*indexes)(QRS) # shuffle X, Y, QRS = shuffle(X, Y, QRS, random_state=0) return X, Y, QRS
def iter_smooth(array, loops, window_len): for l in range(loops): array = sm.smooth(array, window_len, 'flat') return array
def smooth_avg_consumption(self): self.consumption_per_interval_smoothed = smooth( np.array(self.consumption_per_interval), 11)
'/NN_aiSIRIS_ASSAY_MPP_SCIP_SGinf_LITH_ROCKGRP.csv') df = pd.read_csv(fpath, low_memory=False) values = np.log10(df.loc[df['holeid'].isin(df['holeid'].mode()), 'Scpt:0.001_SI']).values plt.plot(values) np.random.seed(1) L = 1e5 Nw = 784 Ns = Nw/2 time = np.arange(L) values = np.random.random(time.shape) spike = int(np.random.choice(time)) values[spike:spike+10000] += 5 values = smooth(values, int(Ns)) # values = electrocardiogram()[10000:20000] # plt.plot(data) X = make_nd_windows(values, Nw, steps=Ns) tensor_x = torch.Tensor(X) # transform to torch tensor syn_dataset = data.TensorDataset(tensor_x) # create your datset syn_dataloader = data.DataLoader(syn_dataset, batch_size=1) # create your dataloader
plt.figure() plt.plot(V, I) plt.figure() plt.plot(chR, chL) #%% Opamp file = 'inversor_div-11-3.txt' #circuito inverson con un divisor de tensión por 11/3 en la lectura completa = os.path.join('Measurements', file) datos = np.loadtxt(completa) datos = datos[15000:20000, :] sr = 42100 chR, chL = np.split(datos, 2, axis=1) chR = smooth(chR.flatten()) chL = smooth(chL.flatten()) chR /= max(chR) chL /= max(chL) fL = fft(chL) fR = fft(chR) N = len(fL) frecs = np.linspace(0.0, 1.0 / (2.0 / sr), N // 2) freqR = frecs[np.argmax(2.0 / N * np.abs(fR[0:N // 2]))] freqL = frecs[np.argmax(2.0 / N * np.abs(fL[0:N // 2]))]
def inception_phase_20_0(sm, num): mode = 0; fp = open("auto_phase_11.py", "w") #start composing MIDI fp.write("#Inception phase 11\n"); fp.write("import midi\n\n\n") fp.write("def midi_gen():\n") fp.write(" hh_midi = midi.Pattern(tracks=[[midi.TimeSignatureEvent(tick=0, data=[4, 2, 24, 8]),") midi_setup = [" midi.KeySignatureEvent(tick=0, data=[0, 0]),\n", " midi.EndOfTrackEvent(tick=1, data=[])],\n", " [midi.ControlChangeEvent(tick=0, channel=0, data=[91, 58]),\n", " midi.ControlChangeEvent(tick=0, channel=0, data=[10, 69]),\n", " midi.ControlChangeEvent(tick=0, channel=0, data=[0, 0]),\n", " midi.ControlChangeEvent(tick=0, channel=0, data=[32, 0]),\n", " midi.ProgramChangeEvent(tick=0, channel=0, data=[0]),\n\n"] fp.writelines(midi_setup) #automatic composing fp.write("#automatic composing\n") tick_0 = 6400 tick_1 = 6400 tick_2 = 9600 beat_0 = 100 beat_1 = 100 beat_2 = 200 onset_0 = 0.1 onset_1 = 0.1 onset_2 = 0.25 # mode == 0 : chord_1 = C3 chord_2 = G3 chord_3 = A2 chord_4 = E3 chord_5 = F3 chord_6 = C3 chord_7 = D3 chord_8 = G3 chord_8_c = C3 chord_1_b = F3 chord_2_b = G3 chord_3_b = C3 chord_4_b = C3 chord_5_b = D3 chord_6_b = D3 chord_7_b = E3 chord_8_b = E3 chorus_b_1 = F_major_bass chorus_b_2 = G_major_bass chorus_b_3 = C_major_bass chorus_b_4 = C_major_bass chorus_b_5 = D_minor_bass chorus_b_6 = D_minor_bass chorus_b_7 = E_major_bass chorus_b_8 = E_major_bass chorus_c_1 = C_major_bass chorus_c_2 = G_major_bass chorus_c_3 = A_minor_bass chorus_c_4 = E_minor_bass chorus_c_5 = F_major_bass chorus_c_6 = C_major_bass chorus_c_7 = D_minor_bass chorus_c_8 = G_major_bass chorus_c_8_c = C_major_bass scale_1 = C_major scale_2 = G_major scale_3 = A_minor scale_4 = E_minor scale_5 = F_major scale_6 = C_major scale_7 = D_minor scale_8 = G_major scale_8_c = C_major scale_1_b = F_major scale_2_b = G_major scale_3_b = C_major scale_4_b = C_major scale_5_b = D_minor scale_6_b = D_minor scale_7_b = E_major scale_8_b = E_major tone = 'major' tick_mode = tick_0 beat = beat_0 onset_mode = onset_0 tick = 0 note = 72; note_last = 72 sm_th = 9 # AB while (tick < 6400): tick_round = tick%6400 if tick_round < 800: chord = chord_1 scale = scale_1 elif 800 <= tick_round < 1600: chord = chord_2 scale = scale_2 elif 1600 <= tick_round < 2400: chord = chord_3 scale = scale_3 elif 2400 <= tick_round < 3200: chord = chord_4 scale = scale_4 elif 3200 <= tick_round < 4000: chord = chord_5 scale = scale_5 elif 4000 <= tick_round < 4800: chord = chord_6 scale = scale_6 elif 4800 <= tick_round < 5600: chord = chord_7 scale = scale_7 elif 5600 <= tick_round < 6400: chord = chord_8 scale = scale_8 fp.write(" midi.NoteOnEvent(tick={}, data = [{}, 80]),\n".format(100,chord)) note = pt.gen_next_note(scale[0],mode) note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, 80]),\n".format(0,note)) onset = rand.random() if onset > 0.3: v = 80 note = pt.gen_next_note(note,mode) else: v = 0 note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) note = pt.gen_next_note(note,mode) note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) onset = rand.random() if onset > 0.4: v = 80 note = pt.gen_next_note(note,mode) else: v = 0 note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) tick = tick + 400 note_last = 72 sm_th = 9 tick = 0 # bridge while (tick < 6400): tick_round = tick%6400 if tick_round < 800: chord = chord_1_b scale = scale_1_b elif 800 <= tick_round < 1600: chord = chord_2_b scale = scale_2_b elif 1600 <= tick_round < 2400: chord = chord_3_b scale = scale_3_b elif 2400 <= tick_round < 3200: chord = chord_4_b scale = scale_4_b elif 3200 <= tick_round < 4000: chord = chord_5_b scale = scale_5_b elif 4000 <= tick_round < 4800: chord = chord_6_b scale = scale_6_b elif 4800 <= tick_round < 5600: chord = chord_7_b scale = scale_7_b elif 5600 <= tick_round < 6400: chord = chord_8_b scale = scale_8_b fp.write(" midi.NoteOnEvent(tick={}, data = [{}, 80]),\n".format(100,chord)) note = pt.gen_next_note(scale[0],mode) note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, 80]),\n".format(0,note)) onset = rand.random() if onset > 0.2: v = 80 note = pt.gen_next_note(note,mode) else: v = 0 note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) note = pt.gen_next_note(note,mode) note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) onset = rand.random() if onset > 0.2: v = 80 note = pt.gen_next_note(note,mode) else: v = 0 note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) tick = tick + 800 note_last = 72 sm_th = 9 tick = 0 # C while (tick < 6400): tick_round = tick%6400 if tick_round < 800: chord = chord_1 scale = scale_1 elif 800 <= tick_round < 1600: chord = chord_2 scale = scale_2 elif 1600 <= tick_round < 2400: chord = chord_3 scale = scale_3 elif 2400 <= tick_round < 3200: chord = chord_4 scale = scale_4 elif 3200 <= tick_round < 4000: chord = chord_5 scale = scale_5 elif 4000 <= tick_round < 4800: chord = chord_6 scale = scale_6 elif 4800 <= tick_round < 5200: chord = chord_7 scale = scale_7 elif 5200 <= tick_round < 5600: chord = chord_8 scale = scale_8 elif 5600 <= tick_round < 6400: chord = chord_8_c scale = scale_8_c fp.write(" midi.NoteOnEvent(tick={}, data = [{}, 80]),\n".format(100,chord)) note = pt.gen_next_note(scale[0],mode) note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, 80]),\n".format(0,note)) onset = rand.random() if onset > 0.1: v = 80 note = pt.gen_next_note(note,mode) else: v = 0 note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) note = pt.gen_next_note(note,mode) note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) onset = rand.random() if onset > 0.2: v = 80 note = pt.gen_next_note(note,mode) else: v = 0 note = smooth.smooth(note, note_last, sm, sm_th) note_last = note fp.write(" midi.NoteOnEvent(tick={}, data = [{}, {}]),\n".format(100,note,v)) tick = tick + 400 #end automatic composing fp.write("#end automatic composing\n\n") fp.write(" midi.EndOfTrackEvent(tick=1, data=[])]])\n\n\n") fp.write(" midi.write_midifile(\"inception_phase_11_{}_{}.mid\", hh_midi)".format(tone,num)) fp.close()
#prompts user for image path and opens image in a seperate window. Closes import cv2 import numpy as np from smooth import smooth from rotate import rotate from grayscale import gray name = input('input name of image: ') do = input( 'What do you want to do to the image? Input R for rotate, G for grayscale or S for smooth:' ) img = cv2.imread(name) editimg = img.copy() C = (int(len(img[0]))) R = (int(len(img))) if do == 'R': editimg = rotate(R, C, img, editimg) if do == 'G': editimg = gray(R, C, img, editimg) if do == 'S': editimg = smooth(R, C, img, editimg) cv2.imshow('image', editimg) cv2.waitKey(0) cv2.destroyAllWindows()
def find_peaks(pntr_list, annot, ups=100, offset=100, smooth_param=50): """ ups = 100 # how long to include upstream of TSS offset = 100 # how much to discard downstream of TSS for genic TSS detection smooth_param = 50 # approximate width of the TSS initiation region pntr_list: List of genome_db pointers formatted as [[pnt_pos,pnt_neg,Label], [pnt2_pos,pnt2_neg,Label2],...] """ for pntr in pntr_list: df_peak = pd.DataFrame( { pntr[2] + '_Native_value': np.zeros((annot.shape[0])), pntr[2] + '_Native_position': np.zeros((annot.shape[0])), pntr[2] + '_Intragenic_value': np.zeros((annot.shape[0])), pntr[2] + '_Intragenic_position': np.zeros((annot.shape[0])), pntr[2] + '_Native_valueAS': np.zeros((annot.shape[0])), pntr[2] + '_Native_positionAS': np.zeros((annot.shape[0])), pntr[2] + '_Intragenic_valueAS': np.zeros((annot.shape[0])), pntr[2] + '_Intragenic_positionAS': np.zeros((annot.shape[0])) }, index=annot['name']) for ix in tq(range(annot.shape[0])): gene = annot['name'].iloc[ix] chname = annot['chr'].iloc[ix] strand = annot['strand'].iloc[ix] start = annot['start'].iloc[ix] end = annot['end'].iloc[ix] for pntr in pntr_list: if strand == '+': tsP = pntr[0].get_nparray(chname, start - ups, end) tsN = pntr[1].get_nparray(chname, start - ups, end) vec = sm.smooth(tsP, smooth_param)[(smooth_param - 1):-(smooth_param - 1)] vecAS = sm.smooth(tsN, smooth_param)[(smooth_param - 1):-(smooth_param - 1)] elif strand == '-': tsP = pntr[0].get_nparray(chname, start, end + ups) tsN = pntr[1].get_nparray(chname, start, end + ups) vec = np.flipud( sm.smooth(tsN, smooth_param)[(smooth_param - 1):-(smooth_param - 1)]) vecAS = np.flipud( sm.smooth(tsP, smooth_param)[(smooth_param - 1):-(smooth_param - 1)]) df_peak.loc[gene, pntr[2] + '_Native_value'] = np.max( vec[:(ups + offset)]) df_peak.loc[gene, pntr[2] + '_Native_position'] = np.argmax( vec[:(ups + offset)]) - ups df_peak.loc[gene, pntr[2] + '_Intragenic_value'] = np.max( vec[(ups + offset):]) df_peak.loc[gene, pntr[2] + '_Intragenic_position'] = np.argmax( vec[(ups + offset):]) + offset df_peak.loc[gene, pntr[2] + '_Native_valueAS'] = np.max( vecAS[:(ups + offset)]) df_peak.loc[gene, pntr[2] + '_Native_positionAS'] = np.argmax( vecAS[:(ups + offset)]) - ups df_peak.loc[gene, pntr[2] + '_Intragenic_valueAS'] = np.max( vecAS[(ups + offset):]) df_peak.loc[gene, pntr[2] + '_Intragenic_positionAS'] = np.argmax( vecAS[(ups + offset):]) + offset return df_peak