def test_numdifftools_no_bounds(fit_method): pytest.importorskip("numdifftools") np.random.seed(7) x = np.linspace(0, 100, num=50) noise = np.random.normal(scale=0.25, size=x.size) y = exponential(x, amplitude=5, decay=15) + noise mod = ExponentialModel() params = mod.guess(y, x=x) # do fit, here with leastsq model result = mod.fit(y, params, x=x, method='leastsq') result_ndt = mod.fit(y, params, x=x, method=fit_method) # assert that fit converged to the same result vals = [result.params[p].value for p in result.params.valuesdict()] vals_ndt = [ result_ndt.params[p].value for p in result_ndt.params.valuesdict() ] assert_allclose(vals_ndt, vals, rtol=0.1) assert_allclose(result_ndt.chisqr, result.chisqr, rtol=1e-5) # assert that parameter uncertaintes from leastsq and calculated from # the covariance matrix using numdifftools are very similar stderr = [result.params[p].stderr for p in result.params.valuesdict()] stderr_ndt = [ result_ndt.params[p].stderr for p in result_ndt.params.valuesdict() ] perr = np.array(stderr) / np.array(vals) perr_ndt = np.array(stderr_ndt) / np.array(vals_ndt) assert_almost_equal(perr_ndt, perr, decimal=3) # assert that parameter correlatations from leastsq and calculated from # the covariance matrix using numdifftools are very similar for par1 in result.var_names: cor = [ result.params[par1].correl[par2] for par2 in result.params[par1].correl.keys() ] cor_ndt = [ result_ndt.params[par1].correl[par2] for par2 in result_ndt.params[par1].correl.keys() ] assert_almost_equal(cor_ndt, cor, decimal=2)
def predictive_model(data: pd.DataFrame, interesting_rows, day_zero_n_patients: int = 20, days_in_future: int = 30, aggregated: bool = False): data = data[interesting_rows].iloc[:, :] from lmfit.models import StepModel, ExponentialModel fig = plt.figure(figsize=(10, 5)) for c in range(len(data.index)): if aggregated: values = data.values[c, 4:][data.iloc[c, 4:] > day_zero_n_patients] else: values = np.concatenate( ([0], np.diff( data.values[c, 4:][data.iloc[c, 4:] > day_zero_n_patients]))) n = values.shape[0] x = np.asarray(range(values.shape[0]), dtype='float64') y = np.asarray(values, dtype='float64') if len(x) == 0: continue label = "{}-{}".format(data.values[c, 0], data.values[c, 1]) plt.plot(x, y, label=label) if data.values[c, 1] in ["China", "US"]: continue try: model_step = StepModel() model_exp = ExponentialModel() params_step = model_step.guess(y, x=x) params_exp = model_exp.guess(y, x=x) result_step = model_step.fit(y, params_step, x=x) result_exp = model_exp.fit(y, params_exp, x=x) except Exception: continue x_pred = np.asarray(range(days_in_future)) plt.plot(x_pred, model_step.eval(result_step.params, x=x_pred), ':', label='fit-{}'.format(label)) plt.plot(x_pred, model_exp.eval(result_exp.params, x=x_pred), '.', label='fit-{}'.format(label)) # print(result.fit_report()) # result.plot_fit() plt.legend(prop={"size": 7}) plt.yscale('log') plt.xticks(rotation=45) plt.grid(which='both') now = datetime.now() dt_string = now.strftime("%d%m%Y-%H%M%S")
def exponential_fit(x, y, errors=True): from lmfit.models import ExponentialModel mod = ExponentialModel() par = mod.guess(data=y, x=x) out = mod.fit(y, params=par, x=x) a = out.params['amplitude'] d = out.params['decay'] return a, d
def exponential_fit(x, y, errors=True): from lmfit.models import ExponentialModel mod = ExponentialModel() par = mod.guess(data=y, x=x) out = mod.fit(y, params=par, x=x) a = out.params["amplitude"] d = out.params["decay"] return a, d
def test_numdifftools_no_bounds(fit_method): pytest.importorskip("numdifftools") np.random.seed(7) x = np.linspace(0, 100, num=50) noise = np.random.normal(scale=0.25, size=x.size) y = exponential(x, amplitude=5, decay=15) + noise mod = ExponentialModel() params = mod.guess(y, x=x) # do fit, here with leastsq model result = mod.fit(y, params, x=x, method='leastsq') result_ndt = mod.fit(y, params, x=x, method=fit_method) # assert that fit converged to the same result vals = [result.params[p].value for p in result.params.valuesdict()] vals_ndt = [result_ndt.params[p].value for p in result_ndt.params.valuesdict()] assert_allclose(vals_ndt, vals, rtol=0.1) assert_allclose(result_ndt.chisqr, result.chisqr, rtol=1e-5) # assert that parameter uncertaintes from leastsq and calculated from # the covariance matrix using numdifftools are very similar stderr = [result.params[p].stderr for p in result.params.valuesdict()] stderr_ndt = [result_ndt.params[p].stderr for p in result_ndt.params.valuesdict()] perr = np.array(stderr) / np.array(vals) perr_ndt = np.array(stderr_ndt) / np.array(vals_ndt) assert_almost_equal(perr_ndt, perr, decimal=3) # assert that parameter correlatations from leastsq and calculated from # the covariance matrix using numdifftools are very similar for par1 in result.var_names: cor = [result.params[par1].correl[par2] for par2 in result.params[par1].correl.keys()] cor_ndt = [result_ndt.params[par1].correl[par2] for par2 in result_ndt.params[par1].correl.keys()] assert_almost_equal(cor_ndt, cor, decimal=2)
def predict_detector_position(datatable=None, target=20500): dt = array(datatable) mod = ExponentialModel() prediction = [] for i in range(1, 5): signal = dt[:, i] pars = mod.guess(signal, x=dt[:, 0]) out = mod.fit(signal, pars, x=dt[:, 0]) #out.plot() #print(whisper(out.fit_report(min_correl=0))) amp = out.params['amplitude'] tau = out.params['decay'] prediction.append(ceil(tau * log(amp / target))) #print(go_msg(f'predictions are {prediction}')) return prediction
def fit_exponential(x, y): """ Fits a data set to an exponential with a constant offset, returns best-fit parameters: `amplitude`, `decay,` and `c` Args: x (numpy.ndarray): A one-dimensional real-valued numpy array, the independent variable y (numpy.ndarray): A one-dimensional real-valued numpy array, the dependent variable. Returns: type: lmfit.model.ModelResult """ em = ExponentialModel() exponential_fit_result = em.fit( y, x=x, ) return exponential_fit_result
def single_taufit(self, x, y, t0, t1): """ Perform single exponential fit to voltage traces Parameters ---------- x : numpy array time corresponding to data in y y : numpy array voltage trace (single trace) t0 : float start time for fit (ms) t1 : float end time for fit (ms) Returns ------- tuple of (best fit parameters as dict, x times, best fit y values) If fit fails, return is (None, None, None) """ (cx, cy) = pu.clipdata(y, x, t0, t1, minFlag = False) expmodel = ExponentialModel() #, prefix='', missing=None, name=None, **kwargs) expmodel.set_param_hint('decay', min=0.1, max=50.0) cye = np.mean(cy[-5:]) try: result = expmodel.fit(cy-cye, x=cx-t0, amplitude=0., decay=10.) if verbose: print(result.fit_report()) # print the result rbv = result.best_values fitr = {'a': 0, 'tau': 5., 'dc': 0.} fitr['a'] = rbv['amplitude'] fitr['tau'] = rbv['decay'] return fitr, cx, result.best_fit+cye except: return None, None, None
def produce_taufits(filepath,meth='iso',pulseperiod=None,snr_cut=None, verbose=True, plotparams=False, plotflux=False, savefigure=False): pulsar, nch, nbins,nsub, lm_rms, tsub = read_headerfull(filepath) if verbose == True: verboseTag = True else: verboseTag = False print0 = "Pulsar name: %s" %pulsar print1 = "Number of freq. channels: %d \nFreq channels will be labeled 0 - %d" %(nch, nch-1) print2 = "Number of bins: %d" %nbins print3 = "RMS: %.2f" %lm_rms print4 = "Tsub: %.2f sec" %tsub for k in range(5): print eval('print{0}'.format(k)) print"--------------------------------------------------------" if pulseperiod==None: ## Define time axis, and time/bins conversions print ("Using Tsub in header to convert bins to time. Assumption is that tsub corresponds to 1.0 phase, corresponding to nbins. This should be adapted for search data.") pulseperiod = tsub else: pulseperiod = pulseperiod #set to provided pulseperiod in seconds profilexaxis = np.linspace(0,pulseperiod,nbins) pbs = pulseperiod/nbins tbs = tsub/nbins """Initialise vector outputs""" obtainedtaus, lmfittausstds = [], [] """freqmsMHz will correctly associate scattering time values (tau) with frequency, taking into account frequency integration across a sub-band. Whereas freqcsMHz is the centre freq. to the subband""" freqmsMHz, freqcsMHz = [], [] noiselessmodels =[] results, datas, comp_SNRs, comp_rmss = [], [], [], [] redchis, paramset, paramset_std, correls = [], [], [], [] halfway = nbins/2. for i in range(nch): print"--------------------------------------------------------" print "Channel %d" %i """Read in (pdv) data""" data, freqc, freqm = read_data(filepath,i,nbins) freqmsMHz.append(freqm) freqcsMHz.append(freqc) # roll the data of lowest freq channel to middle of bins if i ==0: peakbin = np.argmax(data) shift = int(halfway -int(peakbin)) if verboseTag: print 'peak bin at lowest freq channel:%d' %peakbin else: peakbin = peakbin shift = int(halfway - int(peakbin)) data = np.roll(data,shift) if verboseTag: print "Rolling data by -%d bins" %shift comp_rms = find_rms(data,nbins) if meth is None: print "No fitting method was chosen. Will default to an isotropic fitting model. \n Use option -m with 'onedim' to change." result, noiselessmodel, besttau, taustd, bestparams, bestparams_std, redchi, corsig = tau_fitter(data,nbins,verbose=verboseTag) elif meth == 'iso': result, noiselessmodel, besttau, taustd, bestparams, bestparams_std, redchi, corsig = tau_fitter(data,nbins,verbose=verboseTag) elif meth == 'onedim': result, noiselessmodel, besttau, taustd, bestparams, bestparams_std, redchi, corsig = tau_1D_fitter(data,nbins) comp_SNR_model = find_peaksnr(noiselessmodel,comp_rms) if verboseTag: print 'Estimated SNR (from model peak and data rms): %.2f' % comp_SNR_model comp_SNR = find_peaksnr_smooth(data,comp_rms) print 'Estimated SNR (from data peak and rms): %.2f' % comp_SNR print 'Channel Tau (ms): %.2f \pm %.2f ms' %(besttau,taustd) obtainedtaus.append(besttau) lmfittausstds.append(taustd) noiselessmodels.append(noiselessmodel) results.append(result) datas.append(data) comp_SNRs.append(comp_SNR) #new: comp_rmss.append(comp_rms) redchis.append(redchi) paramset.append(bestparams) paramset_std.append(bestparams_std) # if plotflux == True: # correls.append(corsig) #if plotflux == True: # cor_sigA = np.zeros(len(correls)) # for i in range(len(correls)): # cor_sigA[i] = correls[i]['A'] paramset = np.transpose(paramset) paramset_std = np.transpose(paramset_std) """Next check if any of the subbands contain only zeros. This happens with high RFI excision in LOFAR bands""" zero_ch = [] for i in range(nch): all_zeros = not np.any(datas[i]) if all_zeros: zero_ch.append(i) print"--------------------------------------------------------" if zero_ch: print "\n" print "%d channels have all zeroes (channels(s):" %len(zero_ch), zero_ch, ") and will be removed." if verboseTag: print "All zero channels are assigned SNR of 0" if snr_cut: print "Using SNR cutoff of %.2f" %snr_cut comp_SNRs = np.nan_to_num(comp_SNRs) (ind_lowSNR,) = np.where(np.array(comp_SNRs) < snr_cut) print "SNR cutoff will exclude %d channels (channel(s): %s)" %(len(ind_lowSNR), ind_lowSNR) data_highsnr = np.delete(np.array(datas),ind_lowSNR,0) model_highsnr = np.delete(np.array(noiselessmodels),ind_lowSNR,0) taus_highsnr = np.delete(np.array(obtainedtaus),ind_lowSNR) lmfitstds_highsnr = np.delete(np.array(lmfittausstds),ind_lowSNR) freqMHz_highsnr = np.delete(np.array(freqmsMHz),ind_lowSNR) #New: comp_rmss_highsnr = np.delete(np.array(comp_rmss),ind_lowSNR) redchis_highsnr = np.delete(np.array(redchis),ind_lowSNR) #corsigA_highsnr = np.delete(np.array(cor_sigA),ind_lowSNR) paramset_highsnr = np.zeros([len(paramset),len(data_highsnr)]) paramsetstd_highsnr = np.zeros([len(paramset),len(data_highsnr)]) for i in range(len(paramset)): paramset_highsnr[i]= np.delete(paramset[i],ind_lowSNR) paramsetstd_highsnr[i]= np.delete(paramset_std[i],ind_lowSNR) elif (snr_cut == None) and (zero_ch != []): print "Used no SNR cutoff" """Rename array to be same as when cut-off is used""" """If no SNR cutoff is used, remove channels with all zeroes -- these will automatically be removed by any snr_cut > 0""" data_highsnr = np.delete(np.array(datas),zero_ch,0) model_highsnr = np.delete(np.array(noiselessmodels),zero_ch,0) taus_highsnr = np.delete(np.array(obtainedtaus),zero_ch) lmfitstds_highsnr = np.delete(np.array(lmfittausstds),zero_ch) freqMHz_highsnr = np.delete(np.array(freqmsMHz),zero_ch) # New: comp_rmss_highsnr = np.delete(np.array(comp_rmss),zero_ch) redchis_highsnr = np.delete(np.array(redchis),zero_ch) #corsigA_highsnr = np.delete(np.array(cor_sigA),zero_ch) paramset_highsnr = np.zeros([len(paramset),len(data_highsnr)]) paramsetstd_highsnr = np.zeros([len(paramset),len(data_highsnr)]) for i in range(len(paramset)): paramset_highsnr[i]= np.delete(paramset[i],zero_ch) paramsetstd_highsnr[i]= np.delete(paramset_std[i],zero_ch) else: print "Used no SNR cutoff and there are no empty channels" data_highsnr = np.array(datas) model_highsnr = np.array(noiselessmodels) taus_highsnr = np.array(obtainedtaus) lmfitstds_highsnr = np.array(lmfittausstds) freqMHz_highsnr = np.array(freqmsMHz) # New: comp_rmss_highsnr = np.array(comp_rmss) redchis_highsnr = np.array(redchis) paramset_highsnr = np.array(paramset) paramsetstd_highsnr = np.array(paramset_std) taussec_highsnr = taus_highsnr*pbs lmfitstdssec_highsnr = lmfitstds_highsnr*pbs number_of_plotted_channels = len(data_highsnr) npch = number_of_plotted_channels print "Will plot remaining %d/%d channels" %(npch, nch) """Plotting starts""" #plot onedim in blue dashed #else plot in red if meth == 'onedim': prof = 'b--' lcol='b' else: prof = 'r-' lcol ='r' """1. PLOT PROFILES""" dimx, dimy = 3., 3. numsubplots = dimx*dimy numplots = int(np.ceil(npch/numsubplots)) print "Num profile plots:", numplots """Compute residuals""" #"""Plot 1: Pulse profiles and fits""" if npch > 0: resdata = data_highsnr - model_highsnr resnormed = (resdata-resdata.mean())/resdata.std() if taussec_highsnr[0] > 1: taulabel = taussec_highsnr taulabelerr = lmfitstdssec_highsnr taustring = 'sec' else: taulabel = taussec_highsnr*1000 taulabelerr = lmfitstdssec_highsnr*1000 taustring = 'ms' for k in range(numplots): j = int(numsubplots*k) figg = plt.figure(k+1,figsize=(int(4*dimx),int(3*dimy))) plots_remaining = int(npch - numsubplots*k) #print "Plots remaining", plots_remaining for i in range(np.min([int(numsubplots),int(plots_remaining)])): figg.subplots_adjust(left = 0.08, right = 0.98, wspace=0.35,hspace=0.35,bottom=0.15) #plt.rc('text', usetex=True) plt.rc('font', family='serif') plt.subplot(dimx,dimy,i+1) plt.plot(profilexaxis,data_highsnr[j+i],alpha = 0.20) plt.plot(profilexaxis,model_highsnr[j+i],lw = 2.0, alpha = 0.85, label=r'$\tau: %.2f \pm%.2f$ %s' %(taulabel[j+i], taulabelerr[j+i], taustring)) plt.title('%s at %.1f MHz' %(pulsar, freqMHz_highsnr[j+i])) plt.ylim(ymax=1.3*np.max(data_highsnr[j+i])) plt.xlim(xmax=pulseperiod) plt.xticks(fontsize=11) plt.yticks(fontsize=11) plt.xlabel('time (s)',fontsize=11) plt.legend(fontsize=11,numpoints=1) plt.ylabel('normalized intensity',fontsize=11) plt.tight_layout() if savefigure == True: figname = '%s_%s_%s_%d.png' %(os.path.basename(filepath),'fitted_profiles', meth, k) plt.savefig(figname, dpi=200) print "Saved figure %s in ./" %figname if noshow == False: plt.show() if verboseTag: for i in range(npch): print "Channel %d" %i print'Tau (ms): %.2f' %(1000*taussec_highsnr[i]) tau1GHz = tauatfreq(freqMHz_highsnr[i]/1000.,taussec_highsnr[i],1.0,4) print 'tau1GHz_alpha_4 (ms) ~ %.4f' %(tau1GHz*1000) lmfitstdssec_highsnr = lmfitstdssec_highsnr[np.nonzero(lmfitstdssec_highsnr)] taussec_highsnr = taussec_highsnr[np.nonzero(lmfitstdssec_highsnr)] freqMHz_highsnr = freqMHz_highsnr[np.nonzero(lmfitstdssec_highsnr)] """Plot 2: Plot Gaussian fitting parameters and DM if selected""" if plotparams==True: print "\nPlotting Gaussian fit parameters w.r.t frequency\n" """Set plotting parameters""" alfval = 0.6 markr= '*' msize=12 plt.figure(numplots+1, figsize=(12,8)) plt.subplots_adjust(left = 0.055, right=0.98,wspace=0.35,hspace=0.4,bottom=0.08) """Fit models to sigma""" powmod = PowerLawModel() powpars = powmod.guess(paramset_highsnr[0], x=freqMHz_highsnr) powout = powmod.fit(paramset_highsnr[0], powpars, x=freqMHz_highsnr, weights=1/((paramsetstd_highsnr[0])**2)) linmod = LinearModel() if len(freqMHz_highsnr) < 3: raise RuntimeError("plotparams == True: Less than three frequency channels. Cannot compute quadratic or exponential fit for width evolution. Consider lowering snr_cut.") else: quadmod = QuadraticModel() quadpars = quadmod.guess(paramset_highsnr[0], x=freqMHz_highsnr) quadout = quadmod.fit(paramset_highsnr[0], quadpars, x=freqMHz_highsnr, weights=1/((paramsetstd_highsnr[0])**2)) expmod = ExponentialModel() exppars = expmod.guess(paramset_highsnr[0], x=freqMHz_highsnr) expout = expmod.fit(paramset_highsnr[0], exppars, x=freqMHz_highsnr, weights=1/((paramsetstd_highsnr[0])**2)) """Fit a DM model to delta mu""" delnuarray = [-(1/freqMHz_highsnr[-1]**2-1/freqMHz_highsnr[i]**2) for i in range(npch)] ##in MHz delmuarray = [(paramset_highsnr[1][-1] - paramset_highsnr[1][i])*pbs for i in range(npch)] ##in seconds delmu_stdarray = [(paramsetstd_highsnr[1][-1] - paramsetstd_highsnr[1][i])*pbs for i in range(npch)] DM_linpars = linmod.guess(delmuarray, x=delnuarray) DM_linout = linmod.fit(delmuarray, DM_linpars, x=delnuarray) DM_CCval = DM_linout.best_values['slope'] DM_CCvalstd = DM_linout.params['slope'].stderr DMmodelfit = DM_linout.best_fit ##model gives deltime in seconds (used to shift data) DMconstant = 4148.808 #uncertainty in the constant is 0.003 - only affects the Delta DM value in the 9th decimal DMval = (DM_CCval/DMconstant) DMvalstd = (DM_CCvalstd/DMconstant) #DMcheck = psr.DM_checker(freqmsMHz,bestpT_highSNR[1]*pbs) ## Plot reduced chi square: plt.subplot(2,3,1) plt.plot(freqMHz_highsnr, redchis_highsnr/np.power(comp_rmss_highsnr,2), markr,alpha=alfval,markersize = msize) plt.title(r'Reduced $\chi^2$ values', fontsize=12) plt.yticks(fontsize=12) plt.xticks(fontsize=12) plt.xlabel(r'$\nu$ MHz',fontsize=12) plt.ylabel(r'$\chi^2$',fontsize=12) ## Plot sigma: plt.subplot(2,3,2) #plt.errorbar(freqMHz_highsnr,paramset_highsnr[0]*pbs) plt.errorbar(freqMHz_highsnr,paramset_highsnr[0]*pbs,yerr =paramsetstd_highsnr[0]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval) plt.plot(freqMHz_highsnr,powout.best_fit*pbs,'-', alpha=alfval,label='pow = %.2f' %powout.best_values['exponent']) plt.plot(freqMHz_highsnr,quadout.best_fit*pbs,'-',alpha=alfval, label='quad: a,b = %.3f,%.3f' %(quadout.best_values['a'],quadout.best_values['b'])) plt.ylabel(r'$\sigma$ (sec)') plt.title(r'Width evolution', fontsize=12) plt.yticks(fontsize=12) plt.xticks(fontsize=12) plt.xlabel(r'$\nu$ MHz',fontsize=14) plt.legend(fontsize = 10, loc='best') ## Plot mean: plt.subplot(2,3,3) #plt.errorbar(freqMHz_highsnr,paramset_highsnr[1]*pbs) plt.errorbar(freqMHz_highsnr,paramset_highsnr[1]*pbs,yerr =paramsetstd_highsnr[1]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval) plt.ylabel(r'$\mu$ (sec)') plt.title(r'Centroid evolution', fontsize=12) plt.yticks(fontsize=12) plt.xticks(fontsize=12) plt.xlabel(r'$\nu$ MHz',fontsize=14) #plt.legend(fontsize = 9, loc='best') ## Plot amplitude: plt.subplot(2,3,4) #plt.errorbar(freqMHz_highsnr,paramset_highsnr[2]*pbs) plt.errorbar(freqMHz_highsnr,paramset_highsnr[2]*pbs,yerr =paramsetstd_highsnr[2]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval) plt.ylabel(r'$\mu$ (sec)') plt.title(r'Amplitude evolution', fontsize=12) plt.yticks(fontsize=12) plt.xticks(fontsize=12) plt.xlabel(r'$\nu$ MHz',fontsize=14) #plt.legend(fontsize = 9, loc='best') ## Plot DC: plt.subplot(2,3,5) #plt.errorbar(freqMHz_highsnr,paramset_highsnr[2]*pbs) plt.errorbar(freqMHz_highsnr,paramset_highsnr[3]*pbs,yerr =paramsetstd_highsnr[3]*pbs, fmt = markr,markersize=msize,capthick=2,linewidth=1.5,alpha=alfval) plt.ylabel(r'$\mu$ (sec)') plt.title(r'DC offset', fontsize=12) plt.yticks(fontsize=12) plt.xticks(fontsize=12) plt.xlabel(r'$\nu$ MHz',fontsize=14) #plt.legend(fontsize = 9, loc='best') ## Plot DM: plt.subplot(2,3,6) plt.errorbar(delmuarray,freqMHz_highsnr, fmt = markr, xerr=delmu_stdarray, alpha = alfval, markersize=msize) plt.plot(DMmodelfit,freqMHz_highsnr, '-', label=r'DM: $%.3f \pm %.3f$ $\rm{pc.cm}^{-3}$' %(DMval,DMvalstd), alpha = alfval) plt.xlabel(r'$\Delta \mu$ (sec)', fontsize =12) plt.yticks(fontsize=12) plt.xticks(fontsize=12) plt.title('Delta DM', fontsize=12) plt.ylabel(r'$\nu$ (MHz)',fontsize=14) plt.ticklabel_format(style='sci', axis='x',scilimits=(0,0)) plt.legend(fontsize = 10, loc='best') plt.tight_layout() if savefigure == True: figname2 = '%s_%s.png' %(os.path.basename(filepath),'fitting_parameters') plt.savefig(figname2, dpi=200) print "Saved figure %s in ./" %figname2 if noshow == False: plt.show() if plotflux == True: ##Flux section needs debugging ls = 'solid' """Plot flux, and corrected flux spectrum""" """Create unscattered profiles, i.e. Guassians""" bins, profiles = [],[makeprofile(nbins = nbins, ncomps = 1, amps = paramset_highsnr[2][j], means = paramset_highsnr[1][j], sigmas = paramset_highsnr[0][j]) for j in range(npch)] unscatflux = [] for i in range(npch): unscatfl = np.sum(profiles[j])/nbins unscatflux.append(unscatfl) #smootheddata = smooth(data_highsnr[j],int(0.05*nbins)) scatflux = [find_modelflux(model_highsnr[i],nbins) for i in range(npch)] climbvals = [] for i in range(npch): climb = returnclimb(np.linspace(1,nbins,nbins),paramset_highsnr[1][i],paramset_highsnr[0][i],paramset_highsnr[2][i],taussec_highsnr[i],paramset_highsnr[3][i],nbins) climbvals.append(climb) correctedflux = np.array([scatflux[i] + climbvals[i] for i in range(npch)]) print scatflux print climbvals #per bin meancorflux = np.mean(correctedflux) meancorfluxerr = np.sqrt(np.sum(correctedflux**2))/len(correctedflux) """Calculate error in Flux""" sigmaWIDTH = paramsetstd_highsnr[0]*pbs #in seconds sigmaAMP = paramsetstd_highsnr[2] #in mJy WIDTHS =paramset_highsnr[0]*pbs #in seconds AMPS =paramset_highsnr[2] #in mJy Expr1 = np.sqrt(2*np.pi)*AMPS Expr2 = np.sqrt(WIDTHS) AreaExpression = Expr1*Expr2 sigmaEx1 = np.sqrt(2*np.pi)*sigmaAMP sigmaEx2 = Expr2*0.5*sigmaWIDTH/WIDTHS sigmaFlux =AreaExpression*np.sqrt(np.power(sigmaEx1/Expr1,2)+ np.power(sigmaEx2/Expr2,2))#+ 2*corsigA_highsnr*sigmaEx1*sigmaEx2/(Expr1*Expr2) plt.figure(figsize=(10,6)) plt.plot(freqMHz_highsnr, correctedflux,'k-', linewidth=2.0) plt.plot(freqMHz_highsnr, scatflux,'r--', linewidth=2.0) plt.fill_between(freqMHz_highsnr,scatflux,correctedflux, alpha=alfval,facecolor='r') eb = plt.errorbar(freqMHz_highsnr,correctedflux,yerr=sigmaFlux, fmt=markr,markersize=10.0, alpha=1.0,capthick=2,linewidth=1.5) eb[-1][0].set_linestyle(ls) #plt.errorbar(freqMHz_highsnr, unscatflux,yerr=sigmaFlux, fmt=markr,markersize=10.0, alpha=alfval) plt.title('Flux Spectrum', fontsize=12) plt.xticks(fontsize=12) plt.yticks(fontsize=12) plt.xlabel(r'$\nu$ (MHz)',fontsize=12) plt.ylabel(r'Calibrated flux (mJy)',fontsize=12) return freqMHz_highsnr, taussec_highsnr, lmfitstdssec_highsnr
class gene_set(): def __init__(self,gene_id, cluster): self.cluster= cluster self.gene_id = gene_id self.norm_vals = [float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0]] #TPM. self.get_model() #self.half_life() #self.printing() #self.saving() def get_model(self): self.x = np.array([0,1,2,6,12,24]) self.y = np.array(self.norm_vals) self.model_flag = None self.model_list = [] # First model with Gaussian curve. self.background1 = ExponentialModel(prefix='e1_') self.pars1 = self.background1.guess(self.y, x=self.x) self.peak1 = GaussianModel(prefix='p1_') self.pars1 += self.peak1.guess(self.y, x=self.x) self.comp_mod1 = self.peak1 + self.background1 self.init1 = self.comp_mod1.eval(self.pars1, x=self.x) self.comp_out1 = self.comp_mod1.fit(self.y, x=self.x, fit_kws={'nan_policy': 'omit'}) self.comp_list1 = self.comp_out1.fit_report().split('\n') self.comp_chisq1 = float(self.comp_list1[6][-5:]) # Second model with Voigt curve. self.background2 = ExponentialModel(prefix='e2_') self.pars2 = self.background2.guess(self.y, x=self.x) self.peak2 = VoigtModel(prefix='p2_') self.pars2 += self.peak2.guess(self.y, x=self.x) self.comp_mod2 = self.peak2 + self.background2 self.init2 = self.comp_mod2.eval(self.pars2, x=self.x) self.comp_out2 = self.comp_mod2.fit(self.y, x=self.x, fit_kws={'nan_policy': 'omit'}) self.comp_list2 = self.comp_out2.fit_report().split('\n') self.comp_chisq2 = float(self.comp_list2[6][-5:]) # Exponential model for reference self.exp_mod = ExponentialModel(prefix='onlye_') self.pars = self.exp_mod.guess(self.y, x=self.x) self.init = self.exp_mod.eval(self.pars, x=self.x) self.exp_out = self.exp_mod.fit(self.y, x=self.x, missing='drop') self.exp_list = self.exp_out.fit_report().split('\n') self.exp_chisq = float(self.exp_list[6][-5:]) self.model_list = [self.comp_chisq1, self.comp_chisq2, self.exp_chisq] if np.count_nonzero(np.isinf(self.comp_out1.best_fit)) == 5 and np.count_nonzero(np.isinf(self.comp_out2.best_fit)): model_flag = "exponential" self.out = self.exp_out elif len(self.model_list) == len(set(self.model_list)): if min(self.model_list) == self.comp_chisq1: self.model_flag = "Gaussian compound" self.out = self.comp_out1 elif min(self.model_list) == self.comp_chisq2: self.model_flag = "Voigt compound" self.out = self.comp_out2 elif min(self.model_list) == self.exp_chisq: self.model_flag = "exponential" self.out = self.exp_out elif len(self.model_list) != len(set(self.model_list)): if min(self.model_list) == self.comp_chisq1: self.model_flag = "Gaussian compound" self.out = self.comp_out1 elif min(self.model_list) == self.comp_chisq2: self.model_flag = "Voigt compound" self.out = self.comp_out2 elif min(self.model_list) == self.exp_chisq: self.model_flag = "exponential" self.out = self.exp_out if min(self.model_list) == self.comp_chisq1 and self.comp_chisq1 == self.comp_chisq2: self.model_flag = "Both compounds" self.out = self.comp_out2 if min(self.model_list) == self.comp_chisq2 and self.comp_chisq2 == self.exp_chisq: self.model_flag = "Voigt compound and exponential" self.out = self.comp_out2 if min(self.model_list) == self.exp_chisq and self.exp_chisq == self.comp_chisq1: self.model_flag = "Gaussian compound and exponential" self.out = self.comp_out1 return self.comp_out1, self.comp_chisq1, self.comp_out2, self.comp_chisq2, self.exp_out, self.exp_chisq, self.model_flag
class gene_set(): def __init__(self, gene_id, cluster): self.cluster = cluster self.gene_id = gene_id self.norm_vals = [ float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0] ] #TPM. self.get_model() self.half_life() self.printing() self.saving() def get_model(self): self.x = np.array([0, 1, 2, 6, 12, 24]) self.y = np.array([self.norm_vals]) self.peak = GaussianModel(prefix='g_') self.background = ExponentialModel(prefix='e_') self.comp_mod = self.peak + self.background #guess?pars? #pars = exp_mod.guess(y, x=x) #pars = gauss_mod.guess(y, x=x) #pars += step_mod.guess() #pars.update(gauss1.make_params()) #init = mod.eval(pars, x=x) self.comp_out = self.comp_mod.fit(self.y, x=self.x, fit_kws={'nan_policy': 'omit'}) self.comp_list = self.comp_out.fit_report().split('\n') self.comp_chisq = float(self.comp_list[6][-5:]) self.exp_mod = ExponentialModel(prefix='onlye_') self.exp_out = self.exp_mod.fit(self.y, x=self.x, missing='drop') self.exp_list = self.exp_out.fit_report().split('\n') self.exp_chisq = float(self.exp_list[6][-5:]) if np.count_nonzero(np.isinf(self.comp_out.best_fit)) == 5: self.out = self.exp_out elif self.comp_chisq < self.exp_chisq: self.out = self.comp_out elif self.comp_chisq == self.exp_chisq: self.out = self.comp_out elif self.comp_chisq > self.exp_chisq: self.out = self.exp_out return self.comp_out, self.comp_chisq, self.exp_out, self.exp_chisq, self.out def half_life(self): self.new_x = np.array([0]) self.hl_eval = np.array([0]) self.hl_array = np.array([0]) self.hl_coord = np.array([0]) self.bestfit = self.out.best_fit self.idx = np.argmin(np.abs(self.bestfit - 0.5)) if self.idx == 5 and self.bestfit[self.idx - 1] < 0.5: self.bestfit = self.out.best_fit[:-1] self.idx = np.argmin(np.abs(self.bestfit - 0.5)) if self.bestfit[self.idx] == 0.5: self.half_life_y = self.bestfit[self.idx] self.half_life_x = self.idx # New elif 0.5 < self.bestfit[self.idx] and self.bestfit[ self.idx] == self.bestfit[5]: self.min = 0 self.max = 0 elif 0.5 > self.bestfit[self.idx] and self.bestfit[self.idx - 1] > 0.5: self.max = self.x[self.idx] self.min = self.x[self.idx - 1] self.flag = 'first' elif 0.5 < self.bestfit[self.idx] and self.bestfit[self.idx + 1] < 0.5: self.min = self.x[self.idx] self.max = self.x[self.idx + 1] self.flag = 'second' #New elif 0.5 < self.bestfit[self.idx] and self.bestfit[ self.idx + 1] > 0.5 and self.bestfit[ self.idx - 1] > 0.5 and self.bestfit[self.idx - 3] < 0.5: self.min = self.bestfit[self.idx - 2] self.max = self.bestfit[self.idx - 3] elif 0.5 < self.bestfit[self.idx] and self.bestfit[ self.idx + 1] > 0.5 and self.bestfit[self.idx + 2] < 0.5: self.min = self.x[self.idx + 1] self.max = self.x[self.idx + 2] #New elif 0.5 > self.bestfit[self.idx] and self.bestfit[ self.idx - 1] < 0.5 and self.bestfit[self.idx - 2] > 0.5: self.min = self.x[self.idx - 2] self.max = self.x[self.idx - 1] elif 0.5 > self.bestfit[self.idx] and self.bestfit[ self.idx + 1] < 0.5 and self.bestfit[self.idx - 2] > 0.5: self.min = self.x[self.idx - 2] self.max = self.x[self.idx] if self.min == 0 and self.max == 0: self.half_life_y = 0 self.half_life_x = 0 else: self.ranging = np.arange(self.min, self.max, 0.001) for j in np.nditer(self.ranging): self.new_x = np.array([j]) self.hl_eval = self.out.eval(x=self.new_x) if self.hl_eval >= 0.50 and self.hl_eval <= 0.51: self.hl_array = np.append(self.hl_array, self.hl_eval) self.hl_coord = np.append(self.hl_coord, self.new_x) self.half_life_id = np.argmin(np.abs(self.hl_array - 0.5)) self.half_life_y = self.hl_array[self.half_life_id] self.half_life_x = self.hl_coord[self.half_life_id] self.bestfit = self.out.best_fit return self.half_life_y, self.half_life_x def printing(self): if self.comp_chisq < self.exp_chisq: print(self.gene_id, self.cluster, "composite", self.comp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5]) elif self.comp_chisq == self.exp_chisq: print(self.gene_id, self.cluster, "equal", self.comp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5]) elif self.comp_chisq > self.exp_chisq: print(self.gene_id, self.cluster, "exponential", self.exp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5]) def saving(self): with open('model_fit_c3_average_filtering_compound.txt', 'a') as f: if self.comp_chisq < self.exp_chisq: (f.write( "%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n" % (self.gene_id, self.cluster, "composite", self.comp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5]))) elif self.comp_chisq == self.exp_chisq: (f.write( "%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n" % (self.gene_id, self.cluster, "equal", self.comp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5]))) elif self.comp_chisq > self.exp_chisq: (f.write( "%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n" % (self.gene_id, self.cluster, "exponential", self.exp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5])))
amps.append(np.array(amp)) amps_std.append(np.array(amp_err)) print(o_i.fit_report(min_correl=0.5)) amps = np.vstack(amps) amps_std = np.vstack(amps_std) #print(out.fit_report(min_correl=0.5)) ##ax.plot_wireframe(X, Y, Z_noise.reshape(50,50),color="r") #ax.plot_wireframe(X, Y, Z_conv.reshape(50,50),color="r") ax.plot_wireframe(X, Y, test_data_noise.reshape(50,50),color="r") colors = ["r","g","b","orange","k","c"] #for i,c in zip(exp_test_data,colors): # ax.plot_wireframe(X, Y, i.reshape(50,50),color=c) ax.plot_wireframe(X,Y,out.best_fit.reshape(50,50),color="b") ##ax.contour(X,Y,Z) ax.set_ylabel("y") ax.set_xlabel("x") plt.show() t = np.arange(6) for i in range(len(exp_test_data)): mod = ExponentialModel() pars = mod.guess(amps[:,i],x=t) fit = mod.fit(amps[:,i],pars,x=t) plt.errorbar(t,amps[:,i],yerr=amps_std[:,i],fmt="ro") plt.plot(t,fit.best_fit,"k--") plt.show()
x = langevin.time_series(A=A,D=D,delta_t=delta_t,N=N) # see http://docs.scipy.org/doc/scipy-0.15.1/reference/generated/scipy.signal.fftconvolve.html # autocorr = signal.fftconvolve(x, x[::-1], mode='full') f = np.fft.rfft(x) acf = np.fft.irfft(f * np.conjugate(f)) acf = np.fft.fftshift(acf) / N autocorr=acf[int(N/2):] # n=len(autocorr) # autocorr=autocorr[int((n-1)/2):]*2.0/(n+1) acf_avg=acf_avg+autocorr acf_var=acf_var+autocorr**2 y = autocorr[:min(int(N/2),1000)] t = np.arange(min(int(N/2),1000)) out = mod.fit(y, amplitude=1.0, decay=100.0, x=t) #print(out.fit_report(min_correl=0.25)) t_list.append(out.values['decay']) tstd_list.append(out.covar[0,0]) A_list.append(out.values['amplitude']) Astd_list.append(out.covar[1,1]) mean_list.append(x.mean()) std_list.append(x.std()) print('mean: ',x.mean(),'std: ',x.std(),'amplitude: ',out.values['amplitude'],'decay: ',out.values['decay']) acf_avg=acf_avg/M acf_stderr=np.sqrt((acf_var/M-(acf_avg/M)**2)/M) y = acf_avg dy=acf_stderr t = np.arange(int(N/2))
data_results = [data[0]**2, data[-1]**2, np.sum(data[1:-2]**2), np.sum(data[:-1]*data[1:])] # calculate autocorrelation function f = np.fft.rfft(data) acf = np.fft.irfft(f * np.conjugate(f)) acf = np.fft.fftshift(acf) / N autocorr = acf[int(N / 2):] y = autocorr[:min(int(N / 2), P)] t = np.arange(min(int(N / 2), P)) mod = ExponentialModel() pars = mod.guess(y, x=t) try: out = mod.fit(y, pars, x=t) except: fit_results = np.zeros(4) print('fit did not work') else: fit_results = np.array([out.values['decay']*delta_t, np.sqrt(out.covar[0,0])*delta_t, out.values['amplitude'], np.sqrt(out.covar[1,1])]) print(out.fit_report(min_correl=0.25)) trace = sm.run(x=data, aB=alpha_B, bB=beta_B, aA=alpha_A, bA=beta_A,
def calculate_feature2(image_path, value=[1,0,0,0,1,0], return_image=False): debug_image = [] image = cv2.imread(image_path) from detector_borde_plastico import detector_borde_plastico image_bp, images_dp_debug = detector_borde_plastico(image) debug_image.extend(images_dp_debug) image_bottom = image_bp[int(0.7*image_bp.shape[0]):image_bp.shape[0],:,:] #image_sharr = cv2.Scharr(image_bottom, cv2.CV_32F, 2, 0) image_bottom_lab = cv2.cvtColor(image_bottom, cv2.COLOR_BGR2LAB) #features = image_bottom_lab.flatten()[:500] #debug_image.append(image_bottom_lab[:,:,0]) #debug_image.append(image_bottom_lab[:,:,2]) image_lab_a = image_bottom_lab[:, :, 1] image_lab_b = image_bottom_lab[:, :, 2] debug_image.append(image_lab_a) debug_image.append(image_lab_b) thresh = cv2.threshold(image_lab_a, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1] #debug_image.append(thresh) _, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # bigest_area = amg max_area = 0 for cnt in contours: area = cv2.contourArea(cnt) if max_area < area: max_area = area biggest_area = cnt """ # points = [(100,h1[1][1]-h1[0][1]-5), (200, h1[1][1]-h1[0][1]-5), (300, h1[1][1]-h1[0][1]-5)] bad = 0 good = 0 for x in xrange(0, th2.shape[1], 10): p = (x, th2.shape[0] - 5) if cv2.pointPolygonTest(biggest_area, p, False) <= 0: bad += 1 else: good += 1 # print "good:", good, "bad:", bad isgood = False if bad < 25: isgood = True """ mask_pasto = np.zeros(thresh.shape, np.uint8) cv2.drawContours(mask_pasto, [biggest_area], 0, 255, 3) debug_image.append(mask_pasto) #features = [mask_pasto == 1] """ image_kmeans = kmeans(image_bottom_lab, K=40) image_kmeans_rgb = cv2.cvtColor(image_kmeans, cv2.COLOR_LAB2BGR) debug_image.append(image_kmeans_rgb) image_kmeans_rgb_cut = image_kmeans_rgb[0:image_kmeans_rgb.shape[0], int(image_kmeans_rgb.shape[1]*0.167):int(image_kmeans_rgb.shape[1]*0.905)] debug_image.append(image_kmeans_rgb_cut) image_kmeans2 = kmeans(image_kmeans_rgb_cut, K=5) #image_kmeans2_rgb = cv2.cvtColor(image_kmeans2, cv2.COLOR_LAB2BGR) debug_image.append(image_kmeans2) lowerBound = np.array(value[0:3])*255.0 upperBound = np.array(value[3:6])*255.0 # this gives you the mask for those in the ranges you specified, # but you want the inverse, so we'll add bitwise_not... image_inrange = cv2.inRange(image_kmeans2, lowerBound, upperBound) image_inrange_color = cv2.bitwise_not(image_kmeans_rgb_cut, mask=image_inrange) debug_image.append(image_inrange_color) """ """ cv2.kmeans(image_bottom_lab, 40, ) from sklearn.cluster import MiniBatchKMeans image_bottom_lab_reshape = image_bottom_lab.reshape((image_bottom_lab.shape[0] * image_bottom_lab.shape[1], 3)) clt = MiniBatchKMeans(n_clusters=10) labels = clt.fit_predict(image_bottom_lab_reshape) quant = clt.cluster_centers_.astype("uint8")[labels] quant = quant.reshape(image_bottom_lab.shape) image_bottom_lab_res = image_bottom_lab_reshape.reshape(image_bottom_lab.shape) # convert from L*a*b* to RGB quant = cv2.cvtColor(quant, cv2.COLOR_LAB2BGR) image_bottom_lab_res = cv2.cvtColor(image_bottom_lab_res, cv2.COLOR_LAB2BGR) sobelx64f = cv2.Sobel(image_bottom, cv2.CV_64F, 1, 0, ksize=3) abs_sobel64f = np.absolute(sobelx64f) sobel_8u = np.uint8(abs_sobel64f) lowerBound = np.array(value[0:3])*255.0 upperBound = np.array(value[3:6])*255.0 # this gives you the mask for those in the ranges you specified, # but you want the inverse, so we'll add bitwise_not... image_inrange = cv2.inRange(sobel_8u, lowerBound, upperBound) image_inrange_color = cv2.bitwise_not(image_bottom, mask=image_inrange) features = image.flatten()[:500] debug_image.append(image_bottom) debug_image.append(sobel_8u) debug_image.append(image_inrange) debug_image.append(image_inrange_color) debug_image.append(quant) debug_image.append(image_bottom_lab_res) """ x = [] y = [] for x_ in xrange(mask_pasto.shape[1]): for y_ in xrange(10, mask_pasto.shape[0]): if mask_pasto[y_,x_] == 255: x.append(x_) y.append(y_) break x = np.array(x) y = np.array(y) x1 = x[x<(mask_pasto.shape[1]/2)] y1 = y[x<(mask_pasto.shape[1]/2)] x1 = x1[y1>(mask_pasto.shape[0]/2)] y1 = y1[y1>(mask_pasto.shape[0]/2)] y1 = y1.max() - y1 x = x-x[0] # x offset mask_pasto2 = np.zeros(thresh.shape, np.uint8) for i in xrange(len(x1)): mask_pasto2[y1[i], x1[i]] = 255 debug_image.append(mask_pasto2) x2 = x[x>(mask_pasto.shape[1]/2)] y2 = y[x>(mask_pasto.shape[1]/2)] y2 = y2 - y2.max() from lmfit.models import ExponentialModel exp_mod = ExponentialModel(prefix='exp_') pars = exp_mod.guess(y1, x=x1) out = exp_mod.fit(y, pars, x=x) print "error en amplitud:", out.params['exp_amplitude'].stderr, print "error en decay:", out.params['exp_decay'].stderr, features = np.array([out.best_values['exp_decay'], out.params['exp_decay'].stderr]) #print(out.fit_report(min_correl=0.5)) #print pars if return_image: return features, debug_image return features
amps.append(np.array(amp)) amps_std.append(np.array(amp_err)) print(o_i.fit_report(min_correl=0.5)) amps = np.vstack(amps) amps_std = np.vstack(amps_std) #print(out.fit_report(min_correl=0.5)) ##ax.plot_wireframe(X, Y, Z_noise.reshape(50,50),color="r") #ax.plot_wireframe(X, Y, Z_conv.reshape(50,50),color="r") ax.plot_wireframe(X, Y, test_data_noise.reshape(50, 50), color="r") colors = ["r", "g", "b", "orange", "k", "c"] #for i,c in zip(exp_test_data,colors): # ax.plot_wireframe(X, Y, i.reshape(50,50),color=c) ax.plot_wireframe(X, Y, out.best_fit.reshape(50, 50), color="b") ##ax.contour(X,Y,Z) ax.set_ylabel("y") ax.set_xlabel("x") plt.show() t = np.arange(6) for i in range(len(exp_test_data)): mod = ExponentialModel() pars = mod.guess(amps[:, i], x=t) fit = mod.fit(amps[:, i], pars, x=t) plt.errorbar(t, amps[:, i], yerr=amps_std[:, i], fmt="ro") plt.plot(t, fit.best_fit, "k--") plt.show()
model += GaussianModel(prefix='g1_') model.set_param_hint('g1_center', value=105, min=75, max=125) model.set_param_hint('g1_sigma', value=15, min=3) model.set_param_hint('g1_amplitude', value=2000, min=10) model += GaussianModel(prefix='g2_') model.set_param_hint('g2_center', value=155, min=125, max=175) model.set_param_hint('g2_delta_sigma', value=1.5, min=0.8) model.set_param_hint('g2_sigma', expr='g2_delta_sigma*g1_sigma') model.set_param_hint('g2_amplitude', value=2000, min=10) pars = model.make_params() init = model.eval(pars, x=x) out = model.fit(y, pars, x=x) print(out.fit_report(min_correl=0.5)) model.plo fig, axes = plt.subplots(1, 2, figsize=(12.8, 4.8)) axes[0].plot(x, y, 'b') axes[0].plot(x, init, 'k--', label='initial fit') axes[0].plot(x, out.best_fit, 'r-', label='best fit') axes[0].legend(loc='best') comps = out.eval_components(x=x) axes[1].plot(x, y, 'b') axes[1].plot(x, comps['g1_'], 'g--', label='Gaussian component 1') axes[1].plot(x, comps['g2_'], 'm--', label='Gaussian component 2')
class gene_set(): def __init__(self, gene_id, cluster): self.cluster = cluster self.gene_id = gene_id self.norm_vals = [ float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0] ] #TPM. self.get_model() self.def_peaks() self.model_resetting() self.half_life() #self.printing() self.saving() def get_model(self): self.x = np.array([0, 1, 2, 6, 12, 24]) self.y = np.array(self.norm_vals) # Compound model with Voigt curve. self.background = ExponentialModel(prefix='b_') self.pars = self.background.guess(self.y, x=self.x) self.peak = VoigtModel(prefix='p_') self.pars += self.peak.guess(self.y, x=self.x) self.comp_mod = self.peak + self.background self.init = self.comp_mod.eval(self.pars, x=self.x) self.comp_out = self.comp_mod.fit( self.y, x=self.x, fit_kws={'nan_policy': 'propagate' }) # instead of 'omit', it keeps up the zero vals. self.comp_list = self.comp_out.fit_report().split('\n') self.comp_chisq = float(self.comp_list[6][-5:]) self.out = self.comp_out self.chisq = float(self.comp_list[6][-5:]) self.usedmod = self.comp_mod self.model_flag = "composite (exponential+Voigt)" return self.comp_out, self.comp_chisq, self.out, self.chisq, self.usedmod, self.model_flag def def_peaks(self): self.bestfit = self.comp_out.best_fit self.idx = np.argmin(np.abs(self.bestfit - 0.5)) self.mysort = np.argsort(np.abs(self.bestfit - 0.5)) self.peak_flag = None if all(i > 0.5 for i in self.bestfit): # Meaning that it is never reaching the half-life, and we don't do extrapolation (not enough data points). self.min = 0 self.max = 0 self.peak_flag = "No predictable half-life" else: if self.bestfit[self.idx] == 0.5: # If by accident one time point hits the half-life. self.half_life_y = self.bestfit[self.idx] self.half_life_x = self.idx self.peak_flag = "Exact compound half-life" elif self.bestfit[0] > 0.5 and self.bestfit[1] < 0.5: self.min = self.x[0] self.max = self.x[1] self.peak_flag = "Compound" elif self.idx == 5 and self.bestfit[self.idx - 1] < 0.5: # Last value crosses only self.max = self.x[self.idx] self.min = self.x[self.idx - 1] self.peak_flag = "Compound" elif np.abs(self.idx - self.mysort[1]) == 1: if self.bestfit[self.idx] < 0.5: self.min = self.x[self.idx - 1] self.max = self.x[self.idx] self.peak_flag = "Compound" elif self.bestfit[self.idx] > 0.5: self.min = self.x[self.idx] self.max = self.x[self.idx + 1] self.peak_flag = "Compound" elif np.abs(self.idx - self.mysort[1]) > 1: # Meaning that the steps are not linear, there's a bump. if self.bestfit[self.idx] < 0.5: self.min = self.x[self.idx - 1] self.max = self.x[self.idx] self.peak_flag = "Compound" elif self.bestfit[self.idx] > 0.5 and self.bestfit[ self.mysort[1]] < 0.5: if self.bestfit[self.idx + 1] < 0.5: self.min = self.x[self.idx] self.max = self.x[self.idx + 1] self.peak_flag = "Compound" #resetting!! else: self.min = self.x[self.mysort[1] - 1] self.max = self.x[self.mysort[1]] self.peak_flag = "Resetting" elif self.bestfit[self.idx] > 0.5 and self.bestfit[ self.mysort[1]] > 0.5: if self.bestfit[self.idx + 1] < 0.5: self.min = self.x[self.idx] self.max = self.x[self.idx + 1] self.peak_flag = "Compound" #resetting!! elif self.bestfit[self.idx + 1] > 0.5 and self.bestfit[ self.mysort[1] + 1] < 0.5: self.min = self.x[self.mysort[1] - 1] self.max = self.x[self.mysort[1]] self.peak_flag = "Resetting" return self.min, self.max, self.peak_flag, self.bestfit def model_resetting(self): if self.peak_flag != "Resetting": #go for the previous method pass elif self.peak_flag == "Resetting": # mostly for plotting, half-life needs new zeros self.scnd_peak = np.sort(self.bestfit)[-2] self.scnd_idx = np.argsort(self.bestfit)[-2] self.newzero = self.x[self.scnd_idx] # Cutting the new time scale, reset to 0. self.x2 = np.array( [i - self.newzero for i in self.x[self.scnd_idx:]]) #x2 = np.array([i for i in x[scnd_idx:]]) # Re-normalized and cutted array self.y2 = np.array( [i / self.y[self.scnd_idx] for i in self.y[self.scnd_idx:]]) #newarray = myarray[scnd_idx:] self.exp_mod = ExponentialModel(prefix='e_') self.pars = self.exp_mod.guess(self.y2, x=self.x2) self.init = self.exp_mod.eval(self.pars, x=self.x2) self.exp_out = self.exp_mod.fit(self.y2, x=self.x2, missing='drop') self.exp_list = self.exp_out.fit_report().split('\n') self.exp_chisq = float(self.exp_list[6][-5:]) self.out = self.exp_out self.chisq = float(self.exp_list[6][-5:]) self.usedmod = self.exp_mod self.bestfit = self.exp_out.best_fit self.idx = np.argmin(np.abs(self.bestfit - 0.5)) self.mysort = np.argsort(np.abs(self.bestfit - 0.5)) self.peak_flag = None if self.bestfit[self.idx] < 0.5: self.min = self.x2[self.idx - 1] self.max = self.x2[self.idx] self.peak_flag = "Resetted exponential" elif self.bestfit[self.idx] > 0.5: self.min = self.x2[self.idx] self.max = self.x2[self.idx + 1] self.peak_flag = "Resetted exponential" # For printing. if len(self.bestfit) < 6: l = [self.bestfit[-1]] * (6 - len(self.bestfit)) self.bestfit = np.append(self.bestfit, l) self.x = self.x2 self.y = self.y2 self.model_flag = "exponential" return self.min, self.max, self.peak_flag, self.bestfit, self.x, self.out, self.chisq, self.usedmod, self.model_flag def half_life(self): self.new_x = np.array([0]) self.hl_eval = np.array([0]) self.hl_array = np.array([0]) self.hl_coord = np.array([0]) self.step = None if self.max == 0: self.half_life_y = 0 self.half_life_x = 0 self.peak_flag = "No predictable half-life" else: self.half_life_y = 0 self.half_life_x = 0 self.step = 0.1 self.max_allowed = 3 self.attempt = 0 #while self.attempt < 3 or self.half_life_y == 0: while self.half_life_y == 0 and self.attempt < 3: self.attempt += 1 self.step = self.step / 100 self.ranging = np.arange( self.min, self.max, self.step ) # normally it 0.001, but the slope is so radical, can't catxh half-life. for j in np.nditer(self.ranging): self.new_x = np.array([j]) #self.h = self.out.eval_components(self.out.params,x=self.new_x) #self.hl_eval = list(self.h.values())[-1] self.hl_eval = self.out.eval(self.out.params, x=self.new_x) if self.hl_eval >= 0.50 and self.hl_eval <= 0.51: self.hl_array = np.append(self.hl_array, self.hl_eval) self.hl_coord = np.append(self.hl_coord, self.new_x) self.half_life_id = np.argmin(np.abs(self.hl_array - 0.5)) self.half_life_y = self.hl_array[self.half_life_id] self.half_life_x = self.hl_coord[self.half_life_id] self.peak_flag = self.peak_flag if self.half_life_y == 0: self.peak_flag = "Above permitted interpolation iterations" return self.half_life_y, self.half_life_x, self.peak_flag def saving(self): with open('model_fit_c5_average_filtering_compound.txt', 'a') as f: f.write( "%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%s\n" % (self.gene_id, self.cluster, self.model_flag, self.chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5], self.peak_flag))
class gene_set(): def __init__(self, gene_id, cluster): self.cluster = cluster self.gene_id = gene_id self.norm_vals = [ float(y) for y in [x[1:] for x in reference if x[0] == gene_id][0] ] #TPM. self.get_model() self.half_life() self.printing() self.saving() def get_model(self): self.x = np.array([0, 1, 2, 6, 12, 24]) self.y = np.array(self.norm_vals) #x = np.array([0,1,2,6,12,24]) # Exponential model for reference self.exp_mod = ExponentialModel(prefix='onlye_') self.pars = self.exp_mod.guess(self.y, x=self.x) self.init = self.exp_mod.eval(self.pars, x=self.x) self.exp_out = self.exp_mod.fit(self.y, x=self.x, missing='drop') self.exp_list = self.exp_out.fit_report().split('\n') self.exp_chisq = float(self.exp_list[6][-5:]) return self.exp_out, self.exp_chisq def half_life(self): self.new_x = np.array([0]) self.hl_eval = np.array([0]) self.hl_array = np.array([0]) self.hl_coord = np.array([0]) self.bestfit = self.exp_out.best_fit self.idx = np.argmin(np.abs(self.bestfit - 0.5)) if self.idx == 5 and self.bestfit[self.idx - 1] < 0.5: self.bestfit = self.exp_out.best_fit[:-1] self.idx = np.argmin(np.abs(self.bestfit - 0.5)) if self.bestfit[self.idx] == 0.5: self.half_life_y = self.bestfit[self.idx] self.half_life_x = self.idx elif 0.5 > self.bestfit[self.idx] and self.bestfit[self.idx - 1] > 0.5: self.max = self.x[self.idx] self.min = self.x[self.idx - 1] elif 0.5 < self.bestfit[self.idx] and self.bestfit[ self.idx] == self.bestfit[5]: self.min = 0 self.max = 0 elif 0.5 < self.bestfit[self.idx] and self.bestfit[self.idx + 1] < 0.5: self.min = self.x[self.idx] self.max = self.x[self.idx + 1] elif 0.5 < self.bestfit[self.idx] and self.bestfit[ self.idx + 1] > 0.5 and self.bestfit[self.idx + 2] < 0.5: self.min = self.x[self.idx + 1] self.max = self.x[self.idx + 2] elif 0.5 > self.bestfit[self.idx] and self.bestfit[ self.idx + 1] < 0.5 and self.bestfit[self.idx - 2] > 0.5: self.min = self.x[self.idx - 2] self.max = self.x[self.idx] self.ranging = np.arange(self.min, self.max, 0.001) if self.max > 0: # if self.min > 0 and self.max > 0: for j in np.nditer(self.ranging): self.new_x = np.array([j]) self.hl_eval = self.exp_out.eval(self.exp_out.params, x=self.new_x) if self.hl_eval >= 0.50 and self.hl_eval <= 0.51: self.hl_array = np.append(self.hl_array, self.hl_eval) self.hl_coord = np.append(self.hl_coord, self.new_x) self.half_life_id = np.argmin(np.abs(self.hl_array - 0.5)) self.half_life_y = self.hl_array[self.half_life_id] self.half_life_x = self.hl_coord[self.half_life_id] self.bestfit = self.exp_out.best_fit else: self.half_life_y = 0 self.half_life_x = 0 self.bestfit = self.exp_out.best_fit return self.half_life_y, self.half_life_x def printing(self): print(self.gene_id, self.cluster, "exponential", self.exp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5]) def saving(self): with open('model_fit_c5_average_filtering_newexp.txt', 'a') as f: (f.write( "%s\t%s\t%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n" % (self.gene_id, self.cluster, "exponential", self.exp_chisq, self.half_life_y, self.half_life_x, self.norm_vals[0], self.norm_vals[1], self.norm_vals[2], self.norm_vals[3], self.norm_vals[4], self.norm_vals[5], self.bestfit[0], self.bestfit[1], self.bestfit[2], self.bestfit[3], self.bestfit[4], self.bestfit[5])))
def calculate_feature(image_path, value=[1, 0, 0, 0, 1, 0], return_image=False): debug_image = [] image = cv2.imread(image_path) from detector_borde_plastico import detector_borde_plastico image_bp, images_dp_debug = detector_borde_plastico(image) debug_image.extend(images_dp_debug) #features = [mask_pasto == 1] """ image_kmeans = kmeans(image_bottom_lab, K=40) image_kmeans_rgb = cv2.cvtColor(image_kmeans, cv2.COLOR_LAB2BGR) debug_image.append(image_kmeans_rgb) image_kmeans_rgb_cut = image_kmeans_rgb[0:image_kmeans_rgb.shape[0], int(image_kmeans_rgb.shape[1]*0.167):int(image_kmeans_rgb.shape[1]*0.905)] debug_image.append(image_kmeans_rgb_cut) image_kmeans2 = kmeans(image_kmeans_rgb_cut, K=5) #image_kmeans2_rgb = cv2.cvtColor(image_kmeans2, cv2.COLOR_LAB2BGR) debug_image.append(image_kmeans2) lowerBound = np.array(value[0:3])*255.0 upperBound = np.array(value[3:6])*255.0 # this gives you the mask for those in the ranges you specified, # but you want the inverse, so we'll add bitwise_not... image_inrange = cv2.inRange(image_kmeans2, lowerBound, upperBound) image_inrange_color = cv2.bitwise_not(image_kmeans_rgb_cut, mask=image_inrange) debug_image.append(image_inrange_color) """ """ cv2.kmeans(image_bottom_lab, 40, ) from sklearn.cluster import MiniBatchKMeans image_bottom_lab_reshape = image_bottom_lab.reshape((image_bottom_lab.shape[0] * image_bottom_lab.shape[1], 3)) clt = MiniBatchKMeans(n_clusters=10) labels = clt.fit_predict(image_bottom_lab_reshape) quant = clt.cluster_centers_.astype("uint8")[labels] quant = quant.reshape(image_bottom_lab.shape) image_bottom_lab_res = image_bottom_lab_reshape.reshape(image_bottom_lab.shape) # convert from L*a*b* to RGB quant = cv2.cvtColor(quant, cv2.COLOR_LAB2BGR) image_bottom_lab_res = cv2.cvtColor(image_bottom_lab_res, cv2.COLOR_LAB2BGR) sobelx64f = cv2.Sobel(image_bottom, cv2.CV_64F, 1, 0, ksize=3) abs_sobel64f = np.absolute(sobelx64f) sobel_8u = np.uint8(abs_sobel64f) lowerBound = np.array(value[0:3])*255.0 upperBound = np.array(value[3:6])*255.0 # this gives you the mask for those in the ranges you specified, # but you want the inverse, so we'll add bitwise_not... image_inrange = cv2.inRange(sobel_8u, lowerBound, upperBound) image_inrange_color = cv2.bitwise_not(image_bottom, mask=image_inrange) features = image.flatten()[:500] debug_image.append(image_bottom) debug_image.append(sobel_8u) debug_image.append(image_inrange) debug_image.append(image_inrange_color) debug_image.append(quant) debug_image.append(image_bottom_lab_res) """ x = [] y = [] for x_ in xrange(mask_pasto.shape[1]): for y_ in xrange(10, mask_pasto.shape[0]): if mask_pasto[y_, x_] == 255: x.append(x_) y.append(y_) break x = np.array(x) y = np.array(y) x1 = x[x < (mask_pasto.shape[1] / 2)] y1 = y[x < (mask_pasto.shape[1] / 2)] x1 = x1[y1 > (mask_pasto.shape[0] / 2)] y1 = y1[y1 > (mask_pasto.shape[0] / 2)] y1 = y1.max() - y1 x = x - x[0] # x offset mask_pasto2 = np.zeros(thresh.shape, np.uint8) for i in xrange(len(x1)): mask_pasto2[y1[i], x1[i]] = 255 debug_image.append(mask_pasto2) x2 = x[x > (mask_pasto.shape[1] / 2)] y2 = y[x > (mask_pasto.shape[1] / 2)] y2 = y2 - y2.max() from lmfit.models import ExponentialModel exp_mod = ExponentialModel(prefix='exp_') pars = exp_mod.guess(y1, x=x1) out = exp_mod.fit(y, pars, x=x) print "error en amplitud:", out.params['exp_amplitude'].stderr, print "error en decay:", out.params['exp_decay'].stderr, features = np.array( [out.best_values['exp_decay'], out.params['exp_decay'].stderr]) #print(out.fit_report(min_correl=0.5)) #print pars if return_image: return features, debug_image return features
x = langevin.time_series(A=A, D=D, delta_t=delta_t, N=N) # see http://docs.scipy.org/doc/scipy-0.15.1/reference/generated/scipy.signal.fftconvolve.html # autocorr = signal.fftconvolve(x, x[::-1], mode='full') f = np.fft.rfft(x) acf = np.fft.irfft(f * np.conjugate(f)) acf = np.fft.fftshift(acf) / N autocorr = acf[int(N / 2):] # n=len(autocorr) # autocorr=autocorr[int((n-1)/2):]*2.0/(n+1) acf_avg = acf_avg + autocorr acf_var = acf_var + autocorr**2 y = autocorr[:min(int(N / 2), 1000)] t = np.arange(min(int(N / 2), 1000)) out = mod.fit(y, amplitude=1.0, decay=100.0, x=t) #print(out.fit_report(min_correl=0.25)) t_list.append(out.values['decay']) tstd_list.append(out.covar[0, 0]) A_list.append(out.values['amplitude']) Astd_list.append(out.covar[1, 1]) mean_list.append(x.mean()) std_list.append(x.std()) print('mean: ', x.mean(), 'std: ', x.std(), 'amplitude: ', out.values['amplitude'], 'decay: ', out.values['decay']) acf_avg = acf_avg / M acf_stderr = np.sqrt((acf_var / M - (acf_avg / M)**2) / M) y = acf_avg dy = acf_stderr
for i in range(M): print("***** Iteration ", i, " *****") data = langevin.time_series(A=A, D=D, delta_t=delta_t, N=N) # calculate autocorrelation function f = np.fft.rfft(data) acf = np.fft.irfft(f * np.conjugate(f)) acf = np.fft.fftshift(acf) / N autocorr = acf[int(N / 2):] y = autocorr[:min(int(N / 2), P)] t = np.arange(min(int(N / 2), P)) mod = ExponentialModel() pars = mod.guess(y, x=t) try: out = mod.fit(y, pars, x=t) except: fit_results = np.zeros(4) print('fit did not work') else: fit_results = np.array([ out.values['decay'] * delta_t, np.sqrt(out.covar[0, 0]) * delta_t, out.values['amplitude'], np.sqrt(out.covar[1, 1]) ]) print(out.fit_report(min_correl=0.25)) trace = sm.run(x=data, aD=alpha_D, bD=beta_D, aA=alpha_A,
data = pd.read_csv("fits.csv") groups = data.groupby("assignment") for ind, group in groups: plt.figure(figsize=(4, 3)) plt.errorbar( group.vclist, group.amp, yerr=group.amp_err, fmt="o", markersize=10, label=group.assignment.iloc[0], c="#33a02c", ) mod = ExponentialModel() pars = mod.guess(group.amp, x=group.vclist) fit = mod.fit(group.amp, x=group.vclist, params=pars) sim_x = np.linspace(group.vclist.min(), group.vclist.max()) sim_y = fit.eval(x=sim_x) plt.plot(sim_x, sim_y, "-", c="#1f78b4", linewidth=4) plot = plt.gca() plot.axes.xaxis.set_ticklabels([]) plot.axes.yaxis.set_ticklabels([]) for axis in ["top", "bottom", "left", "right"]: plot.axes.spines[axis].set_linewidth(2.0) # plt.legend() plt.savefig("eg_fit.pdf") plt.show() exit()