def para_fit(X): """Function to calculate results in parallel Input is the r matrix, after normalization by the standard deviations, R/R_std, such that the data is normalized to a standard deviation of 1, upper and lower bounds, the desired confidence interval (.95, for example), and finally the number of Monte-Carlo repetitions to perform (if set to 0, performs linear propagation-of-erro) """ Y=lsq(X[0],X[1],bounds=(X[2],X[3])) rho=Y['x'] Rc=Y['fun']+X[1] if X[5]==0: std=np.sqrt(np.sum(np.linalg.pinv(X[0])**2,axis=1)) nstd=norm.ppf(1/2+X[4]/2) u=nstd*std l=nstd*std else: Y1=list() nmc=max([X[5],np.ceil(2/X[4])]) for k in range(0,X[5]): Y0=lsq(X[0],Rc+np.random.normal(size=X[1].shape)) Y1.append(Y0['x']) std=np.std(Y1,axis=0) Y1sort=np.sort(Y1,axis=0) in_l=np.round(nmc*(1/2-X[4]/2)) in_u=np.round(nmc*(1/2+X[4]/2)) l=rho-Y1sort[int(in_l)] u=Y1sort[int(in_u)]-rho return rho,std,l,u
def arma_est_bip_m(x, p, q, beta_hat_s, a_sc_final): N = len(x) x = np.array(x) F_mm = lambda beta: sp.sqrt(1 / (N - p) * sp.sum( rsp.muler_rho2(rsp.arma_resid(x / a_sc_final, beta, p, q)))) F_bip_mm = lambda beta: sp.sqrt(1 / (N - p) * sp.sum( rsp.muler_rho2(rsp.bip_resid(x / a_sc_final, beta, p, q)))) beta_arma_mm = lsq(F_mm, beta_hat_s, xtol=5 * 1e-5, ftol=5 * 1e-5, method='lm')[0] beta_bip_mm = lsq(F_bip_mm, beta_hat_s, xtol=5 * 1e-5, ftol=5 * 1e-5, method='lm')[0] a_rho2_mm = 1 / (N - p) * np.sum( muler_rho2(rsp.arma_resid(x / a_sc_final, beta_arma_mm, p, q))) a_bip_rho2_mm = 1 / (N - p) * np.sum( muler_rho2(rsp.bip_resid(x / a_sc_final, beta_bip_mm, p, q))) beta_hat = beta_arma_mm if a_rho2_mm < a_bip_rho2 else beta_bip_mm # Output the results phi_bip_mm = -beta_hat[:p] if p > 0 else [] theta_bip_mm = -beta_hat[p:] if q > 0 else [] return phi_bip_mm, theta_bip_mm
def dist_opt(X): """ Optimizes a distribution that yields detector responses, R, where the distribution is required to be positive, and have an integral of 1-S2 Ropt,dist=dist_opt((R,R_std,rhoz,S2,dz)) Note- intput is via tuple """ R,R_std,rhoz,S2=X total=np.atleast_1d(1-S2) """Later, we may need to play with the weighting here- at the moment, we fit to having a sum of 1, but in fact it is not forced....it should be """ ntc=rhoz.shape[1] rhoz=np.concatenate((rhoz/np.repeat(np.atleast_2d(R_std).T,ntc,axis=1), np.atleast_2d(np.ones(ntc))),axis=0) Rin=np.concatenate((R/R_std,total)) dist=0 while np.abs(np.sum(dist)-total)>1e-3: #This is a check to see that the sum condition has been satisfied dist=lsq(rhoz,Rin,bounds=(0,1))['x'] Rin[-1]=Rin[-1]*10 rhoz[-1]=rhoz[-1]*10 Ropt=np.dot(rhoz[:-1],dist)*R_std return Ropt,dist
def baselineWithAmmonia(y, v, baselineIndex, freqthrow=4.11 * u.MHz, v0=8.5, sigmav=1.0 * u.km / u.s, line='oneone', blorder=5, noiserms=None): x = np.linspace(-1, 1, len(y)) chthrow = (freqthrow.to(u.Hz).value / acons.freq_dict[line] * 299792.458 / np.abs(v[0] - v[1])) chthrow = (np.round(chthrow)).astype(np.int) if noiserms is None: noiserms = mad1d((y - np.roll(y, -2))[baselineIndex]) opts = lsq( ammoniaLoss, np.r_[[ np.nanmax(y[baselineIndex]), v[np.nanargmax(y[baselineIndex])], 1.0 ], np.zeros(blorder + 1)], args=(y[baselineIndex], x[baselineIndex], v[baselineIndex], noiserms), kwargs={'chthrow': chthrow}, loss='soft_l1') return y - legendre.legval(x, opts.x[3:])
def get_chi(blorder_max, ydata, xdata, blindex, noise): """ Returns the best-fit Legendre polynomial to an input spectra, based on a comparison of the reduced chi-squared values for each order of the polynomial. blorder_max = maximum order of polynomial to fit ydata = spectrum y values xdata = spectrum x values blindex = the indices of the spectra to include in baseline fitting noise = rms noise of spectrum""" opts = lsq(legendreLoss, np.zeros(blorder_max + 1), args=(ydata[blindex], xdata[blindex], noise), loss='arctan') chis = [] # first entry is order=1, second is order=2, ..., up to order=blorder ymods = [] # Loop through each order of polynomial and create model baseline # and calculate that model's chi-squared values for i in np.arange(blorder_max)+1: # i is the polynomial order ymod = legendre.legval(xdata, np.array(opts.x)[0:i+1]) #+1 to get last term ymods.append(ymod) chi = redchisqg(ydata,ymod,deg=i+1) chis.append(chi) # Find the model with the lowest chi-squared value find_low = np.where(np.array(chis)==min(chis)) low_model = np.array(ymods)[find_low] return ymod
def robustBaseline(y, baselineIndex, blorder=1, noiserms=None): x = np.linspace(-1, 1, len(y)) if noiserms is None: noiserms = mad1d((y - np.roll(y, -2))[baselineIndex]) opts = lsq(legendreLoss, np.zeros(blorder + 1), args=(y[baselineIndex], x[baselineIndex], noiserms), loss='arctan') return y - legendre.legval(x, opts.x)
def alignTrajs(p_gt_C, p_O_C, initial_T_gt_O): state = np.hstack([np.zeros([1, 3]), initial_T_gt_O[0:3, 3:4].T]).reshape([-1]) bound_alignE = functools.partial(alignError, initial_T_gt_O, p_O_C, p_gt_C) optim_state = lsq(bound_alignE, state).x print(optim_state) errs = bound_alignE(optim_state).reshape([3, -1]) ate = np.sqrt(np.sum(np.square(errs), axis=0)).mean() optim_T_gt_O = applyT(initial_T_gt_O, optim_state) return [optim_T_gt_O, ate]
def findFlows(flowModel, flowMeasurements): flowGuess = 0.1 + 5.0 * ranf( flowModel.nFlows + flowModel.nsched) # includes timeoffsets result = lsq(optFunc, flowGuess, bounds=(flowModel.lowerBounds, flowModel.upperBounds), loss='huber', args=(flowMeasurements, flowModel)) plotResids = formatResids(flowMeasurements, result.fun) return result
def robustBaseline(y, v, baselineIndex, blorder=1, noiserms=None): x = np.linspace(-1, 1, len(y)) if noiserms is None: noiserms = mad1d((y - np.roll(y, -2))[baselineIndex]) opts = lsq(legendreLoss, np.zeros(blorder + 1), args=(y[baselineIndex], x[baselineIndex], noiserms), loss='arctan') return y - legendre.legval(x, opts.x)
def _fit_model(self, data, bkgd_shape=None, lsqKwargs=None): ''' Function that performs curve fitting based on use of scipy.optimize.curve_fit Parameters ---------- data : ndarray The data to be used in the model. Prepared by the function MLLSmodel._prepare_model_data() bkgd_shape : string or None If not None, must be a string containing the name of one of the available background shapes: - power law - log-linear - double power law lsqKwargs : dict Keword arguments to be taken by scipy.optimize.curve_fit Returns ------- opt_cofs : ndarray The optimised fit coefficients cov : ndarray The covariance matrix of the fit ''' if lsqKwargs is None: lsqKwargs = {} y_obs = data[0, :] x = data[1, :] components = data[2:, :] def background(t, coeffs): if bkgd_shape == 'power law': return coeffs[-2] * pow(t, coeffs[-1]) if bkgd_shape == 'log-linear': return np.log(coeffs[-2] * pow(t, coeffs[-1])) if bkgd_shape == 'double power law': return coeffs[-4] * pow(t, coeffs[-3]) + coeffs[-2] * pow( t, coeffs[-1]) def model(t, *coeffs): y = 0.0 for i in range(components.shape[0]): y += coeffs[i] * components[i, :] if bkgd_shape: y += background(t, coeffs) return y opt_cofs, cov = lsq(model, x, y_obs, **lsqKwargs) # self.last = [opt_cofs, cov] return opt_cofs, cov
def polyfit2d(f, y, x, unc=None, order=1): """Fit a polynomial surface to 2D data. Assumes the axes are independent of each other. Evaluate the fit via: np.polyval(polyy, y) + np.polyval(polyx, x). Parameters ---------- f: array The array to fit. y, x: array The coordinates for each value of `f`. unc: array, optional The uncertainties on each value of `f`, or a single value for all points. If `None`, then 1 is assumed. order: int, optional The polynomial order(in one dimension) of the fit. Returns ------- polyx, polyy: ndarray The polynomial coefficients, in the same format as from `np.polyfit`. cov: ndarray The covariance matrix. v1.0.0 Written by Michael S. Kelley, UMD, Mar 2009 """ from scipy.optimize import leastsq as lsq # the fitting function def chi(p, y, x, f, unc, order): cy = p[:1+order] cx = p[1+order:] model = np.zeros(f.shape) + np.polyval(cy, y) + np.polyval(cx, x) chi = (f - model) / unc return chi if unc is None: unc = 1.0 # run the fit guess = np.zeros((order + 1) * 2) result = lsq(chi, guess, args=(y, x, f, unc, order), full_output=True) fit = result[0] cov = result[1] cy = fit[:1+order] cx = fit[1+order:] return cx, cy, cov
def polyfit2d(f, y, x, unc=None, order=1): """Fit a polynomial surface to 2D data. Assumes the axes are independent of each other. Evaluate the fit via: np.polyval(polyy, y) + np.polyval(polyx, x). Parameters ---------- f : array The array to fit. y, x : array The coordinates for each value of `f`. unc : array, optional The uncertainties on each value of `f`, or a single value for all points. If `None`, then 1 is assumed. order : int, optional The polynomial order (in one dimension) of the fit. Returns ------- polyx, polyy : ndarray The polynomial coefficients, in the same format as from `np.polyfit`. cov : ndarray The covariance matrix. v1.0.0 Written by Michael S. Kelley, UMD, Mar 2009 """ from scipy.optimize import leastsq as lsq # the fitting function def chi(p, y, x, f, unc, order): cy = p[:1 + order] cx = p[1 + order:] model = np.zeros(f.shape) + np.polyval(cy, y) + np.polyval(cx, x) chi = (f - model) / unc return chi if unc is None: unc = 1.0 # run the fit guess = np.zeros((order + 1) * 2) result = lsq(chi, guess, args=(y, x, f, unc, order), full_output=True) fit = result[0] cov = result[1] cy = fit[:1 + order] cx = fit[1 + order:] return cx, cy, cov
def analyzeLoop(self): def expFun(k,x): return k[0]+k[1]*exp(-x/k[2]) def expLSQ(k,x,y): return (expFun(k,x)-y)**2 if self.infoDict['stimVar'].get()=='E1': sv=0 else: sv=1 if self.ITC.mode[sv]==VC: vm=0 im=1 else: vm=1 im=0 pre=self.params['pre'] dur=self.params['dur'] post=self.params['post'] reps=self.params['reps'] di=list() for i in self.dataIn: di.append(i[:-EXTRA]) #print 'L:%d vs %d' % (len(di[vm]),((pre+dur+post)/DT)*reps) voltage=mean(di[vm].reshape((pre+dur+post)/DT,reps,order='F'),axis=1) current=mean(di[im].reshape((pre+dur+post)/DT,reps,order='F'),axis=1) y=current[round((pre+0.48)/DT):round((pre+dur-0.48)/DT)] x=arange(len(y))*DT k=lsq(expLSQ,array([0,1,10]),(x,y)) #print 'K:%2.0f,%2.0f,%2.0f' % (k[0][0],k[0][1],k[0][2]) I0=mean(current[round(0.48/DT):round((pre-0.48)/DT)]) V0=mean(voltage[round(0.48/DT):round((pre-0.48)/DT)]) Iss=mean(current[(pre+dur-3)/DT:(pre+dur-0.48)/DT]); Iss=k[0][0] Imx=expFun(k[0],-0.48) Vss=mean(voltage[(pre+dur-3)/DT:(pre+dur-0.48)/DT]) Rseries=(Vss-V0)/(Imx-I0); Rin=(Vss-V0)/(Iss-I0); Rm=Rin-Rseries; #charge=mean(exp_fun(k,-0.48:DT:dur-0.48))*dur/DT; x=arange(dur/DT)*DT-0.48 charge=0 for i in x: charge+=expFun(k[0],i) Cm=charge/(Vss-V0); self.analysisDict.update({'I0':I0,'V0':V0,'Iss':Iss,'Imx':Imx,'Vss':Vss,'Rseries':Rseries,'Rin':Rin,'Rm':Rm,'Cm':Cm,'charge':charge,'k':k[0],'current':current})
def arma_est_bip_mm(x,p,q): bip_s_est = rsp.arma_est_bip_s(x,p,q) beta_hat_s = np.array([*bip_s_est['ar_coeffs'], *bip_s_est['ma_coeffs']]) N = len(x) F_mm = lambda beta: 1/(N-p)*rsp.muler_rho2(rsp.arma_s_resid(x,beta,p,q)[0]) F_bip_mm = lambda beta: 1/(N-p)*rsp.muler_rho2(rsp.bip_s_resid(x,beta,p,q)[0]) beta_arma_mm = lsq(F_mm, -beta_hat_s,xtol=5*1e-7,ftol=5*1e-7,method='lm')['x'] beta_bip_mm = lsq(F_bip_mm, -beta_hat_s,xtol=5*1e-7,ftol=5*1e-7,method='lm')['x'] a = rsp.arma_s_resid(x, beta_arma_mm, p, q)[0] a_sc = rsp.m_scale(a) # innovations m-scale for ARMA model a_bip = rsp.bip_s_resid(x, beta_bip_mm, p, q)[0] a_bip_sc = rsp.m_scale(a_bip) # innovations m-scale for BIP-ARMA model # final parameter estimate uses the model that provides smallest m_scale beta_hat = beta_arma_mm if a_sc<a_bip_sc else beta_bip_mm # final m-scale a_m_sc = min(a_sc, a_bip_sc) # Output the results results = {'ar_coeffs': -beta_hat[:p], 'ma_coeffs': -beta_hat[p:], 'inno_scale': a_m_sc, 'cleaned_signal': bip_s_est['cleaned_signal'], 'ar_coeffs_init': bip_s_est['ar_coeffs'], 'ma_coeffs_init': bip_s_est['ma_coeffs']} return results
def model_point(self, coords, comps, init_guess, fit_background=None, lsqKwargs=None): ''' Parameters ---------- coords : tuple of ints comps : list of strings init_guess : list of floats fit_background : string or None Returns ------- coefficients : array of floats ''' if lsqKwargs is None: lsqKwargs = {} def background(t, coeffs): if fit_background == 'power law': return coeffs[-2] * pow(t, coeffs[-1]) if fit_background == 'log-linear': return np.log(coeffs[-2] * pow(t, coeffs[-1])) if fit_background == 'double power law': return coeffs[-4] * pow(t, coeffs[-3]) + coeffs[-2] * pow( t, coeffs[-1]) def model(t, *coeffs): y = 0.0 for i, comp in enumerate(comps): y += coeffs[i] * self.stds.ready[comp].inav[coords].data if fit_background: y += background(t, coeffs) return y y_Obs = self.HL.data.inav[coords].data t = self.HL.data.axes_manager[self.sigDim].axis opt_cofs, cov = lsq(model, t, y_Obs, p0=init_guess, **lsqKwargs) self.last = opt_cofs return opt_cofs
def lsq_excess_fit(ref_excess_dict, red_law, EBV, rv_guess, filters, zp): prefix = zp['prefix'] filter_eff_waves = np.array([snc.get_bandpass(prefix+f).wave_eff for f in filters]) ref_excess = np.array([ref_excess_dict[f] for f in filters]) def lsq_func(Y): RV = Y[0] ftz_curve = red_law(filter_eff_waves, np.zeros(filter_eff_waves.shape), -EBV, RV, return_excess=True) return ftz_curve-ref_excess Y = np.array([rv_guess]) valid_phases = {} return lsq(lsq_func, Y)
def trace(im, err, guess): """Trace the peak pixels along the second axis of an image. Parameters ---------- im : array The image to trace. err : array The image uncertainties. Set to None for unweighted fitting. guess : array The initial guesses for Gaussian fitting the y-value at x=0: (mu, sigma, height, [m, b]) = position (mu), width (sigma), height, and linear background m*x + b. Returns ------- y : ndarray y-axis positions of the peak, along the second axis. """ from scipy.optimize import leastsq as lsq from ..util import gaussian def chi(p, x, y, err): if len(p) > 3: mu, sigma, height, m, b = p else: mu, sigma, height = p m, b = 0, 0 model = gaussian(x, mu, sigma) * height + m * x + b chi = (y - model) / err return chi y = np.zeros(im.shape[1]) y0 = np.arange(im.shape[0]) last = guess for i in xrange(im.shape[1]): if err is None: err = np.ones_like(y) fit, err = lsq(chi, last, (np.array(x), np.array(y), np.array(err))) y[i] = fit[0] last = fit return y
def optimize(self): self.optimization_setup() p = np.array([0.02289848, 0.01645834]) jump_opt = False if jump_opt == False: self.fast_run = True p = np.array([1.]) if self.lam != 0: p = np.array([1., 1.]) ''' Estimate for HG: False T = 5 p = np.array([0.01646613, 0.18810675]) ''' ''' Estimate for HG: True T = 5 ''' p = np.array([0.01343025, 0.05155523]) fit = lsq(self.objective_function, p, bounds=(0, np.inf)) p = fit.x self.fast_run = False self.objective_function(p) print('Plotting complete') print('>>>Finished optimizing with:') print('HG :', self.use_hg) print('Lambda:', self.lam != 0) if jump_opt == False: print('Optimal:', fit.x) else: return lambda_label = str(self.lam != 0) txt = 'opt_lambda_' + lambda_label + '.txt' fname = os.path.join(self.fem_opt_exp_data_vec_dir, txt) np.savetxt(fname, p)
def fwhm(im, yx, bg=True, **kwargs): """Compute the FWHM of an image. Parameters ---------- im : array The image to fit. yx : array The center on which to fit. bg : bool, optional Set to `True` if there is a constant background to be considered. **kwargs Any `radprof` keyword, e.g., `range` or `bins`. Returns ------- fwhm : float The FHWM of the radial profile. """ from scipy.optimize import leastsq as lsq from ..util import gaussian R, I, n = radprof(im, yx, **kwargs)[:3] r = R[I < (I.max() / 2.0)][0] # guess for Gaussian sigma if bg: def fitfunc(p, R, I): return I - gaussian(R, 0, p[0]) * p[1] + p[2] guess = (r, I.max(), I.min()) else: def fitfunc(p, R, I): return I - gaussian(R, 0, p[0]) * p[1] guess = (r, I.max()) fit = lsq(fitfunc, guess, args=(R, I))[0] return abs(fit[0]) * 2.35
def fitSkewedGaussian(array): (y,bins) = numpy.histogram(array,bins=50,normed=True) x = 0.5*(bins[:-1] + bins[1:]) y /= float(len(array)) maxloc = y.argmax() mode = x[maxloc] # fit skewed Gaussian gamma = skew(array) delta = numpy.sqrt(numpy.pi*numpy.abs(gamma)**(2./3.)/2./(numpy.abs(gamma)**(2./3.) + ((4-numpy.pi)/2)**(2./3.))) if delta < 1: alpha = delta/numpy.sqrt(1-delta**2.0) else: alpha = 0.99 if gamma < 0: alpha *= -1 params = numpy.array([mode,array.var(),alpha,y[maxloc]]) out = lsq(errfunc,params,args=(x,y),full_output=1) pfinal = out[0] return pfinal
def fitSkewedGaussian(array): (y, bins) = numpy.histogram(array, bins=50, normed=True) x = 0.5 * (bins[:-1] + bins[1:]) y /= float(len(array)) maxloc = y.argmax() mode = x[maxloc] # fit skewed Gaussian gamma = skew(array) delta = numpy.sqrt(numpy.pi * numpy.abs(gamma)**(2. / 3.) / 2. / (numpy.abs(gamma)**(2. / 3.) + ((4 - numpy.pi) / 2)**(2. / 3.))) if delta < 1: alpha = delta / numpy.sqrt(1 - delta**2.0) else: alpha = 0.99 if gamma < 0: alpha *= -1 params = numpy.array([mode, array.var(), alpha, y[maxloc]]) out = lsq(errfunc, params, args=(x, y), full_output=1) pfinal = out[0] return pfinal
def background(image, mask, fraction): rind = (((skm.binary_dilation(mask,selem=skm.disk(6))) ^ skm.binary_dilation(mask,selem=skm.disk(3))) * np.isfinite(image)) if np.any(rind): rindvals = image[rind] clipval = 4*np.percentile(rindvals,15.87)-\ 3*(np.percentile(rindvals,2.28)) rind *= image <= clipval yv,xv = np.where(rind) x0,y0 = np.median(xv),np.median(yv) dataz = np.c_[np.ones(xv.size), xv-x0, yv-y0] try: lsqcoeffs, _, _, _ = np.linalg.lstsq(dataz,image[rind]) outputs = lsq(myplane, np.r_[lsqcoeffs,0], args=(xv-x0, yv-y0, image[rind]), loss = 'soft_l1') coeffs = outputs.x yhit, xhit = np.where(mask) bg = (coeffs[0] + coeffs[1] * (xhit-x0) + coeffs[2] * (yhit-y0) + coeffs[3] * (yhit-y0) * (xhit-x0)) # fracvals = 1 - np.nan_to_num(fraction[yhit, xhit]) bgavg = np.nansum(bg) # * fraction[yhit, xhit]) # import pdb; pdb.set_trace() return(bgavg) except ValueError: return(np.nan)
def calc_ftz_lsq_fit(S1, S2, filters, zp, ebv, rv_guess, dist_mod_true, dist_mod_weight, constrain_dist_mod=True, weight_dist_mod=True): ''' Least Square Fitter for Fitzpatrick-Massa (1999) reddening law. This function assumes that S1 is an unreddened twin of S2 -- in our case S1 is ::NOTES FOR ME:: must input spectra with matching phases fit S1 to S2 1. calc 12cu mags 2. interpolate with pristine 11fe spectra at 12cu phases 3. run lsq_helper_func with pristine spectra x0 is a dict like: {'ebv': -1.37, 'rv': 1.4} { 'av': 1.85, 'p': -2.1} DOCS -> http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html xtol/ftol == something big like .01? ''' from scipy.optimize import leastsq as lsq # check that given spectra have matching phases s1_phases, s2_phases = np.array([t[0] for t in S1]), np.array([t[0] for t in S2]) if not np.array_equal(s1_phases, s2_phases): return ValueError('Phases in given spectra do not match!') prefix = zp['prefix'] ############################################################################ ##### FUNCTIONS ############################################################ # returns a lightcurve for the given filter def bandmags(f, spectra): bandfluxes = [s.bandflux(prefix+f) for s in spectra] return -2.5*np.log10( bandfluxes/zp[f] ) # function to be used in least-sq optimization; must only take a numpy array (y) as input def lsq_func(Y): rv = Y[0] #dist_mod_shift = Y[1] reddened = [snc.Spectrum(spec.wave, redden_fm(spec.wave, spec.flux, ebv, rv)) for spec in s1_spectra] reddened_mags = [bandmags(f, reddened) for f in filters] ## if constrain_dist_mod: ## dist_mod_shift = dist_mod_true ## if weight_dist_mod: ## S1_REF = np.concatenate( [mag+dist_mod_shift for mag in reddened_mags] ) ## return np.concatenate(( S2_REF - S1_REF , [dist_mod_weight*(dist_mod_shift - dist_mod_true)] )) ## else: ## S1_REF = np.concatenate( [mag+dist_mod_shift for mag in reddened_mags] ) ## return S2_REF - S1_REF S1_REF = np.concatenate( [mags+dist_mod_true for mags in reddened_mags] ) return S2_REF - S1_REF ############################################################################ s1_spectra, s2_spectra = [t[1] for t in S1], [t[1] for t in S2] SN12CU_MW = dict( zip( 'UBVRI', [0.117, 0.098, 0.074, 0.058, 0.041] )) # 12cu Milky Way extinction # get a concatenated array of lightcurves per filter, also correct for 12cu milky way extinction S2_REF = np.concatenate( [bandmags(f, s2_spectra) - SN12CU_MW[f] for f in filters] ) #Y = np.concatenate(( red_vars, [dist_mod_true] )) # best guess vector #Y = np.array([rv_guess, dist_mod_true]) Y = np.array([rv_guess]) return lsq(lsq_func, Y) # , full_output=False, xtol=0.01, ftol=0.01)
xmin = np.min(x) ymin = np.min(y) ymax = np.max(y) binwidth = (xmax - xmin) / 100. bins = np.arange(xmin, xmax + binwidth, binwidth) nx, binx, patchx = axHistx.hist(x, bins=bins) xp = (binx[:-1] + binx[1:]) / 2 binwidth = (ymax - ymin) / 100. bins = np.arange(ymin, ymax + binwidth, binwidth) ny, biny, patchy = axHisty.hist(y, bins=bins, orientation='horizontal') yp = (biny[:-1] + biny[1:]) / 2 # fit with skewed gaussian and plot params = np.array([nx.max(), 0.01, xp.mean(), np.sqrt(xp.var())]) out = lsq(errfunc, params, args=(xp, nx), full_output=1) pfinal = out[0] #print pfinal fitX = np.linspace(xmin, xmax, 200) fitY = fitfunc(pfinal, fitX) axHistx.plot(fitX, fitY) params = np.array([ny.max(), 0.01, yp.mean(), np.sqrt(yp.var())]) out = lsq(errfunc, params, args=(yp, ny), full_output=1) pfinal = out[0] #print pfinal fitX = np.linspace(ymin, ymax, 200) fitY = fitfunc(pfinal, fitX) axHisty.plot(fitX, fitY) # the xaxis of axHistx and yaxis of axHisty are shared with axScatter,
def calc_irlum(catalog = 'cohrs_finalcatalog_clumpflux_medaxis.fits', refresh=False): ctrr = 0 cat = Table.read(catalog) current_open_file = '' if 'ir_luminosity' not in cat.keys(): keylist = ['ir_luminosity','ir_flux','ir_flux_short', 'ir_lum_short','bg_flux','bg_lum'] for thiskey in keylist: c = Column(np.zeros(len(cat))+np.nan,name=thiskey) cat.add_column(c) for cloud in cat: if np.isnan(cloud['ir_luminosity']) or refresh: orig_file = cloud['orig_file']+'.fits' asgn_file = cloud['orig_file']+'_fasgn.fits' higal_file = orig_file.replace('cohrs','higal_xmatch') if os.path.isfile(datadir+'COHRS/'+orig_file) and \ os.path.isfile(datadir+'HIGAL_MATCHED/'+higal_file): if current_open_file != datadir+'COHRS/'+orig_file: hdu = fits.open(datadir+'COHRS/'+orig_file,memmap=False) w = wcs.WCS(hdu[0].header) hdr2 = w.to_header() hdr2['BMAJ'] = 15./3600 hdr2['BMIN'] = 15./3600 hdr2['BPA'] = 0. co = SpectralCube(hdu[0].data,w,header=hdr2) irfull= fits.open(datadir+'HIGAL_MATCHED/'+higal_file,memmap=False) irlong = fits.open(datadir+'HIGAL_MATCHED2/'+higal_file,memmap=False) irmap = (irfull[0].data-irlong[0].data) irmap2 = irfull[0].data asgn = fits.getdata(datadir+'ASSIGNMENTS/'+asgn_file,memmap=False) masked_co = co.with_mask(asgn>0*u.dimensionless_unscaled) moment = masked_co.moment(0) current_open_file = datadir+'COHRS/'+orig_file cat.write('output_catalog.fits',overwrite=True) # mask = np.zeros(co.shape,dtype=np.bool) # mask[asgn == cloud['cloud_id']]=True cloud_cube = co.with_mask(asgn == cloud['cloud_id']) cloud_moment = cloud_cube.moment(0) cloud_cube = 0.0 fraction = (cloud_moment.value/moment.value) planemask = skm.binary_closing(fraction > 0,selem=skm.disk(3)) fraction = np.nanmean(fraction) rind = (skm.binary_dilation(planemask,selem=skm.disk(6)) ^\ skm.binary_dilation(planemask,selem=skm.disk(3)))*\ np.isfinite(irmap2) if np.any(rind): rindvals = irmap2[rind] clipval = 4*np.percentile(rindvals,15.87)-\ 3*(np.percentile(rindvals,2.28)) rind *= irmap2 <= clipval yv,xv = np.where(rind) x0,y0 = np.median(xv),np.median(yv) dataz = np.c_[np.ones(xv.size), xv-x0, yv-y0] try: lsqcoeffs,_,_,_ = np.linalg.lstsq(dataz,irmap2[rind]) outputs = lsq(myplane,np.r_[lsqcoeffs,0], args=(xv-x0, yv-y0, irmap2[rind]), loss = 'soft_l1') coeffs = outputs.x yhit,xhit = np.where(planemask) bg = coeffs[0]+coeffs[1]*(xhit-x0)+\ coeffs[2]*(yhit-y0)+coeffs[3]*(yhit-y0)*(xhit-x0) # I am sitcking a 6e11 in here as the frequency of # the 500 microns bgavg = np.sum(fraction*bg)/6e11 bglum = bgavg*cloud['distance']**2*\ 3.086e18**2*np.pi*4/3.84e33 except ValueError: import pdb; pdb.set_trace() ir_flux = np.nansum(fraction*(irmap[planemask]))/6e11 ir_lum = ir_flux * cloud['distance']**2*\ 3.086e18**2*np.pi*4/3.84e33 ir_flux2 = np.nansum(fraction*irmap2[planemask])/6e11 ir_lum2 = ir_flux2 * cloud['distance']**2*\ 3.086e18**2*np.pi*4/3.84e33 cloud['ir_flux'] = ir_flux2 cloud['ir_luminosity'] = ir_lum2 cloud['ir_flux_short'] = ir_flux cloud['ir_lum_short'] = ir_lum cloud['bg_flux'] = bgavg cloud['bg_lum'] = bglum print(cloud['cloud_id'],ir_flux,ir_lum,ir_lum2,bglum) cat.write('output_catalog.fits',overwrite=True) ctrr+=1 if ctrr==20: return(cat) # if cloud['volume_pc2_kms']>1e2: # import pdb; pdb.set_trace() return(cat)
def lsq_11fe_color_fit(SN, EBV, rv_guess, filters, zp, N_DAYS): ''' This is a helper function to fit an RV to a certain supernova (SN), using an artificially reddened 11fe as a template. EBV is fixed and then 'filters' are the filters you want to include in the fit (with 'zp' as zero-points dictionary). The fitter will only use phases from the comparison supernova which are no more than N_DAYS from the nearest 11fe phase. SN needs to be in the form (populated by the phases and corresponding V-X colors): SN = {'B':[(-6.5, -1.0601018152416941), (-3.5, -1.0864032119751066), (-1.4, -1.08253418576437), (6.5, -1.2348233700462039), (8.5, -1.2831185823642457), (11.5, -1.3001372408790797), ... lsq_out, valid_phases = lsq_11fe_color_fit(SN, -1.29, 1.3, 'UBRI', ZP_CACHE, 2.0) This function outputs the output of SciPy's least-square fit, and also a dictionary with the phases used in the fit, respective to each band: lsq_out = (array([ 1.3776701]), 1) valid_phases = {'B': array([ -3.5, -0.7, 1.1, 5.2, 6.9, 7.9, 8.7, 16.7, 20.3, 28.1]), 'I': array([ -3.5, -2.5, -0.7, 1.1, 3.4, 5.2, 6.9, 7.9, 8.7, 16.7, 20.3, 28.1]), 'R': array([ -3.5, -2.5, -0.7, 1.1, 3.4, 5.2, 6.9, 7.9, 8.7, 16.7, 20.3, 28.1]), 'U': array([ -3.5, -2.5, 3.4, 5.2, 6.9, 7.9, 8.7, 16.7, 20.3, 28.1])} ''' from scipy.optimize import leastsq as lsq def lsq_func(Y): print Y RV = Y[0] sn11fe = l.get_11fe('fm', ebv=EBV, rv=RV) sn11fe_colors = np.array([]) sncomp_colors = np.array([]) for f in filters: # get phases from comparison supernova band_data = SN[f] phases = [t[0] for t in band_data] # interpolate 11fe at 12cu phases which are no more than N_DAYS # away from the nearest 11fe phase valid = find_valid([t[0] for t in sn11fe], phases, N_DAYS) valid_phases[f] = np.array(phases)[valid] sn11fe_int = l.interpolate_spectra(valid_phases[f], sn11fe) # calculate band magnitudes and then colors for interpolated 11fe temp, sn11fe_vmags = calc_v_band_mags(sn11fe_int, zp) sn11fe_bandcolors = calc_colors(f, sn11fe_int, sn11fe_vmags, zp) sn11fe_phases = [t[0] for t in sn11fe_bandcolors] sn11fe_bandcolors = [t[1] for t in sn11fe_bandcolors] sn11fe_colors = np.concatenate(( sn11fe_colors, sn11fe_bandcolors )) # get comparison supernova's color at valid phases sncomp_bandcolors = [t[1] for t in band_data if t[0] in sn11fe_phases] sncomp_colors = np.concatenate(( sncomp_colors, sncomp_bandcolors )) return sncomp_colors - sn11fe_colors Y = np.array([rv_guess]) valid_phases = {} return lsq(lsq_func, Y), valid_phases
ratio = FFT_P/FFT_H ratio_freq = numpy.fft.fftfreq(len(sig_H), d=d) #phase = numpy.arctan(ratio) phase = numpy.arctan2(numpy.imag(ratio), numpy.real(ratio)) freqs = scipy.where( (ratio_freq > 0) & (ratio_freq < 20.0))[0] guess = [-1.0] def calc_error(g, xvals, yvals): e = 0.0; for x, y in zip(xvals, yvals): e+= (x*g[0] - y)**2.0 return e fit = lsq(calc_error, guess, args=(ratio_freq[freqs], phase[freqs])) line = [] for i in ratio_freq[freqs]: line.append(i*fit[0][0]) ax.plot(ratio_freq[freqs], phase[freqs]) ax.plot(ratio_freq[freqs], line) latency = fit[0][0]/(2.0*3.14159) print "Latency : ", numpy.abs(latency) #ax.plot(ratio_freq, numpy.abs(ratio)) fig.show()
def features_training(num_images=1): """ Function to analyse the features of the TRAINING set of the autoencoder """ norm_coef = [] losses_focus, peaks_focus, mins_focus = [], [], [] losses_defocus, peaks_defocus, mins_defocus = [], [], [] # ============================================================================================================ # ### Light Loss - see how the Low Orders modify the total intensity for j in range(N_ext): low_orders = ae_coefs[j, :N_low] norm_coef.append(np.linalg.norm(low_orders)) input_focus = train_noisy[j, :N_crop**2].reshape((N_crop, N_crop)) output_focus = train_clean[j, :N_crop**2].reshape((N_crop, N_crop)) removed_features_focus = input_focus - output_focus loss_focus = np.sum(removed_features_focus) losses_focus.append(loss_focus) peaks_focus.append(np.max(removed_features_focus)) mins_focus.append(np.min(removed_features_focus)) input_defocus = train_noisy[j, N_crop**2:].reshape( (N_crop, N_crop)) output_defocus = train_clean[j, N_crop**2:].reshape( (N_crop, N_crop)) removed_features_defocus = input_defocus - output_defocus loss_defocus = np.sum(removed_features_defocus) losses_defocus.append(loss_defocus) peaks_defocus.append(np.max(removed_features_defocus)) mins_defocus.append(np.min(removed_features_defocus)) norm_coef = np.array(norm_coef) f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, sharey=True) # Focused PSF p_sort = np.argsort(peaks_focus) ax1.scatter(norm_coef[p_sort], np.sort(peaks_focus), color=cm.bwr( np.linspace(0.5 + np.min(peaks_focus), 1, N_ext)), s=4, label='Maxima') m_sort = np.argsort(mins_focus) ax1.scatter(norm_coef[m_sort], np.sort(mins_focus), color=cm.bwr(np.linspace(0, 0.5, N_ext)), s=4, label='Minima') loss_sort = np.argsort(losses_focus) ax1.legend(loc=2) leg = ax1.get_legend() leg.legendHandles[0].set_color('red') leg.legendHandles[1].set_color('blue') ax1.axhline(y=0.0, linestyle='--', color='black') ax1.set_title('Nominal PSF') ax1.set_ylabel(r'Light loss') ax1.set_ylim([-0.5, 0.5]) ax3.scatter(norm_coef[loss_sort], np.sort(losses_focus), color='black', s=3, label='Total') ax3.legend(loc=2) ax3.axhline(y=0.0, linestyle='--', color='black') ax3.set_xlabel(r'Norm of low orders $\Vert a_{low} \Vert$') ax3.set_ylabel(r'Light loss') # Defocused PSF p_sort = np.argsort(losses_defocus) ax2.scatter(norm_coef[p_sort], np.sort(peaks_defocus), color=cm.bwr( np.linspace(0.5 + np.min(peaks_defocus), 1, N_ext)), s=4, label='Maxima') m_sort = np.argsort(mins_defocus) ax2.scatter(norm_coef[m_sort], np.sort(mins_defocus), color=cm.bwr(np.linspace(0, 0.5, N_ext)), s=4, label='Minima') loss_sort = np.argsort(losses_defocus) ax2.legend(loc=2) leg = ax2.get_legend() leg.legendHandles[0].set_color('red') leg.legendHandles[1].set_color('blue') ax2.axhline(y=0.0, linestyle='--', color='black') ax2.set_title('Defocused PSF') ax4.scatter(norm_coef[loss_sort], np.sort(losses_defocus), color='black', s=3, label='Total') ax4.legend(loc=2) ax4.axhline(y=0.0, linestyle='--', color='black') ax4.set_xlabel(r'Norm of low orders $\Vert a_{low} \Vert$') # ============================================================================================================ # ### PCA analysis of the removed features # Focused PSF N_comp = N_low removed_features = train_noisy[:, :N_crop**2] - train_clean[:, :N_crop **2] pca = PCA(n_components=N_comp) pca.fit(X=removed_features) components = pca.components_.reshape((N_comp, N_crop, N_crop)) variance_ratio = pca.explained_variance_ratio_ total_variance = np.sum(variance_ratio) plt.figure() for i in range(N_comp): ax = plt.subplot(2, N_comp, i + 1) plt.imshow(components[i], cmap='seismic', origin='lower') ax.set_title(r'PCA #%d [$\sigma^2_r=%.2f/%.2f$]' % (i + 1, variance_ratio[i], total_variance)) plt.colorbar(orientation="horizontal") cmin = min(components[i].min(), -components[i].max()) plt.clim(cmin, -cmin) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) removed_features_defocus = train_noisy[:, N_crop** 2:] - train_clean[:, N_crop**2:] pca_defocus = PCA(n_components=N_comp) pca_defocus.fit(X=removed_features_defocus) components_defocus = pca_defocus.components_.reshape( (N_comp, N_crop, N_crop)) variance_ratio_defocus = pca_defocus.explained_variance_ratio_ total_variance_defocus = np.sum(variance_ratio_defocus) for i in range(N_comp): ax = plt.subplot(2, N_comp, i + 1 + N_comp) plt.imshow(components_defocus[i], cmap='seismic', origin='lower') ax.set_title( r'PCA #%d [$\sigma^2_r=%.2f/%.2f$]' % (i + 1, variance_ratio_defocus[i], total_variance_defocus)) plt.colorbar(orientation="horizontal") cmin = min(components_defocus[i].min(), -components_defocus[i].max()) plt.clim(cmin, -cmin) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) ### Least Squares fit of the removed features def residuals_lsq(x, image_data, pca_components): model_image = np.dot(x, pca_components) return image_data - model_image # ============================================================================================================ # ### Compare the removed features to the true difference random_images = np.random.randint(train_noisy.shape[0], size=num_images) for j in random_images: im = removed_features[j] res_lsq = lsq(fun=residuals_lsq, x0=np.zeros(N_comp), args=(im, pca.components_)) x_fit = res_lsq['x'] im_fit = (np.dot(x_fit, pca.components_)).reshape((N_crop, N_crop)) im = im.reshape((N_crop, N_crop)) vmin_im = min(im.min(), -im.max()) vmin_fit = min(im_fit.min(), -im_fit.max()) vmin = min(vmin_im, vmin_fit) error = np.sum(np.abs(im_fit - im)) / np.sum(np.abs(im)) plt.figure() cmap = 'seismic' ax1 = plt.subplot(1, 3, 1) plt.imshow(im, cmap=cmap) plt.colorbar(orientation="horizontal") plt.clim(vmin, -vmin) ax1.get_xaxis().set_visible(False) ax1.get_yaxis().set_visible(False) ax1.set_title( r'Removed features: $PSF(\Phi_{low} + \Phi_{high}) - PSF(\Phi_{high})$' ) ax2 = plt.subplot(1, 3, 2) plt.imshow(im_fit, cmap=cmap) plt.colorbar(orientation="horizontal") plt.clim(vmin, -vmin) ax2.get_xaxis().set_visible(False) ax2.get_yaxis().set_visible(False) ax2.set_title(r'Least-Squares fit from PCA: $x_{lsq} \cdot PCA$') res = im_fit - im ax3 = plt.subplot(1, 3, 3) plt.imshow(res, cmap='bwr') plt.colorbar(orientation="horizontal") min_res = min(res.min(), -res.max()) plt.clim(min_res, -min_res) ax3.get_xaxis().set_visible(False) ax3.get_yaxis().set_visible(False) ax3.set_title(r'Residuals ($\epsilon = %.2f$)' % error) # ============================================================================================================ # ### Show some examples of the true features random_images = np.random.choice(train_noisy.shape[0], size=48, replace=False) print(random_images) plt.figure() for i, img_j in enumerate(random_images): ax = plt.subplot(6, 8, i + 1) im = removed_features[img_j].reshape((N_crop, N_crop)) plt.imshow(im, cmap='seismic') min_im = min(im.min(), -im.max()) plt.clim(min_im, -min_im) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) plt.show() return pca
def BIC(): wcsfile = "/data2/nova/BACKUP-jobs/00000046/wcs.fits" folder = os.path.dirname(wcsfile) pub_name = folder.split("/")[-1] # to save on the interwebs rdlsfiles = glob("./rdls*fits") log = folder + "/job.log" words = open(log).readlines()[3].split() print words imagef_ind = words.index("--image") + 1 # wrong filename for the image badfile = words[imagef_ind] getpath = badfile.split("/")[-4:] goodfile = "/home/nova/BACKUP-data/{0}/{1}/{2}/{3}".format(*tuple(getpath)) '''Now we have the right filename for the original.''' try: fits = pyfits.open(goodfile) picture = fits[0].data if type(picture) != np.ndarray: picture = fits[1].data except (IOError): picture = mpimg.imread(goodfile)[::-1] # let's just work on one of the colours: pic = picture[:, :, 0].astype("float64") stdev = MAGIC_NO * MAD(pic) dim = pic.shape print "dimensions are ", dim ext = 0 wcs = tractor.FitsWcs(Tan(wcsfile, ext)) # pick the ra,dec of a star to check cat_srcs = [] for rdfile in rdlsfiles: rdls = pyfits.open(rdfile) radec = rdls[1].data for ra in radec: cat_srcs.append(ra) for run, src in enumerate(cat_srcs): print "###################################" print "######## RUN NO. ", run, "#########" print "###################################" ra, dec = src t_radec = tractor.RaDecPos(ra, dec) print "RaDecPos obj: ", t_radec print wcs.positionToPixel(t_radec) src_xy = np.asarray(wcs.wcs.radec2pixelxy(ra, dec)) print src_xy cent = np.rint(src_xy).astype("int") print "centre:" print cent[0] print cent[1] if cent[0] < 0 or cent[0] >= dim[1] or cent[1] < 0 or cent[1] >= dim[0]: print "rdls not correct, out of image" continue # let's just switch the centre values and see... starr = pic[cent[1] - 7:cent[1] + 8, cent[0] - 7:cent[0] + 8] print "starr is ", starr.shape if starr.shape == (sq, sq): # now try to fit to sky model med = np.median(starr) peak = np.amax(starr) print "median of starr: ", med print "max of starr: ", peak """guess where gaussian is centred""" x0, y0 = (float(pos[0]) for pos in np.where(starr == peak)) """ Define plotstuff object - 0th term is sky only model - 1st term is sky and psf - 2nd is above plus roaming psf """ skyp = colxns.namedtuple('skyp', 'z0 mh mv') # skypsf = colxns.namedtuple("skypsf", "z0 mh mw varx vary cor") # roamp = colxns.namedtuple("roamp", "z0 mh mv w varx vary cor x y") ciorc = colxns.namedtuple("ciorc", "z0 mh mv w var") rokirk = colxns.namedtuple("rokirk", "z0 mh mv w var x y") x0, y0 = 7., 7. # try with the centre this time kwargs1 = dict(z0=med, mh=0., mv=0.) kwargs2 = dict(kwargs1, w=(peak - med), var=3.) kwargs3 = dict(kwargs2, x=x0, y=y0) # kwargs3 = kwargs2.copy() # kwargs3["roam"] = True class RunParams(): pass stuff = [RunParams(), RunParams(), RunParams()] stuff[0].tup, stuff[0].kws, stuff[0].fun = skyp, kwargs1, sky_mod stuff[1].tup, stuff[1].kws, stuff[1].fun = ciorc, kwargs2, ciorcal stuff[2].tup, stuff[2].kws, stuff[2].fun = rokirk, kwargs3, roam_ciorc # and start up the figure for plotting fig = plt.figure() gs = gridspec.GridSpec(3, 3) axlist = [] head = np.percentile(starr, 99.9) toe = np.percentile(starr, 0.1) for no, plst in enumerate(stuff): """ and here we do the fitting and plotting, one row for each model """ if no == 0: print "SKY ONLY" elif no == 1: print "SKY + PSF" elif no == 2: print "ROAMING PSF" # first the real image axlist.append(plt.subplot(gs[no, 0])) # axlist[no * 3].set_axis_off() axlist[no * 3].set_xticks([]) axlist[no * 3].set_yticks([]) if no == 0: axlist[0].set_title(r"Data") axlist[no * 3].imshow(starr, cmap=plt.cm.gray, interpolation="nearest", vmin=toe, vmax=head) if no == 0: axlist[0].set_ylabel(r"sky only") elif no == 1: axlist[3 * no].set_ylabel(r"sky + psf") elif no == 2: axlist[3 * no].set_ylabel(r"sky + roaming psf") # now fit and plot the model param_guess = plst.tup(**plst.kws) xdata = np.ones(225) result = lsq(residuals, param_guess, args=(plst.fun, xdata, starr)) solve_params = result[0] soln = plst.tup(*solve_params) print "optimal params are: ", soln xdata = np.ones(225) bestsky = plst.fun(soln, xdata) if no == 1: """ For the roaming psf, let's initialise with solve from static. """ stuff[2].kws['z0'] = soln.z0 stuff[2].kws['mh'] = soln.mh stuff[2].kws['mv'] = soln.mv stuff[2].kws['w'] = soln.w stuff[2].kws['var'] = soln.var axlist.append(plt.subplot(gs[no, 1])) # axlist[no * 3 + 1].set_axis_off() axlist[no * 3 + 1].set_xticks([]) axlist[no * 3 + 1].set_yticks([]) if no == 0: axlist[1].set_title(r"Model") axlist[no * 3 + 1].imshow(bestsky, cmap=plt.cm.gray, interpolation="nearest", vmin=toe, vmax=head) # and finally the chi image chimg = chi_img(starr, bestsky, stdev) minchi, maxchi, maxchisq, totchisq = chisq_calc(starr, bestsky, stdev) print "min(chi) is ", minchi print "max(chi) is ", maxchi print "max(chi ** 2) is ", maxchisq print "total(chi ** 2) is ", totchisq axlist.append(plt.subplot(gs[no, 2])) # axlist[no * 3 + 2].set_axis_off() axlist[no * 3 + 2].set_xticks([]) axlist[no * 3 + 2].set_yticks([]) if no == 0: axlist[2].set_title(r"Chi Image") axlist[no * 3 + 2].imshow(chimg, cmap=plt.cm.gray, interpolation="nearest", vmin=-5., vmax=5.) axlist[no * 3 + 2].yaxis.set_label_position("right") axlist[no * 3 + 2].set_ylabel(r"$\chi^{2} = %s$" % str(totchisq)) # for ax in axlist: # for t in ax.xaxis.get_ticklines(): # t.set_visible(False) plt.tight_layout() fig.savefig("{0}/{1}.png".format(webname, run)) else: print "out of range" continue
def fitEphem(myLoc, T0, period, simple=False): """ Takes eclipse time data from a file, and fits ephemeris parameters (T0, period) to them from an initial guess. There are two fitting options available: simple and complex. simple: Fits a simple linear regression fit to the data. Report the minimum chisq params complex: Uses an MCMC chain and a gaussian process to fit the data. Also considers that the reported error is not accurate, and allows data from different sources to have their error bars scaled to get a chisq closer to unity. This is far more reliable and accurate, but takes significantly longer to run. Arguments: ---------- myLoc: str Tells the function where to look for both logfiles and prior eclipse data T0: float Initial guess for T0 period: float Initial guess for the period simple: bool If true, use a linear regression fit on the data. If false, use an MCMC chain. Returns: -------- T0: float Best T0 period: float Best period """ printer("\n\n--- Fitting ephemeris to data ---") # Read in the eclipsetimes.txt file fname = path.join(myLoc, 'EPHEMERIS', 'eclipse_times.txt') if not path.isfile(fname): raise FileNotFoundError("I don't have any eclipse times to fit!") source_key, tl = read_ecl_file(fname) ### Fitting x = np.array([float(x[0]) for x in tl]) # cycle number y = np.array([float(x[1]) for x in tl]) # Time ey = np.array([float(x[2]) for x in tl]) # Error obsCodes = np.array([x[3] for x in tl]) # Data source params = [T0, period] def fitfunc(p,x): return p[0] + p[1]*x if simple: def errfunc(p,x,y,err): return (y-fitfunc(p,x)) / err out = lsq(errfunc,params,args=(x,y,ey),full_output=1) pfinal = out[0] covar = out[1] P, P_err = pfinal[1], np.sqrt(covar[1][1]) T0, T0_err = pfinal[0], np.sqrt(covar[0][0]) chisq = (y - fitfunc(pfinal, x))**2 / (ey**2) chisq = sum(chisq) / (len(chisq)-2) resy = y - fitfunc(pfinal, x) else: # Get the number of sources, which will each have their errors scaled differently numObs = len(source_key) # Set up the MCMC chain. npars = 2+numObs # The parameters are the T0, P, and the error scale of each source nwalkers = max(16,4*(2+numObs)) # The number of walklers wants to be at least 16, # or enough to sample our parameter space properly # Construct a list of parameters for the model, i.e. T0, P, and the error scale factor of each source nameList = ['T0','P'] for i, key in enumerate(source_key): # add error factor for each source params.append(1.0) sourceName = source_key[key].replace(' ', '_') nameList.append('obs_err_{:s}'.format(sourceName)) guessP = np.array(params) # Initialise the sampler p0 = emcee.utils.sample_ball(guessP,0.01*guessP,size=nwalkers) sampler = emcee.EnsembleSampler(nwalkers,npars,ln_prob,args=[x,y,ey,obsCodes]) #burn in nburn=5000 pos, prob, state = run_burnin(sampler, p0,nburn) #production sampler.reset() nprod = 2000 chain_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_chain.txt") sampler = run_mcmc_save(sampler, pos, nprod, state, chain_fname) chain = flatchain(sampler.chain, npars, thin=1) # Gather and report the best values bestPars = [] for i in range(npars): par = chain[:,i] lolim,best,uplim = np.percentile(par,[16,50,84]) # printer("{:>20s} = {:.10f} +{:.10f} -{:.10f}".format(nameList[i],best,uplim-best,best-lolim)) bestPars.append(best) if nameList[i] == 'T0': printer("Old T0: {:.8f}".format(T0)) T0 = best T0_err = uplim-lolim printer("New T0: {:.8f}+/-{:.8f}\n".format(T0, T0_err)) if nameList[i] == 'P': printer("Old P: {:.8f}".format(period)) P = best P_err = uplim-lolim printer("New P: {:.8f}+/-{:.8f}\n".format(P, P_err)) fig = thumbPlot(chain,nameList) corner_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_corner_plot.pdf") fig.savefig(corner_fname) printer("Saved a corner plot of the MCMC fit (including error scaling factors) to:\n-> {}".format(corner_fname)) plt.close('all') resy = 86400.0*(y-model(bestPars,x)) errs = ey.copy() # scale errors by amount set by observatory code emul_factors = np.array(bestPars[2:]) multipliers = emul_factors[obsCodes-1] errs = errs*multipliers ey = 86400.0*errs chisq = np.sum(resy**2.0/ey**2.0) printer("\nChisq = {:.1f}, with {:d} degrees of freedom".format(chisq, int(x.size - 2))) ### Reporting printer("Got a T0 of {:>5.15f}+/-{:<.2e}".format(T0, T0_err)) printer("Got a period of {:>5.15f}+/-{:<.2e}".format(P, P_err)) printer("This fit had a reduced chisq value of {:.3f}".format(chisq)) printer('') printer("Source | (Obs) - (Calc), sec | Cycle Number") for t in tl: dT = fitfunc([T0, P], t[0]) - t[1] dT *= 24*60*60 printer(" {:<14s} | {:>20.4f} | {:d}".format(source_key[str(int(t[3]))], dT , int(t[0]) )) # Each error code wants to be a different color codes = set(obsCodes) # set() strips duplicates codes = list(codes) # list() allows indexing CB_color_cycle = ['black', '#ff7f00', '#4daf4a', '#377eb8', '#a65628', '#984ea3', '#999999', '#e41a1c', '#dede00'] colors = [] for code in obsCodes: i = codes.index(code) colors.append(CB_color_cycle[i]) plt.errorbar(x,resy,yerr=ey,marker='',ecolor='k', linestyle='', zorder=1) plt.scatter(x, resy, c=colors, marker='o', zorder=2) plt.axhline(ls='--',color='k', zorder=1) plt.xlabel('Cycle No.') plt.ylabel('O-C (s)') scatter_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_scatter.pdf") ephem_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_fit.txt") with open(ephem_fname, 'w') as f: f.write("Got a T0 of {:>5.15f}+/-{:<.2e}\n".format(T0, T0_err)) f.write("Got a period of {:>5.15f}+/-{:<.2e}\n".format(P, P_err)) f.write("This fit had a reduced chisq value of {:.3f}\n".format(chisq)) f.write('\n') f.write("Source | (Obs) - (Calc), sec | Cycle Number\n") for t in tl: dT = fitfunc([T0, P], t[0]) - t[1] dT *= 24*60*60 f.write(" {:<14s} | {:>20.4f} | {:d}\n".format(source_key[str(int(t[3]))], dT , int(t[0]) )) plt.savefig(scatter_fname) plt.show() printer("") return T0, P
# An arbitrary list for the scale of the radius. according to every bin. rlist = [.6, .8, 1, 1.2, 1.4, 1.6, 1.8, 2] # Create gtrue, expected true value of gamma. gtrue = [] for z in rlist: gtrue.append(gf(.35, z)) p0 = [.5] # initial parameter guess # This is an attempt to fit the data using the perceived function gf. p0 is the initial guess # and the first argument in err. args will consist of any other arguments that might be necessary # in the gf function. fit = lsq(err, p0, args=(glist, rlist)) print(fit[0]) # Plotting stuff """ax = plot.figure().add_subplot(1,1,1) for w in ax.plot([4,6, 8, 9.5],glist, 'bo',) ax.set_xlabel('Distance from Mass r (Megaparsec)') ax.set_ylabel('Shear') ax.set_xlim(0, 10) ax.set_ylim(0, 25) """ plt.plot(rlist, glist, 'ro', rlist, peval(rlist, fit[0]), rlist, gtrue) plt.legend(['Measured', 'Least Squares Fit', 'True']) plt.title('Singular Isothermal Sphere \n Least-squares fit to Measured Shear')
def scale(data,model): guess = numpy.abs(data.mean()) / numpy.abs(model.mean()) errfunc = lambda scale, data, model: data-scale*model out = lsq(errfunc,guess,args=(data,model),full_output=1) return out[0]
def fit(self): ydata = self.yraw / self.yraw[0] x0 = np.array([1,1]) R = lsq(_residuals, x0, args=(self.x, self.model, ydata)) self.fitted_pars = return R.x
pl.plot(x1, yplot, color = 'black', linewidth = 3, label = 'sklearn') #fit made with function minimization def line(x, m, q): return m*x + q def residual(p,x,y): return y - line(x,*p) guess = (3,3) x2 = np.array([cleanData.logtime]) y2 = np.array([cleanData.mag]) xfit = x2[0,:] yfit = y2[0,:] m, q = np.polyfit(xfit,yfit,1) print("Fit parameters obtained from polyfit: m,q =" + str(m) +" " + str(q)) pl.plot(x1,line(x1,m,q), color = 'red', linewidth = 1, label = 'polyfit') pl.legend() msq, qsq = lsq(residual,guess,args = (xfit,yfit))
mean_data = np.mean(y_data) ss_total = np.sum((y_data - mean_data)**2) ss_res = np.sum((quadratic_residual(x_fit, y_data, aberr_coef))**2) R2 = 1 - ss_res / ss_total return R2 # Aberrations that behave like a parabola parab_indices = [0, 1, 2, 4] parab_fit = [] parab_shift = [] for k in parab_indices: y_data = peaks[k] x_solve = lsq(quadratic_residual, x0=[-0.1, 0.0], args=( y_data, coef, )) a_fit, x_shift = x_solve['x'] print(a_fit, x_shift) parab_fit.append(a_fit) parab_shift.append(x_shift) R2 = r2([a_fit, x_shift], y_data, coef) plt.figure() lab = label = r'Fit: $f(x)=1 - %.3f (x + %.3f)^2$' % (-a_fit, x_shift) plt.scatter(coef, y_data, label='Data', s=10) plt.plot(coef, quadratic([a_fit, x_shift], coef),
def scale(data, model): guess = numpy.abs(data.mean()) / numpy.abs(model.mean()) errfunc = lambda scale, data, model: data - scale * model out = lsq(errfunc, guess, args=(data, model), full_output=1) return out[0]