def para_fit(X):
"""Function to calculate results in parallel
Input is the r matrix, after normalization by the standard deviations, R/R_std,
such that the data is normalized to a standard deviation of 1, upper and
lower bounds, the desired confidence interval (.95, for example), and finally
the number of Monte-Carlo repetitions to perform (if set to 0, performs
linear propagation-of-erro)
"""
Y=lsq(X[0],X[1],bounds=(X[2],X[3]))
rho=Y['x']
Rc=Y['fun']+X[1]
if X[5]==0:
std=np.sqrt(np.sum(np.linalg.pinv(X[0])**2,axis=1))
nstd=norm.ppf(1/2+X[4]/2)
u=nstd*std
l=nstd*std
else:
Y1=list()
nmc=max([X[5],np.ceil(2/X[4])])
for k in range(0,X[5]):
Y0=lsq(X[0],Rc+np.random.normal(size=X[1].shape))
Y1.append(Y0['x'])
std=np.std(Y1,axis=0)
Y1sort=np.sort(Y1,axis=0)
in_l=np.round(nmc*(1/2-X[4]/2))
in_u=np.round(nmc*(1/2+X[4]/2))
l=rho-Y1sort[int(in_l)]
u=Y1sort[int(in_u)]-rho
return rho,std,l,u
def arma_est_bip_m(x, p, q, beta_hat_s, a_sc_final):
N = len(x)
x = np.array(x)
F_mm = lambda beta: sp.sqrt(1 / (N - p) * sp.sum(
rsp.muler_rho2(rsp.arma_resid(x / a_sc_final, beta, p, q))))
F_bip_mm = lambda beta: sp.sqrt(1 / (N - p) * sp.sum(
rsp.muler_rho2(rsp.bip_resid(x / a_sc_final, beta, p, q))))
beta_arma_mm = lsq(F_mm,
beta_hat_s,
xtol=5 * 1e-5,
ftol=5 * 1e-5,
method='lm')[0]
beta_bip_mm = lsq(F_bip_mm,
beta_hat_s,
xtol=5 * 1e-5,
ftol=5 * 1e-5,
method='lm')[0]
a_rho2_mm = 1 / (N - p) * np.sum(
muler_rho2(rsp.arma_resid(x / a_sc_final, beta_arma_mm, p, q)))
a_bip_rho2_mm = 1 / (N - p) * np.sum(
muler_rho2(rsp.bip_resid(x / a_sc_final, beta_bip_mm, p, q)))
beta_hat = beta_arma_mm if a_rho2_mm < a_bip_rho2 else beta_bip_mm
# Output the results
phi_bip_mm = -beta_hat[:p] if p > 0 else []
theta_bip_mm = -beta_hat[p:] if q > 0 else []
return phi_bip_mm, theta_bip_mm
def dist_opt(X):
"""
Optimizes a distribution that yields detector responses, R, where the
distribution is required to be positive, and have an integral of 1-S2
Ropt,dist=dist_opt((R,R_std,rhoz,S2,dz))
Note- intput is via tuple
"""
R,R_std,rhoz,S2=X
total=np.atleast_1d(1-S2)
"""Later, we may need to play with the weighting here- at the moment, we
fit to having a sum of 1, but in fact it is not forced....it should be
"""
ntc=rhoz.shape[1]
rhoz=np.concatenate((rhoz/np.repeat(np.atleast_2d(R_std).T,ntc,axis=1),
np.atleast_2d(np.ones(ntc))),axis=0)
Rin=np.concatenate((R/R_std,total))
dist=0
while np.abs(np.sum(dist)-total)>1e-3: #This is a check to see that the sum condition has been satisfied
dist=lsq(rhoz,Rin,bounds=(0,1))['x']
Rin[-1]=Rin[-1]*10
rhoz[-1]=rhoz[-1]*10
Ropt=np.dot(rhoz[:-1],dist)*R_std
return Ropt,dist
def baselineWithAmmonia(y,
v,
baselineIndex,
freqthrow=4.11 * u.MHz,
v0=8.5,
sigmav=1.0 * u.km / u.s,
line='oneone',
blorder=5,
noiserms=None):
x = np.linspace(-1, 1, len(y))
chthrow = (freqthrow.to(u.Hz).value / acons.freq_dict[line] * 299792.458 /
np.abs(v[0] - v[1]))
chthrow = (np.round(chthrow)).astype(np.int)
if noiserms is None:
noiserms = mad1d((y - np.roll(y, -2))[baselineIndex])
opts = lsq(
ammoniaLoss,
np.r_[[
np.nanmax(y[baselineIndex]), v[np.nanargmax(y[baselineIndex])], 1.0
],
np.zeros(blorder + 1)],
args=(y[baselineIndex], x[baselineIndex], v[baselineIndex], noiserms),
kwargs={'chthrow': chthrow},
loss='soft_l1')
return y - legendre.legval(x, opts.x[3:])
def get_chi(blorder_max, ydata, xdata, blindex, noise):
"""
Returns the best-fit Legendre polynomial to an input spectra,
based on a comparison of the reduced chi-squared values for each order
of the polynomial.
blorder_max = maximum order of polynomial to fit
ydata = spectrum y values
xdata = spectrum x values
blindex = the indices of the spectra to include in baseline fitting
noise = rms noise of spectrum"""
opts = lsq(legendreLoss, np.zeros(blorder_max + 1), args=(ydata[blindex],
xdata[blindex],
noise), loss='arctan')
chis = [] # first entry is order=1, second is order=2, ..., up to order=blorder
ymods = []
# Loop through each order of polynomial and create model baseline
# and calculate that model's chi-squared values
for i in np.arange(blorder_max)+1:
# i is the polynomial order
ymod = legendre.legval(xdata, np.array(opts.x)[0:i+1]) #+1 to get last term
ymods.append(ymod)
chi = redchisqg(ydata,ymod,deg=i+1)
chis.append(chi)
# Find the model with the lowest chi-squared value
find_low = np.where(np.array(chis)==min(chis))
low_model = np.array(ymods)[find_low]
return ymod
def robustBaseline(y, baselineIndex, blorder=1, noiserms=None):
x = np.linspace(-1, 1, len(y))
if noiserms is None:
noiserms = mad1d((y - np.roll(y, -2))[baselineIndex])
opts = lsq(legendreLoss, np.zeros(blorder + 1), args=(y[baselineIndex],
x[baselineIndex],
noiserms),
loss='arctan')
return y - legendre.legval(x, opts.x)
def alignTrajs(p_gt_C, p_O_C, initial_T_gt_O):
state = np.hstack([np.zeros([1, 3]), initial_T_gt_O[0:3, 3:4].T]).reshape([-1])
bound_alignE = functools.partial(alignError, initial_T_gt_O, p_O_C, p_gt_C)
optim_state = lsq(bound_alignE, state).x
print(optim_state)
errs = bound_alignE(optim_state).reshape([3, -1])
ate = np.sqrt(np.sum(np.square(errs), axis=0)).mean()
optim_T_gt_O = applyT(initial_T_gt_O, optim_state)
return [optim_T_gt_O, ate]
def findFlows(flowModel, flowMeasurements):
flowGuess = 0.1 + 5.0 * ranf(
flowModel.nFlows + flowModel.nsched) # includes timeoffsets
result = lsq(optFunc,
flowGuess,
bounds=(flowModel.lowerBounds, flowModel.upperBounds),
loss='huber',
args=(flowMeasurements, flowModel))
plotResids = formatResids(flowMeasurements, result.fun)
return result
def robustBaseline(y, v, baselineIndex, blorder=1, noiserms=None):
x = np.linspace(-1, 1, len(y))
if noiserms is None:
noiserms = mad1d((y - np.roll(y, -2))[baselineIndex])
opts = lsq(legendreLoss,
np.zeros(blorder + 1),
args=(y[baselineIndex], x[baselineIndex], noiserms),
loss='arctan')
return y - legendre.legval(x, opts.x)
def _fit_model(self, data, bkgd_shape=None, lsqKwargs=None):
'''
Function that performs curve fitting based on use of
scipy.optimize.curve_fit
Parameters
----------
data : ndarray
The data to be used in the model. Prepared by the function
MLLSmodel._prepare_model_data()
bkgd_shape : string or None
If not None, must be a string containing the name of one of
the available background shapes:
- power law
- log-linear
- double power law
lsqKwargs : dict
Keword arguments to be taken by scipy.optimize.curve_fit
Returns
-------
opt_cofs : ndarray
The optimised fit coefficients
cov : ndarray
The covariance matrix of the fit
'''
if lsqKwargs is None:
lsqKwargs = {}
y_obs = data[0, :]
x = data[1, :]
components = data[2:, :]
def background(t, coeffs):
if bkgd_shape == 'power law':
return coeffs[-2] * pow(t, coeffs[-1])
if bkgd_shape == 'log-linear':
return np.log(coeffs[-2] * pow(t, coeffs[-1]))
if bkgd_shape == 'double power law':
return coeffs[-4] * pow(t, coeffs[-3]) + coeffs[-2] * pow(
t, coeffs[-1])
def model(t, *coeffs):
y = 0.0
for i in range(components.shape[0]):
y += coeffs[i] * components[i, :]
if bkgd_shape:
y += background(t, coeffs)
return y
opt_cofs, cov = lsq(model, x, y_obs, **lsqKwargs)
# self.last = [opt_cofs, cov]
return opt_cofs, cov
def polyfit2d(f, y, x, unc=None, order=1):
"""Fit a polynomial surface to 2D data.
Assumes the axes are independent of each other.
Evaluate the fit via: np.polyval(polyy, y) + np.polyval(polyx, x).
Parameters
----------
f: array
The array to fit.
y, x: array
The coordinates for each value of `f`.
unc: array, optional
The uncertainties on each value of `f`, or a single value for
all points. If `None`, then 1 is assumed.
order: int, optional
The polynomial order(in one dimension) of the fit.
Returns
-------
polyx, polyy: ndarray
The polynomial coefficients, in the same format as from
`np.polyfit`.
cov: ndarray
The covariance matrix.
v1.0.0 Written by Michael S. Kelley, UMD, Mar 2009
"""
from scipy.optimize import leastsq as lsq
# the fitting function
def chi(p, y, x, f, unc, order):
cy = p[:1+order]
cx = p[1+order:]
model = np.zeros(f.shape) + np.polyval(cy, y) + np.polyval(cx, x)
chi = (f - model) / unc
return chi
if unc is None:
unc = 1.0
# run the fit
guess = np.zeros((order + 1) * 2)
result = lsq(chi, guess, args=(y, x, f, unc, order), full_output=True)
fit = result[0]
cov = result[1]
cy = fit[:1+order]
cx = fit[1+order:]
return cx, cy, cov
def polyfit2d(f, y, x, unc=None, order=1):
"""Fit a polynomial surface to 2D data.
Assumes the axes are independent of each other.
Evaluate the fit via: np.polyval(polyy, y) + np.polyval(polyx, x).
Parameters
----------
f : array
The array to fit.
y, x : array
The coordinates for each value of `f`.
unc : array, optional
The uncertainties on each value of `f`, or a single value for
all points. If `None`, then 1 is assumed.
order : int, optional
The polynomial order (in one dimension) of the fit.
Returns
-------
polyx, polyy : ndarray
The polynomial coefficients, in the same format as from
`np.polyfit`.
cov : ndarray
The covariance matrix.
v1.0.0 Written by Michael S. Kelley, UMD, Mar 2009
"""
from scipy.optimize import leastsq as lsq
# the fitting function
def chi(p, y, x, f, unc, order):
cy = p[:1 + order]
cx = p[1 + order:]
model = np.zeros(f.shape) + np.polyval(cy, y) + np.polyval(cx, x)
chi = (f - model) / unc
return chi
if unc is None:
unc = 1.0
# run the fit
guess = np.zeros((order + 1) * 2)
result = lsq(chi, guess, args=(y, x, f, unc, order), full_output=True)
fit = result[0]
cov = result[1]
cy = fit[:1 + order]
cx = fit[1 + order:]
return cx, cy, cov
def analyzeLoop(self):
def expFun(k,x):
return k[0]+k[1]*exp(-x/k[2])
def expLSQ(k,x,y):
return (expFun(k,x)-y)**2
if self.infoDict['stimVar'].get()=='E1':
sv=0
else:
sv=1
if self.ITC.mode[sv]==VC:
vm=0
im=1
else:
vm=1
im=0
pre=self.params['pre']
dur=self.params['dur']
post=self.params['post']
reps=self.params['reps']
di=list()
for i in self.dataIn:
di.append(i[:-EXTRA])
#print 'L:%d vs %d' % (len(di[vm]),((pre+dur+post)/DT)*reps)
voltage=mean(di[vm].reshape((pre+dur+post)/DT,reps,order='F'),axis=1)
current=mean(di[im].reshape((pre+dur+post)/DT,reps,order='F'),axis=1)
y=current[round((pre+0.48)/DT):round((pre+dur-0.48)/DT)]
x=arange(len(y))*DT
k=lsq(expLSQ,array([0,1,10]),(x,y))
#print 'K:%2.0f,%2.0f,%2.0f' % (k[0][0],k[0][1],k[0][2])
I0=mean(current[round(0.48/DT):round((pre-0.48)/DT)])
V0=mean(voltage[round(0.48/DT):round((pre-0.48)/DT)])
Iss=mean(current[(pre+dur-3)/DT:(pre+dur-0.48)/DT]);
Iss=k[0][0]
Imx=expFun(k[0],-0.48)
Vss=mean(voltage[(pre+dur-3)/DT:(pre+dur-0.48)/DT])
Rseries=(Vss-V0)/(Imx-I0);
Rin=(Vss-V0)/(Iss-I0);
Rm=Rin-Rseries;
#charge=mean(exp_fun(k,-0.48:DT:dur-0.48))*dur/DT;
x=arange(dur/DT)*DT-0.48
charge=0
for i in x:
charge+=expFun(k[0],i)
Cm=charge/(Vss-V0);
self.analysisDict.update({'I0':I0,'V0':V0,'Iss':Iss,'Imx':Imx,'Vss':Vss,'Rseries':Rseries,'Rin':Rin,'Rm':Rm,'Cm':Cm,'charge':charge,'k':k[0],'current':current})
def arma_est_bip_mm(x,p,q):
bip_s_est = rsp.arma_est_bip_s(x,p,q)
beta_hat_s = np.array([*bip_s_est['ar_coeffs'], *bip_s_est['ma_coeffs']])
N = len(x)
F_mm = lambda beta: 1/(N-p)*rsp.muler_rho2(rsp.arma_s_resid(x,beta,p,q)[0])
F_bip_mm = lambda beta: 1/(N-p)*rsp.muler_rho2(rsp.bip_s_resid(x,beta,p,q)[0])
beta_arma_mm = lsq(F_mm, -beta_hat_s,xtol=5*1e-7,ftol=5*1e-7,method='lm')['x']
beta_bip_mm = lsq(F_bip_mm, -beta_hat_s,xtol=5*1e-7,ftol=5*1e-7,method='lm')['x']
a = rsp.arma_s_resid(x, beta_arma_mm, p, q)[0]
a_sc = rsp.m_scale(a) # innovations m-scale for ARMA model
a_bip = rsp.bip_s_resid(x, beta_bip_mm, p, q)[0]
a_bip_sc = rsp.m_scale(a_bip) # innovations m-scale for BIP-ARMA model
# final parameter estimate uses the model that provides smallest m_scale
beta_hat = beta_arma_mm if a_sc<a_bip_sc else beta_bip_mm
# final m-scale
a_m_sc = min(a_sc, a_bip_sc)
# Output the results
results = {'ar_coeffs': -beta_hat[:p],
'ma_coeffs': -beta_hat[p:],
'inno_scale': a_m_sc,
'cleaned_signal': bip_s_est['cleaned_signal'],
'ar_coeffs_init': bip_s_est['ar_coeffs'],
'ma_coeffs_init': bip_s_est['ma_coeffs']}
return results
def model_point(self,
coords,
comps,
init_guess,
fit_background=None,
lsqKwargs=None):
'''
Parameters
----------
coords : tuple of ints
comps : list of strings
init_guess : list of floats
fit_background : string or None
Returns
-------
coefficients : array of floats
'''
if lsqKwargs is None:
lsqKwargs = {}
def background(t, coeffs):
if fit_background == 'power law':
return coeffs[-2] * pow(t, coeffs[-1])
if fit_background == 'log-linear':
return np.log(coeffs[-2] * pow(t, coeffs[-1]))
if fit_background == 'double power law':
return coeffs[-4] * pow(t, coeffs[-3]) + coeffs[-2] * pow(
t, coeffs[-1])
def model(t, *coeffs):
y = 0.0
for i, comp in enumerate(comps):
y += coeffs[i] * self.stds.ready[comp].inav[coords].data
if fit_background:
y += background(t, coeffs)
return y
y_Obs = self.HL.data.inav[coords].data
t = self.HL.data.axes_manager[self.sigDim].axis
opt_cofs, cov = lsq(model, t, y_Obs, p0=init_guess, **lsqKwargs)
self.last = opt_cofs
return opt_cofs
def lsq_excess_fit(ref_excess_dict, red_law, EBV, rv_guess, filters, zp):
prefix = zp['prefix']
filter_eff_waves = np.array([snc.get_bandpass(prefix+f).wave_eff for f in filters])
ref_excess = np.array([ref_excess_dict[f] for f in filters])
def lsq_func(Y):
RV = Y[0]
ftz_curve = red_law(filter_eff_waves,
np.zeros(filter_eff_waves.shape),
-EBV, RV,
return_excess=True)
return ftz_curve-ref_excess
Y = np.array([rv_guess])
valid_phases = {}
return lsq(lsq_func, Y)
def trace(im, err, guess):
"""Trace the peak pixels along the second axis of an image.
Parameters
----------
im : array
The image to trace.
err : array
The image uncertainties. Set to None for unweighted fitting.
guess : array
The initial guesses for Gaussian fitting the y-value at x=0:
(mu, sigma, height, [m, b]) = position (mu), width (sigma),
height, and linear background m*x + b.
Returns
-------
y : ndarray
y-axis positions of the peak, along the second axis.
"""
from scipy.optimize import leastsq as lsq
from ..util import gaussian
def chi(p, x, y, err):
if len(p) > 3:
mu, sigma, height, m, b = p
else:
mu, sigma, height = p
m, b = 0, 0
model = gaussian(x, mu, sigma) * height + m * x + b
chi = (y - model) / err
return chi
y = np.zeros(im.shape[1])
y0 = np.arange(im.shape[0])
last = guess
for i in xrange(im.shape[1]):
if err is None:
err = np.ones_like(y)
fit, err = lsq(chi, last, (np.array(x), np.array(y), np.array(err)))
y[i] = fit[0]
last = fit
return y
def trace(im, err, guess):
"""Trace the peak pixels along the second axis of an image.
Parameters
----------
im : array
The image to trace.
err : array
The image uncertainties. Set to None for unweighted fitting.
guess : array
The initial guesses for Gaussian fitting the y-value at x=0:
(mu, sigma, height, [m, b]) = position (mu), width (sigma),
height, and linear background m*x + b.
Returns
-------
y : ndarray
y-axis positions of the peak, along the second axis.
"""
from scipy.optimize import leastsq as lsq
from ..util import gaussian
def chi(p, x, y, err):
if len(p) > 3:
mu, sigma, height, m, b = p
else:
mu, sigma, height = p
m, b = 0, 0
model = gaussian(x, mu, sigma) * height + m * x + b
chi = (y - model) / err
return chi
y = np.zeros(im.shape[1])
y0 = np.arange(im.shape[0])
last = guess
for i in xrange(im.shape[1]):
if err is None:
err = np.ones_like(y)
fit, err = lsq(chi, last, (np.array(x), np.array(y), np.array(err)))
y[i] = fit[0]
last = fit
return y
def optimize(self):
self.optimization_setup()
p = np.array([0.02289848, 0.01645834])
jump_opt = False
if jump_opt == False:
self.fast_run = True
p = np.array([1.])
if self.lam != 0:
p = np.array([1., 1.])
'''
Estimate for HG: False
T = 5
p = np.array([0.01646613, 0.18810675])
'''
'''
Estimate for HG: True
T = 5
'''
p = np.array([0.01343025, 0.05155523])
fit = lsq(self.objective_function, p, bounds=(0, np.inf))
p = fit.x
self.fast_run = False
self.objective_function(p)
print('Plotting complete')
print('>>>Finished optimizing with:')
print('HG :', self.use_hg)
print('Lambda:', self.lam != 0)
if jump_opt == False:
print('Optimal:', fit.x)
else:
return
lambda_label = str(self.lam != 0)
txt = 'opt_lambda_' + lambda_label + '.txt'
fname = os.path.join(self.fem_opt_exp_data_vec_dir, txt)
np.savetxt(fname, p)
def fwhm(im, yx, bg=True, **kwargs):
"""Compute the FWHM of an image.
Parameters
----------
im : array
The image to fit.
yx : array
The center on which to fit.
bg : bool, optional
Set to `True` if there is a constant background to be
considered.
**kwargs
Any `radprof` keyword, e.g., `range` or `bins`.
Returns
-------
fwhm : float
The FHWM of the radial profile.
"""
from scipy.optimize import leastsq as lsq
from ..util import gaussian
R, I, n = radprof(im, yx, **kwargs)[:3]
r = R[I < (I.max() / 2.0)][0] # guess for Gaussian sigma
if bg:
def fitfunc(p, R, I):
return I - gaussian(R, 0, p[0]) * p[1] + p[2]
guess = (r, I.max(), I.min())
else:
def fitfunc(p, R, I):
return I - gaussian(R, 0, p[0]) * p[1]
guess = (r, I.max())
fit = lsq(fitfunc, guess, args=(R, I))[0]
return abs(fit[0]) * 2.35
def fitSkewedGaussian(array):
(y,bins) = numpy.histogram(array,bins=50,normed=True)
x = 0.5*(bins[:-1] + bins[1:])
y /= float(len(array))
maxloc = y.argmax()
mode = x[maxloc]
# fit skewed Gaussian
gamma = skew(array)
delta = numpy.sqrt(numpy.pi*numpy.abs(gamma)**(2./3.)/2./(numpy.abs(gamma)**(2./3.) + ((4-numpy.pi)/2)**(2./3.)))
if delta < 1:
alpha = delta/numpy.sqrt(1-delta**2.0)
else:
alpha = 0.99
if gamma < 0:
alpha *= -1
params = numpy.array([mode,array.var(),alpha,y[maxloc]])
out = lsq(errfunc,params,args=(x,y),full_output=1)
pfinal = out[0]
return pfinal
def fitSkewedGaussian(array):
(y, bins) = numpy.histogram(array, bins=50, normed=True)
x = 0.5 * (bins[:-1] + bins[1:])
y /= float(len(array))
maxloc = y.argmax()
mode = x[maxloc]
# fit skewed Gaussian
gamma = skew(array)
delta = numpy.sqrt(numpy.pi * numpy.abs(gamma)**(2. / 3.) / 2. /
(numpy.abs(gamma)**(2. / 3.) +
((4 - numpy.pi) / 2)**(2. / 3.)))
if delta < 1:
alpha = delta / numpy.sqrt(1 - delta**2.0)
else:
alpha = 0.99
if gamma < 0:
alpha *= -1
params = numpy.array([mode, array.var(), alpha, y[maxloc]])
out = lsq(errfunc, params, args=(x, y), full_output=1)
pfinal = out[0]
return pfinal
def fwhm(im, yx, bg=True, **kwargs):
"""Compute the FWHM of an image.
Parameters
----------
im : array
The image to fit.
yx : array
The center on which to fit.
bg : bool, optional
Set to `True` if there is a constant background to be
considered.
**kwargs
Any `radprof` keyword, e.g., `range` or `bins`.
Returns
-------
fwhm : float
The FHWM of the radial profile.
"""
from scipy.optimize import leastsq as lsq
from ..util import gaussian
R, I, n = radprof(im, yx, **kwargs)[:3]
r = R[I < (I.max() / 2.0)][0] # guess for Gaussian sigma
if bg:
def fitfunc(p, R, I):
return I - gaussian(R, 0, p[0]) * p[1] + p[2]
guess = (r, I.max(), I.min())
else:
def fitfunc(p, R, I):
return I - gaussian(R, 0, p[0]) * p[1]
guess = (r, I.max())
fit = lsq(fitfunc, guess, args=(R, I))[0]
return abs(fit[0]) * 2.35
def background(image, mask, fraction):
rind = (((skm.binary_dilation(mask,selem=skm.disk(6)))
^ skm.binary_dilation(mask,selem=skm.disk(3)))
* np.isfinite(image))
if np.any(rind):
rindvals = image[rind]
clipval = 4*np.percentile(rindvals,15.87)-\
3*(np.percentile(rindvals,2.28))
rind *= image <= clipval
yv,xv = np.where(rind)
x0,y0 = np.median(xv),np.median(yv)
dataz = np.c_[np.ones(xv.size),
xv-x0,
yv-y0]
try:
lsqcoeffs, _, _, _ = np.linalg.lstsq(dataz,image[rind])
outputs = lsq(myplane, np.r_[lsqcoeffs,0],
args=(xv-x0,
yv-y0,
image[rind]),
loss = 'soft_l1')
coeffs = outputs.x
yhit, xhit = np.where(mask)
bg = (coeffs[0]
+ coeffs[1] * (xhit-x0)
+ coeffs[2] * (yhit-y0)
+ coeffs[3] * (yhit-y0) * (xhit-x0))
# fracvals = 1 - np.nan_to_num(fraction[yhit, xhit])
bgavg = np.nansum(bg) # * fraction[yhit, xhit])
# import pdb; pdb.set_trace()
return(bgavg)
except ValueError:
return(np.nan)
def calc_ftz_lsq_fit(S1, S2, filters, zp, ebv, rv_guess,
dist_mod_true, dist_mod_weight, constrain_dist_mod=True, weight_dist_mod=True):
'''
Least Square Fitter for Fitzpatrick-Massa (1999) reddening law. This function assumes
that S1 is an unreddened twin of S2 -- in our case S1 is
::NOTES FOR ME::
must input spectra with matching phases
fit S1 to S2
1. calc 12cu mags
2. interpolate with pristine 11fe spectra at 12cu phases
3. run lsq_helper_func with pristine spectra
x0 is a dict like:
{'ebv': -1.37, 'rv': 1.4}
{ 'av': 1.85, 'p': -2.1}
DOCS -> http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html
xtol/ftol == something big like .01?
'''
from scipy.optimize import leastsq as lsq
# check that given spectra have matching phases
s1_phases, s2_phases = np.array([t[0] for t in S1]), np.array([t[0] for t in S2])
if not np.array_equal(s1_phases, s2_phases):
return ValueError('Phases in given spectra do not match!')
prefix = zp['prefix']
############################################################################
##### FUNCTIONS ############################################################
# returns a lightcurve for the given filter
def bandmags(f, spectra):
bandfluxes = [s.bandflux(prefix+f) for s in spectra]
return -2.5*np.log10( bandfluxes/zp[f] )
# function to be used in least-sq optimization; must only take a numpy array (y) as input
def lsq_func(Y):
rv = Y[0]
#dist_mod_shift = Y[1]
reddened = [snc.Spectrum(spec.wave, redden_fm(spec.wave, spec.flux, ebv, rv)) for spec in s1_spectra]
reddened_mags = [bandmags(f, reddened) for f in filters]
## if constrain_dist_mod:
## dist_mod_shift = dist_mod_true
## if weight_dist_mod:
## S1_REF = np.concatenate( [mag+dist_mod_shift for mag in reddened_mags] )
## return np.concatenate(( S2_REF - S1_REF , [dist_mod_weight*(dist_mod_shift - dist_mod_true)] ))
## else:
## S1_REF = np.concatenate( [mag+dist_mod_shift for mag in reddened_mags] )
## return S2_REF - S1_REF
S1_REF = np.concatenate( [mags+dist_mod_true for mags in reddened_mags] )
return S2_REF - S1_REF
############################################################################
s1_spectra, s2_spectra = [t[1] for t in S1], [t[1] for t in S2]
SN12CU_MW = dict( zip( 'UBVRI', [0.117, 0.098, 0.074, 0.058, 0.041] )) # 12cu Milky Way extinction
# get a concatenated array of lightcurves per filter, also correct for 12cu milky way extinction
S2_REF = np.concatenate( [bandmags(f, s2_spectra) - SN12CU_MW[f] for f in filters] )
#Y = np.concatenate(( red_vars, [dist_mod_true] )) # best guess vector
#Y = np.array([rv_guess, dist_mod_true])
Y = np.array([rv_guess])
return lsq(lsq_func, Y) # , full_output=False, xtol=0.01, ftol=0.01)
xmin = np.min(x) ymin = np.min(y) ymax = np.max(y) binwidth = (xmax - xmin) / 100. bins = np.arange(xmin, xmax + binwidth, binwidth) nx, binx, patchx = axHistx.hist(x, bins=bins) xp = (binx[:-1] + binx[1:]) / 2 binwidth = (ymax - ymin) / 100. bins = np.arange(ymin, ymax + binwidth, binwidth) ny, biny, patchy = axHisty.hist(y, bins=bins, orientation='horizontal') yp = (biny[:-1] + biny[1:]) / 2 # fit with skewed gaussian and plot params = np.array([nx.max(), 0.01, xp.mean(), np.sqrt(xp.var())]) out = lsq(errfunc, params, args=(xp, nx), full_output=1) pfinal = out[0] #print pfinal fitX = np.linspace(xmin, xmax, 200) fitY = fitfunc(pfinal, fitX) axHistx.plot(fitX, fitY) params = np.array([ny.max(), 0.01, yp.mean(), np.sqrt(yp.var())]) out = lsq(errfunc, params, args=(yp, ny), full_output=1) pfinal = out[0] #print pfinal fitX = np.linspace(ymin, ymax, 200) fitY = fitfunc(pfinal, fitX) axHisty.plot(fitX, fitY) # the xaxis of axHistx and yaxis of axHisty are shared with axScatter,
def calc_irlum(catalog = 'cohrs_finalcatalog_clumpflux_medaxis.fits',
refresh=False):
ctrr = 0
cat = Table.read(catalog)
current_open_file = ''
if 'ir_luminosity' not in cat.keys():
keylist = ['ir_luminosity','ir_flux','ir_flux_short',
'ir_lum_short','bg_flux','bg_lum']
for thiskey in keylist:
c = Column(np.zeros(len(cat))+np.nan,name=thiskey)
cat.add_column(c)
for cloud in cat:
if np.isnan(cloud['ir_luminosity']) or refresh:
orig_file = cloud['orig_file']+'.fits'
asgn_file = cloud['orig_file']+'_fasgn.fits'
higal_file = orig_file.replace('cohrs','higal_xmatch')
if os.path.isfile(datadir+'COHRS/'+orig_file) and \
os.path.isfile(datadir+'HIGAL_MATCHED/'+higal_file):
if current_open_file != datadir+'COHRS/'+orig_file:
hdu = fits.open(datadir+'COHRS/'+orig_file,memmap=False)
w = wcs.WCS(hdu[0].header)
hdr2 = w.to_header()
hdr2['BMAJ'] = 15./3600
hdr2['BMIN'] = 15./3600
hdr2['BPA'] = 0.
co = SpectralCube(hdu[0].data,w,header=hdr2)
irfull= fits.open(datadir+'HIGAL_MATCHED/'+higal_file,memmap=False)
irlong = fits.open(datadir+'HIGAL_MATCHED2/'+higal_file,memmap=False)
irmap = (irfull[0].data-irlong[0].data)
irmap2 = irfull[0].data
asgn = fits.getdata(datadir+'ASSIGNMENTS/'+asgn_file,memmap=False)
masked_co = co.with_mask(asgn>0*u.dimensionless_unscaled)
moment = masked_co.moment(0)
current_open_file = datadir+'COHRS/'+orig_file
cat.write('output_catalog.fits',overwrite=True)
# mask = np.zeros(co.shape,dtype=np.bool)
# mask[asgn == cloud['cloud_id']]=True
cloud_cube = co.with_mask(asgn == cloud['cloud_id'])
cloud_moment = cloud_cube.moment(0)
cloud_cube = 0.0
fraction = (cloud_moment.value/moment.value)
planemask = skm.binary_closing(fraction > 0,selem=skm.disk(3))
fraction = np.nanmean(fraction)
rind = (skm.binary_dilation(planemask,selem=skm.disk(6)) ^\
skm.binary_dilation(planemask,selem=skm.disk(3)))*\
np.isfinite(irmap2)
if np.any(rind):
rindvals = irmap2[rind]
clipval = 4*np.percentile(rindvals,15.87)-\
3*(np.percentile(rindvals,2.28))
rind *= irmap2 <= clipval
yv,xv = np.where(rind)
x0,y0 = np.median(xv),np.median(yv)
dataz = np.c_[np.ones(xv.size),
xv-x0,
yv-y0]
try:
lsqcoeffs,_,_,_ = np.linalg.lstsq(dataz,irmap2[rind])
outputs = lsq(myplane,np.r_[lsqcoeffs,0],
args=(xv-x0,
yv-y0,
irmap2[rind]),
loss = 'soft_l1')
coeffs = outputs.x
yhit,xhit = np.where(planemask)
bg = coeffs[0]+coeffs[1]*(xhit-x0)+\
coeffs[2]*(yhit-y0)+coeffs[3]*(yhit-y0)*(xhit-x0)
# I am sitcking a 6e11 in here as the frequency of
# the 500 microns
bgavg = np.sum(fraction*bg)/6e11
bglum = bgavg*cloud['distance']**2*\
3.086e18**2*np.pi*4/3.84e33
except ValueError:
import pdb; pdb.set_trace()
ir_flux = np.nansum(fraction*(irmap[planemask]))/6e11
ir_lum = ir_flux * cloud['distance']**2*\
3.086e18**2*np.pi*4/3.84e33
ir_flux2 = np.nansum(fraction*irmap2[planemask])/6e11
ir_lum2 = ir_flux2 * cloud['distance']**2*\
3.086e18**2*np.pi*4/3.84e33
cloud['ir_flux'] = ir_flux2
cloud['ir_luminosity'] = ir_lum2
cloud['ir_flux_short'] = ir_flux
cloud['ir_lum_short'] = ir_lum
cloud['bg_flux'] = bgavg
cloud['bg_lum'] = bglum
print(cloud['cloud_id'],ir_flux,ir_lum,ir_lum2,bglum)
cat.write('output_catalog.fits',overwrite=True)
ctrr+=1
if ctrr==20:
return(cat)
# if cloud['volume_pc2_kms']>1e2:
# import pdb; pdb.set_trace()
return(cat)
def lsq_11fe_color_fit(SN, EBV, rv_guess, filters, zp, N_DAYS):
'''
This is a helper function to fit an RV to a certain supernova (SN), using
an artificially reddened 11fe as a template. EBV is fixed and then 'filters'
are the filters you want to include in the fit (with 'zp' as zero-points
dictionary). The fitter will only use phases from the comparison supernova
which are no more than N_DAYS from the nearest 11fe phase. SN needs to be
in the form (populated by the phases and corresponding V-X colors):
SN = {'B':[(-6.5, -1.0601018152416941),
(-3.5, -1.0864032119751066),
(-1.4, -1.08253418576437),
(6.5, -1.2348233700462039),
(8.5, -1.2831185823642457),
(11.5, -1.3001372408790797),
...
lsq_out, valid_phases = lsq_11fe_color_fit(SN, -1.29, 1.3, 'UBRI', ZP_CACHE, 2.0)
This function outputs the output of SciPy's least-square fit, and also a
dictionary with the phases used in the fit, respective to each band:
lsq_out = (array([ 1.3776701]), 1)
valid_phases = {'B': array([ -3.5, -0.7, 1.1, 5.2, 6.9, 7.9,
8.7, 16.7, 20.3, 28.1]),
'I': array([ -3.5, -2.5, -0.7, 1.1, 3.4, 5.2,
6.9, 7.9, 8.7, 16.7, 20.3, 28.1]),
'R': array([ -3.5, -2.5, -0.7, 1.1, 3.4, 5.2,
6.9, 7.9, 8.7, 16.7, 20.3, 28.1]),
'U': array([ -3.5, -2.5, 3.4, 5.2, 6.9, 7.9,
8.7, 16.7, 20.3, 28.1])}
'''
from scipy.optimize import leastsq as lsq
def lsq_func(Y):
print Y
RV = Y[0]
sn11fe = l.get_11fe('fm', ebv=EBV, rv=RV)
sn11fe_colors = np.array([])
sncomp_colors = np.array([])
for f in filters:
# get phases from comparison supernova
band_data = SN[f]
phases = [t[0] for t in band_data]
# interpolate 11fe at 12cu phases which are no more than N_DAYS
# away from the nearest 11fe phase
valid = find_valid([t[0] for t in sn11fe], phases, N_DAYS)
valid_phases[f] = np.array(phases)[valid]
sn11fe_int = l.interpolate_spectra(valid_phases[f], sn11fe)
# calculate band magnitudes and then colors for interpolated 11fe
temp, sn11fe_vmags = calc_v_band_mags(sn11fe_int, zp)
sn11fe_bandcolors = calc_colors(f, sn11fe_int, sn11fe_vmags, zp)
sn11fe_phases = [t[0] for t in sn11fe_bandcolors]
sn11fe_bandcolors = [t[1] for t in sn11fe_bandcolors]
sn11fe_colors = np.concatenate(( sn11fe_colors, sn11fe_bandcolors ))
# get comparison supernova's color at valid phases
sncomp_bandcolors = [t[1] for t in band_data if t[0] in sn11fe_phases]
sncomp_colors = np.concatenate(( sncomp_colors, sncomp_bandcolors ))
return sncomp_colors - sn11fe_colors
Y = np.array([rv_guess])
valid_phases = {}
return lsq(lsq_func, Y), valid_phases
ratio = FFT_P/FFT_H
ratio_freq = numpy.fft.fftfreq(len(sig_H), d=d)
#phase = numpy.arctan(ratio)
phase = numpy.arctan2(numpy.imag(ratio), numpy.real(ratio))
freqs = scipy.where( (ratio_freq > 0) & (ratio_freq < 20.0))[0]
guess = [-1.0]
def calc_error(g, xvals, yvals):
e = 0.0;
for x, y in zip(xvals, yvals):
e+= (x*g[0] - y)**2.0
return e
fit = lsq(calc_error, guess, args=(ratio_freq[freqs], phase[freqs]))
line = []
for i in ratio_freq[freqs]:
line.append(i*fit[0][0])
ax.plot(ratio_freq[freqs], phase[freqs])
ax.plot(ratio_freq[freqs], line)
latency = fit[0][0]/(2.0*3.14159)
print "Latency : ", numpy.abs(latency)
#ax.plot(ratio_freq, numpy.abs(ratio))
fig.show()
def features_training(num_images=1):
"""
Function to analyse the features of the TRAINING set of the autoencoder
"""
norm_coef = []
losses_focus, peaks_focus, mins_focus = [], [], []
losses_defocus, peaks_defocus, mins_defocus = [], [], []
# ============================================================================================================ #
### Light Loss - see how the Low Orders modify the total intensity
for j in range(N_ext):
low_orders = ae_coefs[j, :N_low]
norm_coef.append(np.linalg.norm(low_orders))
input_focus = train_noisy[j, :N_crop**2].reshape((N_crop, N_crop))
output_focus = train_clean[j, :N_crop**2].reshape((N_crop, N_crop))
removed_features_focus = input_focus - output_focus
loss_focus = np.sum(removed_features_focus)
losses_focus.append(loss_focus)
peaks_focus.append(np.max(removed_features_focus))
mins_focus.append(np.min(removed_features_focus))
input_defocus = train_noisy[j, N_crop**2:].reshape(
(N_crop, N_crop))
output_defocus = train_clean[j, N_crop**2:].reshape(
(N_crop, N_crop))
removed_features_defocus = input_defocus - output_defocus
loss_defocus = np.sum(removed_features_defocus)
losses_defocus.append(loss_defocus)
peaks_defocus.append(np.max(removed_features_defocus))
mins_defocus.append(np.min(removed_features_defocus))
norm_coef = np.array(norm_coef)
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, sharey=True)
# Focused PSF
p_sort = np.argsort(peaks_focus)
ax1.scatter(norm_coef[p_sort],
np.sort(peaks_focus),
color=cm.bwr(
np.linspace(0.5 + np.min(peaks_focus), 1, N_ext)),
s=4,
label='Maxima')
m_sort = np.argsort(mins_focus)
ax1.scatter(norm_coef[m_sort],
np.sort(mins_focus),
color=cm.bwr(np.linspace(0, 0.5, N_ext)),
s=4,
label='Minima')
loss_sort = np.argsort(losses_focus)
ax1.legend(loc=2)
leg = ax1.get_legend()
leg.legendHandles[0].set_color('red')
leg.legendHandles[1].set_color('blue')
ax1.axhline(y=0.0, linestyle='--', color='black')
ax1.set_title('Nominal PSF')
ax1.set_ylabel(r'Light loss')
ax1.set_ylim([-0.5, 0.5])
ax3.scatter(norm_coef[loss_sort],
np.sort(losses_focus),
color='black',
s=3,
label='Total')
ax3.legend(loc=2)
ax3.axhline(y=0.0, linestyle='--', color='black')
ax3.set_xlabel(r'Norm of low orders $\Vert a_{low} \Vert$')
ax3.set_ylabel(r'Light loss')
# Defocused PSF
p_sort = np.argsort(losses_defocus)
ax2.scatter(norm_coef[p_sort],
np.sort(peaks_defocus),
color=cm.bwr(
np.linspace(0.5 + np.min(peaks_defocus), 1, N_ext)),
s=4,
label='Maxima')
m_sort = np.argsort(mins_defocus)
ax2.scatter(norm_coef[m_sort],
np.sort(mins_defocus),
color=cm.bwr(np.linspace(0, 0.5, N_ext)),
s=4,
label='Minima')
loss_sort = np.argsort(losses_defocus)
ax2.legend(loc=2)
leg = ax2.get_legend()
leg.legendHandles[0].set_color('red')
leg.legendHandles[1].set_color('blue')
ax2.axhline(y=0.0, linestyle='--', color='black')
ax2.set_title('Defocused PSF')
ax4.scatter(norm_coef[loss_sort],
np.sort(losses_defocus),
color='black',
s=3,
label='Total')
ax4.legend(loc=2)
ax4.axhline(y=0.0, linestyle='--', color='black')
ax4.set_xlabel(r'Norm of low orders $\Vert a_{low} \Vert$')
# ============================================================================================================ #
### PCA analysis of the removed features
# Focused PSF
N_comp = N_low
removed_features = train_noisy[:, :N_crop**2] - train_clean[:, :N_crop
**2]
pca = PCA(n_components=N_comp)
pca.fit(X=removed_features)
components = pca.components_.reshape((N_comp, N_crop, N_crop))
variance_ratio = pca.explained_variance_ratio_
total_variance = np.sum(variance_ratio)
plt.figure()
for i in range(N_comp):
ax = plt.subplot(2, N_comp, i + 1)
plt.imshow(components[i], cmap='seismic', origin='lower')
ax.set_title(r'PCA #%d [$\sigma^2_r=%.2f/%.2f$]' %
(i + 1, variance_ratio[i], total_variance))
plt.colorbar(orientation="horizontal")
cmin = min(components[i].min(), -components[i].max())
plt.clim(cmin, -cmin)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
removed_features_defocus = train_noisy[:, N_crop**
2:] - train_clean[:, N_crop**2:]
pca_defocus = PCA(n_components=N_comp)
pca_defocus.fit(X=removed_features_defocus)
components_defocus = pca_defocus.components_.reshape(
(N_comp, N_crop, N_crop))
variance_ratio_defocus = pca_defocus.explained_variance_ratio_
total_variance_defocus = np.sum(variance_ratio_defocus)
for i in range(N_comp):
ax = plt.subplot(2, N_comp, i + 1 + N_comp)
plt.imshow(components_defocus[i], cmap='seismic', origin='lower')
ax.set_title(
r'PCA #%d [$\sigma^2_r=%.2f/%.2f$]' %
(i + 1, variance_ratio_defocus[i], total_variance_defocus))
plt.colorbar(orientation="horizontal")
cmin = min(components_defocus[i].min(),
-components_defocus[i].max())
plt.clim(cmin, -cmin)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
### Least Squares fit of the removed features
def residuals_lsq(x, image_data, pca_components):
model_image = np.dot(x, pca_components)
return image_data - model_image
# ============================================================================================================ #
### Compare the removed features to the true difference
random_images = np.random.randint(train_noisy.shape[0],
size=num_images)
for j in random_images:
im = removed_features[j]
res_lsq = lsq(fun=residuals_lsq,
x0=np.zeros(N_comp),
args=(im, pca.components_))
x_fit = res_lsq['x']
im_fit = (np.dot(x_fit, pca.components_)).reshape((N_crop, N_crop))
im = im.reshape((N_crop, N_crop))
vmin_im = min(im.min(), -im.max())
vmin_fit = min(im_fit.min(), -im_fit.max())
vmin = min(vmin_im, vmin_fit)
error = np.sum(np.abs(im_fit - im)) / np.sum(np.abs(im))
plt.figure()
cmap = 'seismic'
ax1 = plt.subplot(1, 3, 1)
plt.imshow(im, cmap=cmap)
plt.colorbar(orientation="horizontal")
plt.clim(vmin, -vmin)
ax1.get_xaxis().set_visible(False)
ax1.get_yaxis().set_visible(False)
ax1.set_title(
r'Removed features: $PSF(\Phi_{low} + \Phi_{high}) - PSF(\Phi_{high})$'
)
ax2 = plt.subplot(1, 3, 2)
plt.imshow(im_fit, cmap=cmap)
plt.colorbar(orientation="horizontal")
plt.clim(vmin, -vmin)
ax2.get_xaxis().set_visible(False)
ax2.get_yaxis().set_visible(False)
ax2.set_title(r'Least-Squares fit from PCA: $x_{lsq} \cdot PCA$')
res = im_fit - im
ax3 = plt.subplot(1, 3, 3)
plt.imshow(res, cmap='bwr')
plt.colorbar(orientation="horizontal")
min_res = min(res.min(), -res.max())
plt.clim(min_res, -min_res)
ax3.get_xaxis().set_visible(False)
ax3.get_yaxis().set_visible(False)
ax3.set_title(r'Residuals ($\epsilon = %.2f$)' % error)
# ============================================================================================================ #
### Show some examples of the true features
random_images = np.random.choice(train_noisy.shape[0],
size=48,
replace=False)
print(random_images)
plt.figure()
for i, img_j in enumerate(random_images):
ax = plt.subplot(6, 8, i + 1)
im = removed_features[img_j].reshape((N_crop, N_crop))
plt.imshow(im, cmap='seismic')
min_im = min(im.min(), -im.max())
plt.clim(min_im, -min_im)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
return pca
def BIC():
wcsfile = "/data2/nova/BACKUP-jobs/00000046/wcs.fits"
folder = os.path.dirname(wcsfile)
pub_name = folder.split("/")[-1] # to save on the interwebs
rdlsfiles = glob("./rdls*fits")
log = folder + "/job.log"
words = open(log).readlines()[3].split()
print words
imagef_ind = words.index("--image") + 1 # wrong filename for the image
badfile = words[imagef_ind]
getpath = badfile.split("/")[-4:]
goodfile = "/home/nova/BACKUP-data/{0}/{1}/{2}/{3}".format(*tuple(getpath))
'''Now we have the right filename for the original.'''
try:
fits = pyfits.open(goodfile)
picture = fits[0].data
if type(picture) != np.ndarray:
picture = fits[1].data
except (IOError):
picture = mpimg.imread(goodfile)[::-1]
# let's just work on one of the colours:
pic = picture[:, :, 0].astype("float64")
stdev = MAGIC_NO * MAD(pic)
dim = pic.shape
print "dimensions are ", dim
ext = 0
wcs = tractor.FitsWcs(Tan(wcsfile, ext))
# pick the ra,dec of a star to check
cat_srcs = []
for rdfile in rdlsfiles:
rdls = pyfits.open(rdfile)
radec = rdls[1].data
for ra in radec:
cat_srcs.append(ra)
for run, src in enumerate(cat_srcs):
print "###################################"
print "######## RUN NO. ", run, "#########"
print "###################################"
ra, dec = src
t_radec = tractor.RaDecPos(ra, dec)
print "RaDecPos obj: ", t_radec
print wcs.positionToPixel(t_radec)
src_xy = np.asarray(wcs.wcs.radec2pixelxy(ra, dec))
print src_xy
cent = np.rint(src_xy).astype("int")
print "centre:"
print cent[0]
print cent[1]
if cent[0] < 0 or cent[0] >= dim[1] or cent[1] < 0 or cent[1] >= dim[0]:
print "rdls not correct, out of image"
continue
# let's just switch the centre values and see...
starr = pic[cent[1] - 7:cent[1] + 8, cent[0] - 7:cent[0] + 8]
print "starr is ", starr.shape
if starr.shape == (sq, sq):
# now try to fit to sky model
med = np.median(starr)
peak = np.amax(starr)
print "median of starr: ", med
print "max of starr: ", peak
"""guess where gaussian is centred"""
x0, y0 = (float(pos[0]) for pos in np.where(starr == peak))
"""
Define plotstuff object
- 0th term is sky only model
- 1st term is sky and psf
- 2nd is above plus roaming psf
"""
skyp = colxns.namedtuple('skyp', 'z0 mh mv')
# skypsf = colxns.namedtuple("skypsf", "z0 mh mw varx vary cor")
# roamp = colxns.namedtuple("roamp", "z0 mh mv w varx vary cor x y")
ciorc = colxns.namedtuple("ciorc", "z0 mh mv w var")
rokirk = colxns.namedtuple("rokirk", "z0 mh mv w var x y")
x0, y0 = 7., 7. # try with the centre this time
kwargs1 = dict(z0=med, mh=0., mv=0.)
kwargs2 = dict(kwargs1, w=(peak - med), var=3.)
kwargs3 = dict(kwargs2, x=x0, y=y0)
# kwargs3 = kwargs2.copy()
# kwargs3["roam"] = True
class RunParams():
pass
stuff = [RunParams(), RunParams(), RunParams()]
stuff[0].tup, stuff[0].kws, stuff[0].fun = skyp, kwargs1, sky_mod
stuff[1].tup, stuff[1].kws, stuff[1].fun = ciorc, kwargs2, ciorcal
stuff[2].tup, stuff[2].kws, stuff[2].fun = rokirk, kwargs3, roam_ciorc
# and start up the figure for plotting
fig = plt.figure()
gs = gridspec.GridSpec(3, 3)
axlist = []
head = np.percentile(starr, 99.9)
toe = np.percentile(starr, 0.1)
for no, plst in enumerate(stuff):
"""
and here we do the fitting and plotting, one row for each model
"""
if no == 0:
print "SKY ONLY"
elif no == 1:
print "SKY + PSF"
elif no == 2:
print "ROAMING PSF"
# first the real image
axlist.append(plt.subplot(gs[no, 0]))
# axlist[no * 3].set_axis_off()
axlist[no * 3].set_xticks([])
axlist[no * 3].set_yticks([])
if no == 0:
axlist[0].set_title(r"Data")
axlist[no * 3].imshow(starr, cmap=plt.cm.gray,
interpolation="nearest", vmin=toe, vmax=head)
if no == 0:
axlist[0].set_ylabel(r"sky only")
elif no == 1:
axlist[3 * no].set_ylabel(r"sky + psf")
elif no == 2:
axlist[3 * no].set_ylabel(r"sky + roaming psf")
# now fit and plot the model
param_guess = plst.tup(**plst.kws)
xdata = np.ones(225)
result = lsq(residuals, param_guess, args=(plst.fun, xdata,
starr))
solve_params = result[0]
soln = plst.tup(*solve_params)
print "optimal params are: ", soln
xdata = np.ones(225)
bestsky = plst.fun(soln, xdata)
if no == 1:
"""
For the roaming psf, let's initialise with solve from static.
"""
stuff[2].kws['z0'] = soln.z0
stuff[2].kws['mh'] = soln.mh
stuff[2].kws['mv'] = soln.mv
stuff[2].kws['w'] = soln.w
stuff[2].kws['var'] = soln.var
axlist.append(plt.subplot(gs[no, 1]))
# axlist[no * 3 + 1].set_axis_off()
axlist[no * 3 + 1].set_xticks([])
axlist[no * 3 + 1].set_yticks([])
if no == 0:
axlist[1].set_title(r"Model")
axlist[no * 3 + 1].imshow(bestsky, cmap=plt.cm.gray,
interpolation="nearest",
vmin=toe, vmax=head)
# and finally the chi image
chimg = chi_img(starr, bestsky, stdev)
minchi, maxchi, maxchisq, totchisq = chisq_calc(starr, bestsky, stdev)
print "min(chi) is ", minchi
print "max(chi) is ", maxchi
print "max(chi ** 2) is ", maxchisq
print "total(chi ** 2) is ", totchisq
axlist.append(plt.subplot(gs[no, 2]))
# axlist[no * 3 + 2].set_axis_off()
axlist[no * 3 + 2].set_xticks([])
axlist[no * 3 + 2].set_yticks([])
if no == 0:
axlist[2].set_title(r"Chi Image")
axlist[no * 3 + 2].imshow(chimg, cmap=plt.cm.gray,
interpolation="nearest",
vmin=-5., vmax=5.)
axlist[no * 3 + 2].yaxis.set_label_position("right")
axlist[no * 3 + 2].set_ylabel(r"$\chi^{2} = %s$" % str(totchisq))
# for ax in axlist:
# for t in ax.xaxis.get_ticklines():
# t.set_visible(False)
plt.tight_layout()
fig.savefig("{0}/{1}.png".format(webname, run))
else:
print "out of range"
continue
def fitEphem(myLoc, T0, period, simple=False):
"""
Takes eclipse time data from a file, and fits ephemeris parameters (T0, period) to them
from an initial guess.
There are two fitting options available: simple and complex.
simple:
Fits a simple linear regression fit to the data. Report the minimum chisq params
complex:
Uses an MCMC chain and a gaussian process to fit the data. Also considers that the
reported error is not accurate, and allows data from different sources to have their
error bars scaled to get a chisq closer to unity. This is far more reliable and accurate,
but takes significantly longer to run.
Arguments:
----------
myLoc: str
Tells the function where to look for both logfiles and prior eclipse data
T0: float
Initial guess for T0
period: float
Initial guess for the period
simple: bool
If true, use a linear regression fit on the data. If false, use an MCMC chain.
Returns:
--------
T0: float
Best T0
period: float
Best period
"""
printer("\n\n--- Fitting ephemeris to data ---")
# Read in the eclipsetimes.txt file
fname = path.join(myLoc, 'EPHEMERIS', 'eclipse_times.txt')
if not path.isfile(fname):
raise FileNotFoundError("I don't have any eclipse times to fit!")
source_key, tl = read_ecl_file(fname)
### Fitting
x = np.array([float(x[0]) for x in tl]) # cycle number
y = np.array([float(x[1]) for x in tl]) # Time
ey = np.array([float(x[2]) for x in tl]) # Error
obsCodes = np.array([x[3] for x in tl]) # Data source
params = [T0, period]
def fitfunc(p,x):
return p[0] + p[1]*x
if simple:
def errfunc(p,x,y,err):
return (y-fitfunc(p,x)) / err
out = lsq(errfunc,params,args=(x,y,ey),full_output=1)
pfinal = out[0]
covar = out[1]
P, P_err = pfinal[1], np.sqrt(covar[1][1])
T0, T0_err = pfinal[0], np.sqrt(covar[0][0])
chisq = (y - fitfunc(pfinal, x))**2 / (ey**2)
chisq = sum(chisq) / (len(chisq)-2)
resy = y - fitfunc(pfinal, x)
else:
# Get the number of sources, which will each have their errors scaled differently
numObs = len(source_key)
# Set up the MCMC chain.
npars = 2+numObs # The parameters are the T0, P, and the error scale of each source
nwalkers = max(16,4*(2+numObs)) # The number of walklers wants to be at least 16,
# or enough to sample our parameter space properly
# Construct a list of parameters for the model, i.e. T0, P, and the error scale factor of each source
nameList = ['T0','P']
for i, key in enumerate(source_key):
# add error factor for each source
params.append(1.0)
sourceName = source_key[key].replace(' ', '_')
nameList.append('obs_err_{:s}'.format(sourceName))
guessP = np.array(params)
# Initialise the sampler
p0 = emcee.utils.sample_ball(guessP,0.01*guessP,size=nwalkers)
sampler = emcee.EnsembleSampler(nwalkers,npars,ln_prob,args=[x,y,ey,obsCodes])
#burn in
nburn=5000
pos, prob, state = run_burnin(sampler, p0,nburn)
#production
sampler.reset()
nprod = 2000
chain_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_chain.txt")
sampler = run_mcmc_save(sampler, pos, nprod, state, chain_fname)
chain = flatchain(sampler.chain, npars, thin=1)
# Gather and report the best values
bestPars = []
for i in range(npars):
par = chain[:,i]
lolim,best,uplim = np.percentile(par,[16,50,84])
# printer("{:>20s} = {:.10f} +{:.10f} -{:.10f}".format(nameList[i],best,uplim-best,best-lolim))
bestPars.append(best)
if nameList[i] == 'T0':
printer("Old T0: {:.8f}".format(T0))
T0 = best
T0_err = uplim-lolim
printer("New T0: {:.8f}+/-{:.8f}\n".format(T0, T0_err))
if nameList[i] == 'P':
printer("Old P: {:.8f}".format(period))
P = best
P_err = uplim-lolim
printer("New P: {:.8f}+/-{:.8f}\n".format(P, P_err))
fig = thumbPlot(chain,nameList)
corner_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_corner_plot.pdf")
fig.savefig(corner_fname)
printer("Saved a corner plot of the MCMC fit (including error scaling factors) to:\n-> {}".format(corner_fname))
plt.close('all')
resy = 86400.0*(y-model(bestPars,x))
errs = ey.copy()
# scale errors by amount set by observatory code
emul_factors = np.array(bestPars[2:])
multipliers = emul_factors[obsCodes-1]
errs = errs*multipliers
ey = 86400.0*errs
chisq = np.sum(resy**2.0/ey**2.0)
printer("\nChisq = {:.1f}, with {:d} degrees of freedom".format(chisq, int(x.size - 2)))
### Reporting
printer("Got a T0 of {:>5.15f}+/-{:<.2e}".format(T0, T0_err))
printer("Got a period of {:>5.15f}+/-{:<.2e}".format(P, P_err))
printer("This fit had a reduced chisq value of {:.3f}".format(chisq))
printer('')
printer("Source | (Obs) - (Calc), sec | Cycle Number")
for t in tl:
dT = fitfunc([T0, P], t[0]) - t[1]
dT *= 24*60*60
printer(" {:<14s} | {:>20.4f} | {:d}".format(source_key[str(int(t[3]))], dT , int(t[0]) ))
# Each error code wants to be a different color
codes = set(obsCodes) # set() strips duplicates
codes = list(codes) # list() allows indexing
CB_color_cycle = ['black', '#ff7f00', '#4daf4a',
'#377eb8', '#a65628', '#984ea3',
'#999999', '#e41a1c', '#dede00']
colors = []
for code in obsCodes:
i = codes.index(code)
colors.append(CB_color_cycle[i])
plt.errorbar(x,resy,yerr=ey,marker='',ecolor='k', linestyle='', zorder=1)
plt.scatter(x, resy, c=colors, marker='o', zorder=2)
plt.axhline(ls='--',color='k', zorder=1)
plt.xlabel('Cycle No.')
plt.ylabel('O-C (s)')
scatter_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_scatter.pdf")
ephem_fname = path.join(myLoc, "EPHEMERIS", "ephemeris_fit.txt")
with open(ephem_fname, 'w') as f:
f.write("Got a T0 of {:>5.15f}+/-{:<.2e}\n".format(T0, T0_err))
f.write("Got a period of {:>5.15f}+/-{:<.2e}\n".format(P, P_err))
f.write("This fit had a reduced chisq value of {:.3f}\n".format(chisq))
f.write('\n')
f.write("Source | (Obs) - (Calc), sec | Cycle Number\n")
for t in tl:
dT = fitfunc([T0, P], t[0]) - t[1]
dT *= 24*60*60
f.write(" {:<14s} | {:>20.4f} | {:d}\n".format(source_key[str(int(t[3]))], dT , int(t[0]) ))
plt.savefig(scatter_fname)
plt.show()
printer("")
return T0, P
# An arbitrary list for the scale of the radius. according to every bin.
rlist = [.6, .8, 1, 1.2, 1.4, 1.6, 1.8, 2]
# Create gtrue, expected true value of gamma.
gtrue = []
for z in rlist:
gtrue.append(gf(.35, z))
p0 = [.5] # initial parameter guess
# This is an attempt to fit the data using the perceived function gf. p0 is the initial guess
# and the first argument in err. args will consist of any other arguments that might be necessary
# in the gf function.
fit = lsq(err, p0, args=(glist, rlist))
print(fit[0])
# Plotting stuff
"""ax = plot.figure().add_subplot(1,1,1)
for w in
ax.plot([4,6, 8, 9.5],glist, 'bo',)
ax.set_xlabel('Distance from Mass r (Megaparsec)')
ax.set_ylabel('Shear')
ax.set_xlim(0, 10)
ax.set_ylim(0, 25)
"""
plt.plot(rlist, glist, 'ro', rlist, peval(rlist, fit[0]), rlist, gtrue)
plt.legend(['Measured', 'Least Squares Fit', 'True'])
plt.title('Singular Isothermal Sphere \n Least-squares fit to Measured Shear')
def scale(data,model):
guess = numpy.abs(data.mean()) / numpy.abs(model.mean())
errfunc = lambda scale, data, model: data-scale*model
out = lsq(errfunc,guess,args=(data,model),full_output=1)
return out[0]
def fit(self):
ydata = self.yraw / self.yraw[0]
x0 = np.array([1,1])
R = lsq(_residuals, x0, args=(self.x, self.model, ydata))
self.fitted_pars = return R.x
pl.plot(x1, yplot, color = 'black', linewidth = 3, label = 'sklearn')
#fit made with function minimization
def line(x, m, q):
return m*x + q
def residual(p,x,y):
return y - line(x,*p)
guess = (3,3)
x2 = np.array([cleanData.logtime])
y2 = np.array([cleanData.mag])
xfit = x2[0,:]
yfit = y2[0,:]
m, q = np.polyfit(xfit,yfit,1)
print("Fit parameters obtained from polyfit: m,q ="
+ str(m) +" " + str(q))
pl.plot(x1,line(x1,m,q), color = 'red', linewidth = 1, label = 'polyfit')
pl.legend()
msq, qsq = lsq(residual,guess,args = (xfit,yfit))
mean_data = np.mean(y_data)
ss_total = np.sum((y_data - mean_data)**2)
ss_res = np.sum((quadratic_residual(x_fit, y_data, aberr_coef))**2)
R2 = 1 - ss_res / ss_total
return R2
# Aberrations that behave like a parabola
parab_indices = [0, 1, 2, 4]
parab_fit = []
parab_shift = []
for k in parab_indices:
y_data = peaks[k]
x_solve = lsq(quadratic_residual,
x0=[-0.1, 0.0],
args=(
y_data,
coef,
))
a_fit, x_shift = x_solve['x']
print(a_fit, x_shift)
parab_fit.append(a_fit)
parab_shift.append(x_shift)
R2 = r2([a_fit, x_shift], y_data, coef)
plt.figure()
lab = label = r'Fit: $f(x)=1 - %.3f (x + %.3f)^2$' % (-a_fit, x_shift)
plt.scatter(coef, y_data, label='Data', s=10)
plt.plot(coef,
quadratic([a_fit, x_shift], coef),
def scale(data, model):
guess = numpy.abs(data.mean()) / numpy.abs(model.mean())
errfunc = lambda scale, data, model: data - scale * model
out = lsq(errfunc, guess, args=(data, model), full_output=1)
return out[0]