def getBestDegree(x, y, fitDomainMin=None, fitDomainMax=None, maxDegree=3):
# Min Chi-Square Array
chiSquareArray = []
# Sort Arrays
sortIndexes = np.argsort(x)
x = [x[i] for i in sortIndexes]
y = [y[i] for i in sortIndexes]
# Get Fit Domain Limits
if fitDomainMin == None:
fitDomainMin = min(x)
if fitDomainMax == None:
fitDomainMax = max(x)
# Filter X and Y Arrays
xFiltered = []
yFiltered = []
for i in range(len(x)):
if x[i] >= float(fitDomainMin) and x[i] <= float(fitDomainMax):
xFiltered.append(float(x[i]))
yFiltered.append(float(y[i]))
xFiltered = np.array(xFiltered)
yFiltered = np.array(yFiltered)
for degree in range(1, maxDegree):
# Get Polynomial Parameters Array
polyParams = getPolyParams(degree)
# Fit
fit = kmpfit.simplefit(model, [0, 0, 0, 0, 0],
xFiltered,
yFiltered,
parinfo=[{
'fixed': polyParams[0]
}, {
'fixed': polyParams[1]
}, {
'fixed': polyParams[2]
}, {
'fixed': polyParams[3]
}, {
'fixed': polyParams[4]
}])
chiSquareArray.append(fit.rchi2_min)
return chiSquareArray.index(max(chiSquareArray)) + 1
zorder=1,
alpha=0.5)
compendium1 = zeros((len(data), 5), )
compendium1[:, 0] = redshifts
compendium1[:, 1] = d_red
compendium1[:, 2] = csfh
compendium1[:, 3] = ep_csfh
compendium1[:, 4] = em_csfh
p0 = [0.015, 2.9, 2.7, 7.0]
popt, pcov = curve_fit(func, redshifts, csfh, p0=p0, sigma=ep_csfh)
perr = sqrt(diag(pcov))
print(popt, perr)
fobj = kmpfit.simplefit(model, p0, redshifts, csfh, err=ep_csfh)
dfdp = dfdp_m(fobj.params, redshifts)
confprob = 0.68
yjunk, upperband, lowerband = fobj.confidence_band(redshifts, dfdp, confprob,
model)
## data = loadtxt('md14_UV.txt')
## redshifts1 = 0.5*(data[:,1]+data[:,0])
## d_red = 0.5*(data[:,1]-data[:,0])
## csfh = 10**(data[:,2])
## csfha = csfh
## ep_csfh = 10**(data[:,2]+data[:,3])
## em_csfh = 10**(data[:,2]-data[:,4])
## ax.errorbar(redshifts1, csfh, xerr=d_red, yerr=[csfh-em_csfh,ep_csfh-csfh],
if 'Title' in Data[DictItem]: if Data[DictItem]['Title'] != '': plt.title(Data[DictItem]['Title']) #Plot the data points #========= TO DO: add in an if statement to handle other types of plots =================== l = ax.scatter(Data['x'], Data['y'], s=10, c = Data[DictItem]['PointColour'], marker=str(Data[DictItem]['PointType']), zorder=2) if bool(Data['xErr']) and bool(Data['yErr']) and Data[DictItem]['UseErrors'] == True: plt.errorbar(Data['x'], Data['y'], xerr = Data['xErr'], yerr = Data['yErr'], ls = 'none') #do the fitting and plot it if Data[DictItem]['Fit'] == "alpha": if bool(Data[DictItem]['yColErr']): fitting_obj = kmpfit.simplefit(alpha, [ 1, 1 ], Data['x'], Data['y'], err = Data[DictItem]['yColErr']) else: fitting_obj = kmpfit.simplefit(alpha, [ 1, 1 ], Data['x'], Data['y']) plt.plot(dx, alpha(fitting_obj.params, dx), c = Data[DictItem]['FitColour']) elif Data[DictItem]['Fit'] == "line": fitting_obj = kmpfit.simplefit(line, [ 1, 1 ], Data['x'], Data['y']) plt.plot(dx, line(fitting_obj.params, dx), c = Data[DictItem]['FitColour']) elif Data[DictItem]['Fit'] == "lineErr": linear = odr.Model( lineErr ) mydata = odr.Data( np.asarray(Data['x']), np.asarray(Data['y']), wd = np.asarray(Data['xErr']), we = np.asarray(Data['yErr']) ) myodr = odr.ODR( mydata, linear, beta0 = [ 1, 1 ] ) myoutput = myodr.run()
def plot_param(hist, pixel=700, param='mu', error_plot=False):
plt.figure()
x = 5 * np.arange(0, hist.data.shape[0], 1)
mu = np.zeros(x.shape)
sigma_mu = np.zeros(x.shape)
x_fit = np.arange(0, 1000, 5)
#x_fit = x
y_fit = np.zeros(len(x_fit))
y_fit_max = np.zeros(len(x_fit))
y_fit_min = np.zeros(len(x_fit))
upper_band = np.zeros(len(x_fit))
lower_band = np.zeros(len(x_fit))
y_hat = np.zeros(len(x_fit))
sig_yi = np.zeros(len(x_fit))
sigma_y = np.zeros(len(x_fit))
n_sigma = 1
for j in range(hist.fit_result.shape[2]):
param_names = [
'$\mu$', '$\mu_{XT}$', 'gain', 'baseline', '$\sigma_e$',
'$\sigma_1$', 'amplitude', 'offset'
]
param_units = [
'[p.e.]', '[p.e.]', '[ADC/p.e.]', '[ADC]', '[ADC]', '[ADC]', '[]',
'[ADC]'
]
plt.subplot(3, 3, j + 1)
if error_plot:
y = hist.fit_result[:, pixel, j, 0]
y_err = hist.fit_result[:, pixel, j, 1]
plt.plot(x, y_err / y, label='data', marker='o', linestyle='None')
plt.ylabel('$\sigma$ / ' + param_names[j])
else:
y = hist.fit_result[:, pixel, j, 0] * (hist.fit_result[:, pixel, 2,
0] if
(j == 4 or j == 5) else 1.)
yerr = hist.fit_result[:, pixel, j,
1] * (hist.fit_result[:, pixel, 2, 0] if
(j == 4 or j == 5) else 1.)
if j == 0:
mu = y
sigma_mu = yerr
def model(p, x):
p0, p1, p2, p3, p4 = p
return p0 + p1 * x + p2 * x**2 + p3 * x**3 + p4 * x**4
def residuals(p, data):
x, y = data
p0, p1, p2, p3, p4 = p
#return (y - model(p, x))/
# fit with polyfit and compute symmetric CB
deg = int(4)
param, covariance = np.polyfit(x,
y,
deg=deg,
w=1. / yerr,
cov=True)
param_err = np.sqrt(np.diag(covariance))
xx = np.vstack([x_fit**(deg - i) for i in range(deg + 1)]).T
yi = np.dot(xx, param) #
C_yi = np.dot(xx, np.dot(covariance, xx.T))
sig_yi = np.sqrt(np.diag(C_yi))
y_fit = np.polyval(param, x_fit)
y_fit_max = np.polyval(param + param_err, x_fit)
y_fit_min = np.polyval(param - param_err, x_fit)
# fit with kmpfit module and compute CB
fit = kmpfit.simplefit(model, param, x, y, err=yerr)
dfdp = [
np.ones(len(x_fit)), x_fit, x_fit**2, x_fit**3, x_fit**4
]
#dfdp = [x_fit ** 4, x_fit**3, x_fit**2, x_fit, 1]
y_hat, upper_band, lower_band = fit.confidence_band(
x_fit, dfdp, 0.68, model, abswei=True)
# compute CB
print(' covar matrix kmpfit', fit.covar)
print(' covar matrix polyfit', covariance)
#fitobj = kmpfit.Fitter(residual=residuals, data=(x, y))
#fitobj.fit(params0=param)
#print("\nFit status kmpfit:")
#print("====================")
#print("Best-fit parameters: ", fitobj.params)
#print("Asymptotic error: ", fitobj.xerror)
#print("Error assuming red.chi^2=1: ", fitobj.stderr)
#print
#"Chi^2 min: ", fitobj.chi2_min
#print
#"Reduced Chi^2: ", fitobj.rchi2_min
#print
#"Iterations: ", fitobj.niter
#print
#"Number of free pars.: ", fitobj.nfree
#print
#"Degrees of freedom: ", fitobj.dof
for j in range(len(fit.params)):
for k in range(len(fit.params)):
sigma_y += dfdp[j] * dfdp[k] * fit.covar[
j, k] #covariance[j,k]#
sigma_y = np.sqrt(sigma_y * fit.rchi2_min)
plt.plot(x_fit, y_fit, label='best fit', color='red')
plt.errorbar(x,
y,
yerr=yerr,
label='data',
marker='o',
linestyle='None')
#plt.plot(x_fit, y_fit_max, label='polyfit + 1 $\sigma$')
#plt.plot(x_fit, y_fit_min, label='polyfit - 1 $\sigma$')
#plt.plot(x_fit, y_hat, label='kmpfit')
#plt.fill_between(x_fit, upper_band, lower_band, alpha=0.5, facecolor='blue', label='kmpfit confidence level')
plt.fill_between(x_fit,
yi + sig_yi,
yi - sig_yi,
alpha=0.5,
facecolor='red',
label='confidence level')
plt.ylabel(param_names[0] + ' ' + param_units[0])
print('param ', param, ' ± ', param_err)
print('kmpfit : ', fit.params, ' ± ', fit.xerror)
plt.legend(loc='best')
else:
plt.errorbar(x,
y,
yerr=yerr,
label='data',
marker='o',
linestyle='None')
plt.ylabel(param_names[j] + ' ' + param_units[j])
plt.legend(loc='best')
plt.xlabel('DAC')
plt.figure()
plt.plot(y_hat, (upper_band - lower_band) / y_hat / 2., label='kmpfit')
plt.plot(y_fit, sig_yi / y_fit, label='polyfit')
plt.plot(y_fit, (y_fit_max - y_fit_min) / y_fit / 2.,
label='polyfit max-min')
plt.plot(mu, sigma_mu / mu, label='from mpe fits')
plt.plot(y_fit, 1. / np.sqrt(y_fit), label='Poisson')
#plt.plot(y_fit, sigma_y / y_fit, label='kpmfit redo')
plt.xlabel('$\mu$')
plt.ylabel('${\sigma} / {\mu}$')
plt.legend(loc='best')
#plt.figure()
#plt.errorbar(y_fit, mu, xerr=sig_yi, yerr=sigma_mu)
#plt.xlabel('incoming light [p.e.]')
#plt.ylabel('measured light [p.e.]')
#plt.legend(loc='best')
plt.figure()
plt.errorbar(x,
mu,
yerr=sigma_mu,
label='LED calibration',
marker='o',
linestyle='None',
color='k')
plt.plot(x_fit, y_fit, color='r', label='best fit')
plt.fill_between(x_fit,
y_fit + sig_yi,
y_fit - sig_yi,
alpha=0.25,
facecolor='red',
label='1 $\sigma$ confidence level')
plt.xlabel('DAC')
plt.ylabel('$\mu$ [p.e.]')
plt.legend(loc='best')
plt.figure()
#plt.plot(x_fit, (upper_band - lower_band) / y_hat / 2., label='kmpfit')
plt.plot(x_fit, sig_yi / y_fit, label='polyfit')
#plt.plot(x_fit, (y_fit_max - y_fit_min) / y_fit / 2., label='polyfit max-min')
#plt.plot(x_fit, sigma_y / y_fit, label='kpmfit redo')
plt.xlabel('DAC')
plt.ylabel('${\sigma} / {\mu}$')
plt.legend(loc='best')
#=============================================================================== #============================Start the Fitting================================== #=============================================================================== dx = np.arange(xLowerLimit, xUpperLimit, 0.1) # if 'xLowerLimit' in Data[DictItem] and 'xUpperLimit' in Data[DictItem]: # dx = np.arange(Data[DictItem]['xLowerLimit'], Data[DictItem]['xUpperLimit'] + 500, 0.1) # else: # dx = np.arange(200, 10000, 0.1) #do the fitting and plot it if bool(Data[DictItem]['yColErr']): fitting_obj = kmpfit.simplefit(alpha, [ 0, 0 ], Data['x'], Data['y'], err = Data['yErr']) # p = np.polyfit(Data['x'], Data['y'], 2, w = Data['yErr']) else: fitting_obj = kmpfit.simplefit(alpha, [ 0, 0 ], Data['x'], Data['y']) # p = np.polyfit(Data['x'], Data['y'], 2) # add the fit line plt.plot(dx, alpha(fitting_obj.params, dx), c = 'black') # # add in a polynomial fit # PolyFit = np.poly1d(p) # # show the poly fit # x_new = np.linspace(Data['x'][0], Data['x'][-1], 150) # y_new = PolyFit(x_new) # plt.plot(Data['x'],Data['y'],'o', x_new, y_new, c = 'blue')
a, b = p
y = a + b * x
return y
# Artificial data
N = 50 # Number of data points
mean = 0.0
sigma = 0.6 # Characteristics of the noise we add
x = numpy.linspace(2, 10, N)
paramsreal = [1.0, 1.0]
noise = numpy.random.normal(mean, sigma, N)
y = model(paramsreal, x) + noise
err = numpy.random.normal(mean, sigma, N)
# Simple interface
p0 = (0, 0)
xl = list(range(10))
yl = [k * 0.5 for k in xl]
fitobj = kmpfit.simplefit(model, p0, xl, yl)
print("Best fit parameters:", fitobj.params)
print("Parameter errors: :", fitobj.stderr)
fitobj = kmpfit.simplefit(model, p0, x, y, err=err, xtol=1e-8)
print("Best fit parameters:", fitobj.params)
print("Parameter errors: :", fitobj.xerror)
fitobj = kmpfit.simplefit(model, p0, x, y, maxiter=100)
print("Best fit parameters:", fitobj.params)
print("Parameter errors: :", fitobj.stderr)
a, b = p
y = a + b*x
return y
# Artificial data
N = 50 # Number of data points
mean = 0.0; sigma = 0.6 # Characteristics of the noise we add
x = numpy.linspace(2, 10, N)
paramsreal = [1.0, 1.0]
noise = numpy.random.normal(mean, sigma, N)
y = model(paramsreal, x) + noise
err = numpy.random.normal(mean, sigma, N)
# Simple interface
p0 = (0,0)
xl = list(range(10))
yl = [k*0.5 for k in xl]
fitobj = kmpfit.simplefit(model, p0, xl, yl)
print("Best fit parameters:", fitobj.params)
print("Parameter errors: :", fitobj.stderr)
fitobj = kmpfit.simplefit(model, p0, x, y, err=err, xtol=1e-8)
print("Best fit parameters:", fitobj.params)
print("Parameter errors: :", fitobj.xerror)
fitobj = kmpfit.simplefit(model, p0, x, y, maxiter=100)
print("Best fit parameters:", fitobj.params)
print("Parameter errors: :", fitobj.stderr)
def getCurveFit(x,
y,
fitDomainMin=None,
fitDomainMax=None,
polyParams=[0, 0, 0, 0, 1]):
# Sort Arrays
sortIndexes = np.argsort(x)
x = [x[i] for i in sortIndexes]
y = [y[i] for i in sortIndexes]
# Get Fit Domain Limits
if fitDomainMin == None:
fitDomainMin = min(x)
if fitDomainMax == None:
fitDomainMax = max(x)
# Filter X and Y Arrays
xFiltered = []
yFiltered = []
for i in range(len(x)):
if x[i] >= float(fitDomainMin) and x[i] <= float(fitDomainMax):
xFiltered.append(float(x[i]))
yFiltered.append(float(y[i]))
xFiltered = np.array(xFiltered)
yFiltered = np.array(yFiltered)
# Fit
fit = kmpfit.simplefit(model, [0, 0, 0, 0, 0],
xFiltered,
yFiltered,
parinfo=[{
'fixed': polyParams[0]
}, {
'fixed': polyParams[1]
}, {
'fixed': polyParams[2]
}, {
'fixed': polyParams[3]
}, {
'fixed': polyParams[4]
}])
a, b, c, d, e = fit.params
# Get Error Bands
dfdp = [0, xFiltered, xFiltered**2, xFiltered**3, xFiltered**4]
yFit, dyFitUpper, dyFitLower = fit.confidence_band(xFiltered, dfdp, 0.95,
model)
samplingIndexes = [int(x) for x in np.linspace(0, len(xFiltered) - 1, 100)]
if isinstance(xFiltered[0], (int, np.int64)):
xFiltered = [int(x) for x in xFiltered]
if isinstance(yFiltered[0], (int, np.int64)):
yFiltered = [int(x) for x in yFiltered]
xFilteredSample = [xFiltered[i] for i in samplingIndexes]
yFilteredSample = [yFit[i] for i in samplingIndexes]
dyFitLowerSample = [dyFitLower[i] for i in samplingIndexes]
dyFitUpperSample = [dyFitUpper[i] for i in samplingIndexes]
return [{
'x': xFilteredSample[i],
'y': yFilteredSample[i],
'errorLower': dyFitLowerSample[i],
'errorUpper': dyFitUpperSample[i]
} for i in range(len(samplingIndexes))]
def fit_monthly(alldarks, orbit, verbose=False, kappasigma=False, debug_pixnr=None, short=False, give_errors=False, **kwargs):
"""
fit dark model to two neighbouring monthly calibration orbits
Parameters
----------
alldarks : AllDarks object
used to get underlying dark states from sdmf database in an easy and unambiguous way
orbit : int
absolute orbit number
verbose : bool, optional
if True, provide verbose output
give_errors: bool, optional
if True, return errors of fit parameters
kappasigma : bool, optional
if True, use kappasigma filter
"""
# minimum nr of degrees of freedom
min_degrees = 8
orbit_range = orbit-2, orbit+1.99
if verbose:
print(orbit_range)
n_exec, all_state_phases, pet, coadd, all_readouts, all_sigmas, ephases = alldarks.get_range(orbit_range)
idxje = pet > .5
if np.sum(idxje) == 0:
print("warn: no 1.0s pet")
#
# fit it
#
x = ephases-orbit, pet #, coadd
if verbose:
print(pet.shape)
print(pet)
print(coadd)
# small pets should not be taken very seriously, unless specified or the rest is saturated..
if not short:
idx = pet < .49
if np.sum(idx) > 0:
idx = np.where(idx)[0]
all_sigmas[idx,:] *= 50
aoinfo = dict(fixed=False, limits=[0,10000])
dcinfo = dict(fixed=False, limits=[-10000.,+500000.])
amp1info = dict(fixed=False, limits=[-200,+200])
trendinfo = dict(fixed=False, limits=[-100,+100])
amp2info = dict(fixed=False, limits=[-.4,+.4])
phase1info = dict(fixed=False, limits=[-3.,+3.])
phase2info = dict(fixed=False, limits=[-3.,+3.])
parinfo = [aoinfo,dcinfo,amp1info,trendinfo,phase1info,amp2info,phase2info]
# prepare initial parameters
ao0 = 3000
dc0 = 4000
amp1_0 = 0
trend0 = 0
amp2_0 = 0.1
phase_offset1 = 0
phase_offset2 = 0
p0 = np.array([ao0, dc0, amp1_0, trend0, phase_offset1, amp2_0, phase_offset2])
# ..and fit
n_done = 0
statuses = np.zeros(n_pix)
res_phases = np.zeros(n_pix)
res_phases2 = np.zeros(n_pix)
err_phase1 = np.zeros(n_pix)
err_phase2 = np.zeros(n_pix)
for pixnr in range(n_pix):
pix_readouts = all_readouts[:,pixnr]
pix_sigmas = all_sigmas[:,pixnr]
idx_fin = np.isfinite(pix_readouts) & (pix_sigmas > 0)
if np.sum(idx_fin) > min_degrees:
pix_readouts = pix_readouts[idx_fin]
pix_sigmas = pix_sigmas[idx_fin]
x = (ephases-orbit)[idx_fin], pet[idx_fin]
n = pix_readouts.size
# pass a
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
residual = scia_dark_fun2(fitobj.params, x) - pix_readouts
dev_residual = np.std(residual)
avg_residual = np.mean(residual)
if kappasigma:
# pass b : coarse kappa sigma filter
idx = (residual-avg_residual)**2 > 10.*dev_residual
if np.sum(idx) > 0:
pix_sigmas[idx] *= 500
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
residual = scia_dark_fun2(fitobj.params, x) - pix_readouts
dev_residual = np.std(residual)
avg_residual = np.mean(residual)
# pass c : finer kappa sigma filter
idx = (residual-avg_residual)**2 > 5.*dev_residual
if np.sum(idx) > 0:
pix_sigmas[idx] *= 500
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
n_done += 1
else:
continue
if verbose:
print(pixnr, dev_residual, fitobj.params[4] % 1, fitobj.message)
statuses[pixnr] = fitobj.status
res_phases[pixnr] = fitobj.params[4]
res_phases2[pixnr] = fitobj.params[6]
err_phase1[pixnr] = fitobj.xerror[4]
err_phase2[pixnr] = fitobj.xerror[6]
if (fitobj.status <= 0):
raise FitFailedError('Error message = '+fitobj.message)
# center the negative phase shift
idx_neg = res_phases > 0.5
res_phases[idx_neg] -= 0.5
idx_neg = res_phases2 > 0.5
res_phases2[idx_neg] -= 0.5
# compute channel phase shift
channel_phase1 = np.median(res_phases) % 1
channel_phase2 = np.median(res_phases2) % 1
np.sort(res_phases)
if verbose:
print('channel median phase =', channel_phase1, channel_phase2)
phase1info = dict(fixed=True, limits=[-3.,+3.])
phase2info = dict(fixed=True, limits=[-3.,+3.])
parinfo = [aoinfo,dcinfo,amp1info,trendinfo,phase1info,amp2info,phase2info]
p0[4] = channel_phase1
p0[6] = channel_phase2
#
# pass 2 - fix phase shift
#
aos = np.zeros(n_pix)
lcs = np.zeros(n_pix)
amps = np.zeros(n_pix)
amps2 = np.zeros(n_pix)
trends = np.zeros(n_pix)
err_aos = np.zeros(n_pix)
err_lcs = np.zeros(n_pix)
err_amps = np.zeros(n_pix)
err_amps2 = np.zeros(n_pix)
err_trends = np.zeros(n_pix)
statuses = np.zeros(n_pix)
for pixnr in range(n_pix):
pix_readouts = all_readouts[:,pixnr]
pix_sigmas = all_sigmas[:,pixnr]
idx_fin = np.isfinite(pix_readouts) & (pix_sigmas > 0)
if np.sum(idx_fin) > min_degrees:
pix_readouts = pix_readouts[idx_fin]
pix_sigmas = pix_sigmas[idx_fin]
x = (ephases-orbit)[idx_fin], pet[idx_fin]
n = pix_readouts.size
# pass a
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
residual = scia_dark_fun2(fitobj.params, x) - pix_readouts
dev_residual = np.std(residual)
avg_residual = np.mean(residual)
if kappasigma:
# pass b : coarse kappa sigma filter
idx = (residual-avg_residual)**2 > 10.*dev_residual
if np.sum(idx) > 0:
pix_sigmas[idx] *= 500
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
residual = scia_dark_fun2(fitobj.params, x) - pix_readouts
dev_residual = np.std(residual)
avg_residual = np.mean(residual)
# pass c : finer kappa sigma filter
idx = (residual-avg_residual)**2 > 5.*dev_residual
if np.sum(idx) > 0:
pix_sigmas[idx] *= 500
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
n_done += 1
else:
continue
if verbose:
print(pixnr, fitobj.message)
statuses[pixnr] = fitobj.status
aos[pixnr] = fitobj.params[0]
lcs[pixnr] = fitobj.params[1]
amps[pixnr] = fitobj.params[2]
trends[pixnr] = fitobj.params[3]
amps2[pixnr] = fitobj.params[5]
if (fitobj.status <= 0):
print('Error message = ', fitobj.message)
quit()
err_aos[pixnr] = fitobj.xerror[0]
err_lcs[pixnr] = fitobj.xerror[1]
err_amps[pixnr] = fitobj.xerror[2]
err_trends[pixnr] = fitobj.xerror[3]
err_amps2[pixnr] = fitobj.xerror[5]
channel_amp2 = np.median(amps2[np.where(statuses > 0)])
if debug_pixnr is not None:
import matplotlib.pyplot as plt
plt.ticklabel_format(useOffset=False)
# compute errors
n_bins = 100
x_bins = np.linspace(orbit_range[0], orbit_range[1], num=n_bins) - orbit
err_ds = np.empty([n_bins], dtype=np.float64)
for i in range(n_bins):
phi = x_bins[i]
err_ds[i] = err_lcs[debug_pixnr]**2 \
+ err_amps[debug_pixnr]**2 * (cos(2*pi*(res_phases[debug_pixnr]+phi)) + amps2[debug_pixnr]*(cos(4*pi*(res_phases2[debug_pixnr]+phi))))**2 \
+ err_phase1[debug_pixnr]**2 * (2*pi*amps[debug_pixnr]*sin(2*pi*(res_phases[debug_pixnr]+phi)))**2 \
+ err_amps2[debug_pixnr]**2 * (amps[debug_pixnr]*cos(4*pi*(res_phases2[debug_pixnr]+phi)))**2 \
+ err_phase2[debug_pixnr]**2 * (4*pi*amps[debug_pixnr]*amps2[debug_pixnr]*sin(4*pi*(res_phases2[debug_pixnr]+phi)))**2 \
+ err_trends[debug_pixnr]**2 * phi**2
err_ds = np.sqrt(err_ds)
plt.cla()
plt.suptitle("vardark of pixel "+str(debug_pixnr));
p = aos[debug_pixnr], lcs[debug_pixnr], amps[debug_pixnr], trends[debug_pixnr], channel_phase1, amps2[debug_pixnr], channel_phase2
plt.errorbar(ephases-orbit, all_readouts[:,debug_pixnr], yerr=all_sigmas[:,debug_pixnr], fmt='bo', label="data")
x = x_bins, np.array((0.5-petcorr) + np.zeros(n_bins))
model = scia_dark_fun2(p,x)
plt.plot(x_bins, model, 'g-', label="model", lw=2)
plt.plot(x_bins, model+err_ds, 'g-', label="model hi")
plt.plot(x_bins, model-err_ds, 'g-', label="model lo")
x = x_bins, np.array((1.0-petcorr) + np.zeros(n_bins))
model = scia_dark_fun2(p,x)
plt.plot(x_bins, model, 'g-', label="model", lw=2)
plt.plot(x_bins, model+err_ds, 'g-', label="model hi")
plt.plot(x_bins, model-err_ds, 'g-', label="model lo")
x = x_bins, np.array((0.125-petcorr) + np.zeros(n_bins))
model = scia_dark_fun2(p,x)
plt.plot(x_bins, model, 'g-', label="model", lw=2)
plt.plot(x_bins, model+err_ds, 'g-', label="model hi")
plt.plot(x_bins, model-err_ds, 'g-', label="model lo")
plt.show()
if give_errors:
errors = {"aos":err_aos,
"off":err_lcs,
"amps":err_amps,
"amps2":err_amps2,
"phase1":err_phase1,
"phase2":err_phase2,
"trends":err_trends}
return channel_phase1, channel_phase2, aos, lcs, amps, channel_amp2, trends, errors
else:
return channel_phase1, channel_phase2, aos, lcs, amps, channel_amp2, trends
def fit_eclipse_orbit(alldarks, orbit, aos, lcs, amps, amp2, channel_phaseshift, channel_phaseshift2,
give_errors=False, verbose=False, short=False, kappasigma=False, **kwargs):
"""
fit dark model to an orbits with normal (eclipse) dark states
use parameters computed from nearest calibration orbits
Parameters
----------
alldarks : AllDarks object
used to get underlying dark states from sdmf database in an easy and unambiguous way
orbit : int
absolute orbit number
aos : numpy array, 1024 floats
analog offset per pixel for this orbit
lcs : numpy array, 1024 floats
dark current (constant part) per pixel for this orbit
amps : numpy array, 1024 floats
orbital variation amplitude per pixel for this orbit
channel_amp2 : float
amplitude of first harmonic of orbital variation (channel mean)
channel_phaseshift : float
phase shift of orbital variation (channel mean)
channel_phaseshift2 : float
phase shift of first harmonic orbital variation (channel mean)
give_errors : bool, optional
if set, return also fit uncertainty
verbose : bool, optional
if set, provide verbose output
kappasigma : bool, optional
if set, use kappasigma filter
"""
# minimum nr of degrees of freedom
min_degrees = 3
#
# get all dark data
#
orbit_range = orbit-1, orbit+0.99
n_exec, all_state_phases, pet, coadd, all_readouts, all_sigmas, ephases = alldarks.get_range(orbit_range)
if ephases.size < min_degrees:
raise NotEnoughDataError("Not enough datapoints for fit: "+str(ephases.size))
#
# initialize data points (coadd is always 1 as we only do ch8 nadir)
#
x = ephases-orbit, pet #, coadd
# small pets should not be taken veryseriously, unless specified or the rest is saturated..
if not short:
idx = pet < .49
if np.sum(idx) > 0:
idx = np.where(idx)[0]
all_sigmas[idx,:] *= 50
#
# setup attributes of fit parameters
#
# note the limits are just slightly wider than in the monthly fit. we do this to get rid of float32->float64 conversion errors!
aoinfo = dict(fixed=True, limits=[-0.1,10000.1])
dcinfo = dict(fixed=False, limits=[-10000.1,+500000.1])
amp1info = dict(fixed=True, limits=[-200.1,+200.1])
trendinfo = dict(fixed=False, limits=[-100.1,+100.1])
amp2info = dict(fixed=True, limits=[-.41,+.41])
phase1info = dict(fixed=True, limits=[-3.01,+3.01])
phase2info = dict(fixed=True, limits=[-3.01,+3.01])
parinfo = [aoinfo,dcinfo,amp1info,trendinfo,phase1info,amp2info,phase2info]
#
# setup result data arrays
#
n_done = 0
statuses = np.zeros(n_pix)
res_trends = np.empty(n_pix)
res_lcs = np.empty(n_pix)
res_trends[:] = np.nan
res_lcs[:] = np.nan
err_trends = np.empty(n_pix)
err_lcs = np.empty(n_pix)
err_trends[:] = np.nan
err_lcs[:] = np.nan
uncertainty = np.empty(n_pix)
uncertainty[:] = np.nan
#
# ..and fit
#
for pixnr in range(n_pix):
# prepare initial parameters..
p0 = np.array([aos[pixnr], lcs[pixnr], amps[pixnr], 0, channel_phaseshift, amp2, channel_phaseshift2])
pix_readouts = all_readouts[:,pixnr]
pix_sigmas = all_sigmas[:,pixnr]
idx_fin = np.isfinite(pix_readouts) & (pix_sigmas > 0) & np.isfinite(pix_sigmas)
if np.sum(idx_fin) > min_degrees:
pix_readouts = pix_readouts[idx_fin]
pix_sigmas = pix_sigmas[idx_fin]
x = (ephases-orbit)[idx_fin], pet[idx_fin]
n = pix_readouts.size
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
residual = scia_dark_fun2(fitobj.params, x) - pix_readouts
dev_residual = np.std(residual)
avg_residual = np.mean(residual)
if kappasigma:
# pass b : coarse kappa sigma filter
idx = (residual-avg_residual)**2 > 10.*dev_residual
if np.sum(idx) > 0:
pix_sigmas[idx] *= 50
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
residual = scia_dark_fun2(fitobj.params, x) - pix_readouts
dev_residual = np.std(residual)
avg_residual = np.mean(residual)
# pass c : finer kappa sigma filter
idx = (residual-avg_residual)**2 > 5.*dev_residual
if np.sum(idx) > 0:
pix_sigmas[idx] *= 50
fitobj = kmpfit.simplefit(scia_dark_fun2, p0, x, pix_readouts, err=pix_sigmas, ftol=1e-10, parinfo=parinfo)
n_done += 1
else:
continue
if verbose:
print(pixnr, fitobj.message)
print(p0, x, pix_readouts, pix_sigmas)
if (fitobj.status <= 0):
raise FitFailedError(fitobj.message)
statuses[pixnr] = fitobj.status
res_lcs[pixnr] = fitobj.params[1]
err_lcs[pixnr] = fitobj.xerror[1]
res_trends[pixnr] = fitobj.params[3]
err_trends[pixnr] = fitobj.xerror[3]
uncertainty[pixnr] = np.std(scia_dark_fun2(fitobj.params, x) - pix_readouts)
if give_errors:
return x, res_lcs, res_trends, err_lcs, err_trends, all_readouts, all_sigmas, uncertainty
else:
return x, res_lcs, res_trends, all_readouts, all_sigmas
def extract_gain_info(data_set, ccd, node):
"""
create pha count profile from given data sets and fit model parameters
input: data_set --- a list of extracted fits data
ccd --- ccd #
node --- node #
output: d_list : [al_cent, mn_cent, ti_cent, b, a, be, ae]
al_cent --- center of al line
mn_cent --- center of mn line
ti_cent --- center of ti line
b --- slope of the fitted line
a --- intercept of the fitted line
be --- the error of the slope
ae --- the error of the intercept
"""
#
#--- create a histgram data for specified CCD / Node combination
#
hist = extract_hist_data(data_set, ccd, node)
#
#--- if the hist data is empty, just add an empty list for house keeping
#
if hist == 'na':
return []
#
#--- Mn K alpha
#
y = hist[1200:2500]
ymax = max(y)
xpos = y.index(ymax) + 1200
hwidth = 200
#
#--- if the count rate is too low, we cannot fit the data; so limit data larger than 20 counts
#
start = xpos - hwidth
stop = xpos + hwidth
ybin = hist[start:stop]
chk = 0
for ent in ybin:
chk += int(ent)
if chk < 20:
return []
[mn_amp, mn_cent, mn_width, floor] = fit_pmodel(hist, ymax, xpos, hwidth)
#
#--- AL K alpha
#
ymax = 0.5 * mn_amp
xpos = 0.25 * mn_cent
hwidth = 50
[al_amp, al_cent, al_width, floor] = fit_pmodel(hist, ymax, xpos, hwidth)
#
#--- Ti K alpha
#
ymax = 0.5 * mn_amp
xpos = 0.765 * mn_cent
hwidth = 100
[ti_amp, ti_cent, ti_width, floor] = fit_pmodel(hist, ymax, xpos, hwidth)
#
#--- fit a straight line
#
pos = [al_cent, ti_cent, mn_cent]
p0 = [0, 1]
fitobj = kmpfit.simplefit(lmodel, p0, ev, pos)
[a, b] = fitobj.params
[ae,be] = fitobj.stderr
d_list = [al_cent, mn_cent, ti_cent, b, a, be, ae]
return d_list
def extract_gain_info(data_set, ccd, node):
"""
create pha count profile from given data sets and fit model parameters
input: data_set --- a list of extracted fits data
ccd --- ccd #
node --- node #
output: d_list : [al_cent, mn_cent, ti_cent, b, a, be, ae]
al_cent --- center of al line
mn_cent --- center of mn line
ti_cent --- center of ti line
b --- slope of the fitted line
a --- intercept of the fitted line
be --- the error of the slope
ae --- the error of the intercept
"""
#
#--- create a histgram data for specified CCD / Node combination
#
hist = extract_hist_data(data_set, ccd, node)
#
#--- if the hist data is empty, just add an empty list for house keeping
#
if hist == 'na':
return []
#
#--- Mn K alpha
#
y = hist[1200:2500]
ymax = max(y)
xpos = y.index(ymax) + 1200
hwidth = 200
#
#--- if the count rate is too low, we cannot fit the data; so limit data larger than 20 counts
#
start = xpos - hwidth
stop = xpos + hwidth
ybin = hist[start:stop]
chk = 0
for ent in ybin:
chk += int(ent)
if chk < 20:
return []
[mn_amp, mn_cent, mn_width, floor] = fit_pmodel(hist, ymax, xpos, hwidth)
#
#--- AL K alpha
#
ymax = 0.5 * mn_amp
xpos = 0.25 * mn_cent
hwidth = 50
[al_amp, al_cent, al_width, floor] = fit_pmodel(hist, ymax, xpos, hwidth)
#
#--- Ti K alpha
#
ymax = 0.5 * mn_amp
xpos = 0.765 * mn_cent
hwidth = 100
[ti_amp, ti_cent, ti_width, floor] = fit_pmodel(hist, ymax, xpos, hwidth)
#
#--- fit a straight line
#
pos = [al_cent, ti_cent, mn_cent]
p0 = [0, 1]
fitobj = kmpfit.simplefit(lmodel, p0, ev, pos)
[a, b] = fitobj.params
[ae, be] = fitobj.stderr
d_list = [al_cent, mn_cent, ti_cent, b, a, be, ae]
return d_list