def test_pearson(self):
p_x = np.array([0.0, 0.9, 1.8, 2.6, 3.3, 4.4, 5.2, 6.1, 6.5, 7.4])
p_y = np.array([5.9, 5.4, 4.4, 4.6, 3.5, 3.7, 2.8, 2.8, 2.4, 1.5])
p_sx = np.array([0.03, 0.03, 0.04, 0.035, 0.07, 0.11, 0.13, 0.22, 0.74, 1.0])
p_sy = np.array([1.0, 0.74, 0.5, 0.35, 0.22, 0.22, 0.12, 0.12, 0.1, 0.04])
p_dat = RealData(p_x, p_y, sx=p_sx, sy=p_sy)
# Reverse the data to test invariance of results
pr_dat = RealData(p_y, p_x, sx=p_sy, sy=p_sx)
p_mod = Model(self.pearson_fcn, meta=dict(name="Uni-linear Fit"))
p_odr = ODR(p_dat, p_mod, beta0=[1.0, 1.0])
pr_odr = ODR(pr_dat, p_mod, beta0=[1.0, 1.0])
out = p_odr.run()
assert_array_almost_equal(out.beta, np.array([5.4767400299231674, -0.4796082367610305]))
assert_array_almost_equal(out.sd_beta, np.array([0.3590121690702467, 0.0706291186037444]))
assert_array_almost_equal(
out.cov_beta,
np.array([[0.0854275622946333, -0.0161807025443155], [-0.0161807025443155, 0.003306337993922]]),
)
rout = pr_odr.run()
assert_array_almost_equal(rout.beta, np.array([11.4192022410781231, -2.0850374506165474]))
assert_array_almost_equal(rout.sd_beta, np.array([0.9820231665657161, 0.3070515616198911]))
assert_array_almost_equal(
rout.cov_beta,
np.array([[0.6391799462548782, -0.1955657291119177], [-0.1955657291119177, 0.0624888159223392]]),
)
def do_bestfit(self):
"""
do bestfit using scipy.odr
"""
self.check_important_variables()
x = np.array(self.args["x"])
y = np.array(self.args["y"])
if self.args.get("use_RealData", True):
realdata_kwargs = self.args.get("RealData_kwargs", {})
data = RealData(x, y, **realdata_kwargs)
else:
data_kwargs = self.args.get("Data_kwargs", {})
data = Data(x, y, **data_kwargs)
model = self.args.get("Model", None)
if model is None:
if "func" not in self.args.keys():
raise KeyError("Need fitting function")
model_kwargs = self.args.get("Model_kwargs", {})
model = Model(self.args["func"], **model_kwargs)
odr_kwargs = self.args.get("ODR_kwargs", {})
odr = ODR(data, model, **odr_kwargs)
self.output = odr.run()
if self.args.get("pprint", False):
self.output.pprint()
self.fit_args = self.output.beta
return self.fit_args
def test_explicit(self):
explicit_mod = Model(
self.explicit_fcn,
fjacb=self.explicit_fjb,
fjacd=self.explicit_fjd,
meta=dict(name='Sample Explicit Model',
ref='ODRPACK UG, pg. 39'),
)
explicit_dat = Data([0.,0.,5.,7.,7.5,10.,16.,26.,30.,34.,34.5,100.],
[1265.,1263.6,1258.,1254.,1253.,1249.8,1237.,1218.,1220.6,
1213.8,1215.5,1212.])
explicit_odr = ODR(explicit_dat, explicit_mod, beta0=[1500.0, -50.0, -0.1],
ifixx=[0,0,1,1,1,1,1,1,1,1,1,0])
explicit_odr.set_job(deriv=2)
out = explicit_odr.run()
assert_array_almost_equal(
out.beta,
np.array([ 1.2646548050648876e+03, -5.4018409956678255e+01,
-8.7849712165253724e-02]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([ 1.0349270280543437, 1.583997785262061 , 0.0063321988657267]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[ 4.4949592379003039e-01, -3.7421976890364739e-01,
-8.0978217468468912e-04],
[ -3.7421976890364739e-01, 1.0529686462751804e+00,
-1.9453521827942002e-03],
[ -8.0978217468468912e-04, -1.9453521827942002e-03,
1.6827336938454476e-05]]),
)
def ortho_regress(x, y):
linreg = linregress(x, y)
mod = Model(f)
dat = Data(x, y)
od = ODR(dat, mod, beta0=linreg[0:2])
out = od.run()
#print(list(out.beta))
#return list(out.beta) + [np.nan, np.nan, np.nan]
return(list(out.beta))
def orth_regression(model,obs):
linear = Model(f)
mydata = RealData(obs, model)
myodr = ODR(mydata, linear, beta0=[1., 0.])
myoutput = myodr.run()
params = myoutput.beta
gradient = params[0]
y_intercept = params[1]
res_var = myoutput.res_var
return np.around(gradient,2), np.around(y_intercept,2), np.around(res_var,2)
def odrlin(x,y, sx, sy):
"""
Linear fit of 2-D data set made with Orthogonal Distance Regression
@params x, y: data to fit
@param sx, sy: respective errors of data to fit
"""
model = models.unilinear # defines model as beta[0]*x + beta[1]
data = RealData(x,y,sx=sx,sy=sy)
kinit = (y[-1]-y[0])/(x[-1]-x[0])
init = (kinit, y[0]-kinit*x[0])
linodr = ODR(data, model, init)
return linodr.run()
def _run_odr(self):
"""Run an ODR regression"""
linear = Model(self._modelODR)
mydata = Data(ravel(self._datax), ravel(self._datay), 1)
myodr = ODR(mydata, linear, beta0=self._guess, maxit=10000)
myoutput = myodr.run()
self._result = myoutput.beta
self._stdev = myoutput.sd_beta
self._covar = myoutput.cov_beta
self._odr = myoutput
def test_ifixx(self):
x1 = [-2.01, -0.99, -0.001, 1.02, 1.98]
x2 = [3.98, 1.01, 0.001, 0.998, 4.01]
fix = np.vstack((np.zeros_like(x1, dtype=int), np.ones_like(x2, dtype=int)))
data = Data(np.vstack((x1, x2)), y=1, fix=fix)
model = Model(lambda beta, x: x[1, :] - beta[0] * x[0, :]**2., implicit=True)
odr1 = ODR(data, model, beta0=np.array([1.]))
sol1 = odr1.run()
odr2 = ODR(data, model, beta0=np.array([1.]), ifixx=fix)
sol2 = odr2.run()
assert_equal(sol1.beta, sol2.beta)
def test_ticket_1253(self):
def linear(c, x):
return c[0]*x+c[1]
c = [2.0, 3.0]
x = np.linspace(0, 10)
y = linear(c, x)
model = Model(linear)
data = Data(x, y, wd=1.0, we=1.0)
job = ODR(data, model, beta0=[1.0, 1.0])
result = job.run()
assert_equal(result.info, 2)
def slope(x, y, xerr, yerr, verbose=True, null=np.double(-9.99999488e+08)):
"""
Calculates the slope of a color trajectory.
Uses the scipy ODRpack and total least squares algorithm.
Parameters
----------
x, y : array-like
Colors or magnitudes to calculate slopes of.
xerr, yerr : array-like
Corresponding errors (stdev) on `x` and `y`.
Be sure to make sure that the HMKPNT errors are properly adjusted!
verbose : bool. optional
Whether to print a verbose output. Default true.
Returns
-------
slope : float
Slope (in rise/run) of the linear fit.
intercept : float
Y-value where the linear fit intercepts the Y-axis.
slope_error : float
The standard error on the fitted slope: an indication of fit quality.
Notes
-----
Much of this code is borrowed from the dosctring example found
at https://github.com/scipy/scipy/blob/master/scipy/odr/odrpack.py#L27 .
See http://docs.scipy.org/doc/scipy/reference/odr.html and
http://stackoverflow.com/questions/9376886/orthogonal-regression-fitting-in-scipy-least-squares-method
for further discussion.
"""
if (len(x) != len(y)) or len(x) == 0 or len(y) == 0:
return null, null, null
mydata = RealData(x, y, sx=xerr, sy=yerr)
# Someday, we may want to improve the "initial guess" with
# a leastsq first-pass.
myodr = ODR(mydata, linear, beta0=[1.,2.])
myoutput = myodr.run()
if verbose:
print myoutput.pprint()
return myoutput.beta[0], myoutput.beta[1], myoutput.sd_beta[0]
def fit_odr(self, t_scale, xmin=-sys.maxint, xmax=sys.maxint):
# initial guess for the parameters
params_initial = [1e-4, 1.0] #np.array([1e-4, 1.0]) #a, b
# function to fit
def f(B, x):
return B[0]*pow(x/t_scale, -B[1])
powerlaw = Model(f)
def powerlaw_err(x, a, b, aerr, berr, cov):
val = f([a,b], x)#f(x, a, b)
err = np.sqrt(pow(aerr/a, 2)+pow(np.log(x/t_scale), 2)*cov)#*val
return err
xdata = []
ydata = []
yerr = []
for ipoint, xpoint in enumerate(self.get_xdata()):
if xpoint>=xmin and xpoint<xmax:
xdata.append(xpoint)
ydata.append(self.get_ydata()[ipoint])
yerr.append(np.sqrt(self.get_yerr()[0][ipoint]*self.get_yerr()[1][ipoint]))
xdata = np.array(xdata)
ydata = np.array(ydata)
yerr = np.array(yerr)
if len(xdata)<3:
logging.info('Only {0} data points. Fitting is skipped.'.format(len(xdata)))
return 1
mydata = RealData(x=xdata, y=ydata, sy=yerr)
myodr = ODR(mydata, powerlaw, beta0=params_initial)
myoutput = myodr.run()
myoutput.pprint()
params_optimal = myoutput.beta
cov = myoutput.cov_beta
params_err = myoutput.sd_beta
#logging.info("""Opimized parameters: {0}
#Error: {1}""".format(params_optimal, params_err))
#params_err = np.sqrt(np.diag(cov))
for iparam, param, in enumerate(params_optimal):
logging.info("""Parameter {0}: {1} +/- {2}""".format(iparam, params_optimal[iparam], params_err[iparam]))
x_draw = np.linspace(min(xdata), max(xdata), 100)
y_draw = np.zeros_like(x_draw)
yerr_draw = np.zeros_like(y_draw)
for ix, x in enumerate(x_draw):
y_draw[ix] = f(params_optimal, x) #f(x, params_optimal[0], params_optimal[1])
yerr_draw[ix] = powerlaw_err(x, params_optimal[0], params_optimal[1], params_err[0], params_err[1], cov[1][1])
return ((params_optimal, params_err), (x_draw, y_draw, yerr_draw))
def calculate_ortho_regression(x, y, samples):
sx = numpy.cov(x, y)[0][0]
sy = numpy.cov(x, y)[1][1]
linear = Model(func)
# data = Data(x, y, wd=1./pow(sx, 2), we=1./pow(sy, 2))
data = Data(x, y)
odr = ODR(data, linear, beta0=[1., 2.])
out = odr.run()
print '\n'
out.pprint()
return (out.beta[0], out.beta[1], out.res_var)
def scipyODR(recipe, *args, **kwargs):
from scipy.odr import Data, Model, ODR, RealData, odr_stop
# FIXME
# temporarily change _weights to _weights**2 to fit the ODR fits
a = [w ** 2 for w in recipe._weights]
recipe._weights = a
model = Model(recipe.evaluateODR,
# implicit=1,
meta=dict(name='ODR fit'),
)
x = [recipe._contributions.values()[0].profile.x]
y = [recipe._contributions.values()[0].profile.y]
dy = [recipe._contributions.values()[0].profile.dy]
cont = recipe._contributions.values()
for i in range(1, len(cont)):
xplus = x[-1][-1] - x[-1][-2] + x[-1][-1]
x.append(cont[i].profile.x + x[-1][-1] + xplus)
y.append(cont[i].profile.y)
dy.append(cont[i].profile.dy)
x.append(np.arange(len(recipe._restraintlist))
* (x[-1][-1] - x[-1][-2]) + x[-1][-1])
y.append(np.zeros_like(recipe._restraintlist))
dy = np.concatenate(dy)
dy = np.concatenate(
[dy, np.ones_like(recipe._restraintlist) + np.average(dy)])
data = RealData(x=np.concatenate(x), y=np.concatenate(y), sy=dy)
odr_kwargs = {}
if kwargs.has_key('maxiter'):
odr_kwargs['maxit'] = kwargs['maxiter']
odr = ODR(data, model, beta0=recipe.getValues(), **odr_kwargs)
odr.set_job(deriv=1)
out = odr.run()
# out.pprint()
# FIXME
# revert back
a = [np.sqrt(w) for w in recipe._weights]
recipe._weights = a
return {'x': out.beta,
'esd': out.sd_beta,
'cov': out.cov_beta,
'raw': out,
}
def test_implicit(self):
implicit_mod = Model(
self.implicit_fcn,
implicit=1,
meta=dict(name='Sample Implicit Model',
ref='ODRPACK UG, pg. 49'),
)
implicit_dat = Data([
[0.5,1.2,1.6,1.86,2.12,2.36,2.44,2.36,2.06,1.74,1.34,0.9,-0.28,
-0.78,-1.36,-1.9,-2.5,-2.88,-3.18,-3.44],
[-0.12,-0.6,-1.,-1.4,-2.54,-3.36,-4.,-4.75,-5.25,-5.64,-5.97,-6.32,
-6.44,-6.44,-6.41,-6.25,-5.88,-5.5,-5.24,-4.86]],
1,
)
implicit_odr = ODR(implicit_dat, implicit_mod,
beta0=[-1.0, -3.0, 0.09, 0.02, 0.08])
out = implicit_odr.run()
assert_array_almost_equal(
out.beta,
np.array([-0.9993809167281279, -2.9310484652026476, 0.0875730502693354,
0.0162299708984738, 0.0797537982976416]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([0.1113840353364371, 0.1097673310686467, 0.0041060738314314,
0.0027500347539902, 0.0034962501532468]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[2.1089274602333052e+00, -1.9437686411979040e+00,
7.0263550868344446e-02, -4.7175267373474862e-02,
5.2515575927380355e-02],
[-1.9437686411979040e+00, 2.0481509222414456e+00,
-6.1600515853057307e-02, 4.6268827806232933e-02,
-5.8822307501391467e-02],
[7.0263550868344446e-02, -6.1600515853057307e-02,
2.8659542561579308e-03, -1.4628662260014491e-03,
1.4528860663055824e-03],
[-4.7175267373474862e-02, 4.6268827806232933e-02,
-1.4628662260014491e-03, 1.2855592885514335e-03,
-1.2692942951415293e-03],
[5.2515575927380355e-02, -5.8822307501391467e-02,
1.4528860663055824e-03, -1.2692942951415293e-03,
2.0778813389755596e-03]]),
)
def test_lorentz(self):
l_sy = np.array([.29]*18)
l_sx = np.array([.000972971,.000948268,.000707632,.000706679,
.000706074, .000703918,.000698955,.000456856,
.000455207,.000662717,.000654619,.000652694,
.000000859202,.00106589,.00106378,.00125483, .00140818,.00241839])
l_dat = RealData(
[3.9094, 3.85945, 3.84976, 3.84716, 3.84551, 3.83964, 3.82608,
3.78847, 3.78163, 3.72558, 3.70274, 3.6973, 3.67373, 3.65982,
3.6562, 3.62498, 3.55525, 3.41886],
[652, 910.5, 984, 1000, 1007.5, 1053, 1160.5, 1409.5, 1430, 1122,
957.5, 920, 777.5, 709.5, 698, 578.5, 418.5, 275.5],
sx=l_sx,
sy=l_sy,
)
l_mod = Model(self.lorentz, meta=dict(name='Lorentz Peak'))
l_odr = ODR(l_dat, l_mod, beta0=(1000., .1, 3.8))
out = l_odr.run()
assert_array_almost_equal(
out.beta,
np.array([1.4306780846149925e+03, 1.3390509034538309e-01,
3.7798193600109009e+00]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([7.3621186811330963e-01, 3.5068899941471650e-04,
2.4451209281408992e-04]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[2.4714409064597873e-01, -6.9067261911110836e-05,
-3.1236953270424990e-05],
[-6.9067261911110836e-05, 5.6077531517333009e-08,
3.6133261832722601e-08],
[-3.1236953270424990e-05, 3.6133261832722601e-08,
2.7261220025171730e-08]]),
)
def size_linewidth_slope(catalog):
"""
Measures a catalog's slope of sizes and linewidths using scipy.odr.
ODR means "orthogonal distance regression", and is a way of fitting
models to data where both the x and y values have scatter.
Parameters
----------
catalog : astropy.table.table.Table
Table containing columns for 'size' and 'v_rms'.
Returns
-------
odr_output : scipy.odr.odrpack.Output
Output of the scipy ODR routine. The fit parameters
are stored in odr_output.beta .
"""
if 'size' not in catalog.colnames or 'v_rms' not in catalog.colnames:
raise ValueError("'size' and 'v_rms' must be columns in `catalog`!")
if 'error_size_plus' not in catalog.colnames or 'error_size_minus' not in catalog.colnames:
mean_size_error = None
else:
mean_size_error = (catalog['error_size_plus']*catalog['error_size_minus'])**(1/2)
# if any clouds have no error in their size, the procedure will puke unless we do this:
mean_size_error[mean_size_error==0] = 0.01
size_linewidth_data = RealData(catalog['size'], catalog['v_rms'], sx=mean_size_error)
powerlaw_model = Model(powerlaw_function)
# The provided `beta0` is a throwaway guess that shouldn't change the outcome.
odr_object = ODR(size_linewidth_data, powerlaw_model, beta0=[1., 1.])
odr_output = odr_object.run()
return odr_output
def test_pearson(self):
p_x = np.array([0., .9, 1.8, 2.6, 3.3, 4.4, 5.2, 6.1, 6.5, 7.4])
p_y = np.array([5.9, 5.4, 4.4, 4.6, 3.5, 3.7, 2.8, 2.8, 2.4, 1.5])
p_sx = np.array([.03, .03, .04, .035, .07, .11, .13, .22, .74, 1.])
p_sy = np.array([1., .74, .5, .35, .22, .22, .12, .12, .1, .04])
p_dat = RealData(p_x, p_y, sx=p_sx, sy=p_sy)
# Reverse the data to test invariance of results
pr_dat = RealData(p_y, p_x, sx=p_sy, sy=p_sx)
p_mod = Model(self.pearson_fcn, meta=dict(name='Uni-linear Fit'))
p_odr = ODR(p_dat, p_mod, beta0=[1., 1.])
pr_odr = ODR(pr_dat, p_mod, beta0=[1., 1.])
out = p_odr.run()
assert_array_almost_equal(
out.beta,
np.array([5.4767400299231674, -0.4796082367610305]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([0.3590121690702467, 0.0706291186037444]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[0.0854275622946333, -0.0161807025443155],
[-0.0161807025443155, 0.003306337993922]]),
)
rout = pr_odr.run()
assert_array_almost_equal(
rout.beta,
np.array([11.4192022410781231, -2.0850374506165474]),
)
assert_array_almost_equal(
rout.sd_beta,
np.array([0.9820231665657161, 0.3070515616198911]),
)
assert_array_almost_equal(
rout.cov_beta,
np.array([[0.6391799462548782, -0.1955657291119177],
[-0.1955657291119177, 0.0624888159223392]]),
)
GCover6.append(i)
if datatype[i] == 3:
notGC.append(i)
### Defining variables used to plot and to work with ###
M = metal[index]
Merr = metal_err[index]
LxM = np.log10(lx_m[index])
err = lx_m[index] + lx_m_err[index]
LxMerr = np.log10(err) - LxM
xn = np.linspace(-2.5, 0.5, 100)
### Setting up the ODR process, the plot arrays and chi squared ###
data = RealData(M, LxM, sx=Merr, sy=LxMerr)
modelLM = Model(funcLin)
odrLM = ODR(data, modelLM, [28, 1])
odrLM.set_job(fit_type=0)
outLM = odrLM.run()
ynLinearMetal = funcLin(outLM.beta, xn)
Res = funcLin(outLM.beta, M) - LxM
StDev = Res / LxMerr
ChiSq = np.sum(((funcLin(outLM.beta, M) - LxM) / LxMerr)**2)
RedChiSq = ChiSq / (len(M) - len(outLM.beta))
### PLotting ###
fig7 = plt.figure(7)
plt.clf()
#frame1 = fig7.add_axes((0.1,0.3,0.8,0.6))
plt.errorbar(M,
LxM,
a, b, c = M
return a*pow((x/300), b)*np.exp(-c/x)
data1 = np.loadtxt('/home/labosat/Desktop/Finazzi-Ferreira/Labo-7/Mediciones/Graficos y txt/txts/data_rq_final.txt', skiprows=1)
T_celcius = (data1[:, 3] - 1000)/3.815
T = [T_celcius[i] + 273 for i in range(len(T_celcius))]
T_err = (data1[:, 4])/3.815
Rq = data1[:, 1]
Rq_err = data1[:, 2]
plt.figure(1)
plt.errorbar(T, Rq, xerr= T_err, yerr= Rq_err, fmt='k.', capsize= 3)
model = Model(Exp)
data = RealData(T, Rq, sx=T_err, sy=Rq_err)
odr = ODR(data, model, beta0=[400000., 1., -65], maxit=100000)
out = odr.run()
a = out.beta[0]
b = out.beta[1]
c = out.beta[2]
a_err = out.sd_beta[0]
b_err = out.sd_beta[1]
c_err = out.sd_beta[2]
chi2 = out.res_var
T_2 = copy.deepcopy(T)
T_3 = np.linspace(min(T), max(T), 2000)
T_2.sort()
#plt.plot(T_2, Exp(out.beta, T_2), lw=3)
plt.plot(T_3, Exp(out.beta, T_3), 'b-' ,lw = 3)
plt.xlabel('Temperatura (C)')
if datatype[i] == 2:
GCover6.append(i)
if datatype[i] == 3:
notGC.append(i)
LxM = np.log10(lx_m[index])
err = lx_m[index] + lx_m_err[index]
LxMerr = np.log10(err) - LxM
A = age[index]
Aerr = age_err[index]
data = RealData(A, LxM, sx=Aerr, sy=LxMerr)
xn = np.linspace(2, 13.5, 100)
modelLA = Model(funcLin)
odrLA = ODR(data, modelLA, [28, -2])
odrLA.set_job(fit_type=0)
outLA = odrLA.run()
ynLA = funcLin(outLA.beta, xn)
Res = funcLin(outLA.beta, A) - LxM
StDev = Res / LxMerr
ChiSq = np.sum(((funcLin(outLA.beta, A) - LxM) / LxMerr)**2)
RedChiSq = ChiSq / (len(A) - len(outLA.beta))
fig1 = plt.figure(1)
plt.clf()
#frame1 = fig1.add_axes((0.1,0.3,0.8,0.6))
plt.errorbar(A,
LxM,
yerr=LxMerr,
xerr=Aerr,
def R_Rq(path, tolerancia):
"""
Calcula la resistencia de quenching y la temperatura (escrita en funcion de la
resistencia que mide T) de un conjunto de mediciones, haciendo estadistica sobre los
datos.
Es decir, si mido muchas curvas IV para una dada T, guardo las mediciones en una carpeta
que contenga: una carpeta '/iv/' que contenga las curvas '/i (iv).txt' y otra carpeta
'/res/' que contenga las resistencias '/i (res).txt'.
Esta funcion va a calcular la Rq y la T para cada par (iv, res), y luego va a realizar
un promedio pesado sobre las Rq y las T, devolviendo dichos parametros con sus errores.
La funcion se va a encargar de filtrar aquellas mediciones en que la temperatura
fluctuo mas que la tolerancia deseada, utilizando al funcion pulidor(). La tolerancia
tipica es de 0.025 para mediciones estacionarias en T.
El path de la funcion debe ser la carpeta donde se encuentren las carpetas iv y res.
La funcion asume que la temperatura se midio con una RTD, sourceando corriente y
midiendo voltaje.
Input: (path, tolerancia) [string, float]
Returns: (R, R_err, Rq, Rq_err, chi2_out, array) [float, float, float, float, float, list]
.
.
"""
array = pulidor(tolerancia, path)
celdas = 18980
R = []
R_err = []
Rq = []
Rq_err = []
chi2_out = []
for i in array:
data1 = np.loadtxt(path+'/res/%s (res).txt' % i, skiprows=1)
Res = data1[:, 1]
I = data1[:, 2]
V = I*Res
V_err = error_V(V, source = False)
I_err = error_I(I, source = True)
Res_err_estadistico = dispersion(Res)
Res_err_sistematico = [np.sqrt((1/I[i]**2) * V_err[i]**2 + ((V[i]/(I[i]**2))**2)*I_err[i]**2) for i in range(len(I))]
Res_err = [np.sqrt(Res_err_estadistico**2 + Res_err_sistematico[i]**2) for i in range(len(Res_err_sistematico))]
R.append(weightedMean(Res, Res_err))
R_err.append(weightedError(Res, Res_err))
data2 = np.loadtxt(path+'/iv/%s (iv).txt' % i)
V = data2[:, 0]
I = data2[:, 1]
dI = error_I(I)
dV = error_V(V)
chi2 = []
Rq_err_temp = []
m = []
for j in range(0, len(V) - 2):
V_temp = V[j:]
I_temp = I[j:]
dI_temp = dI[j:]
dV_temp = dV[j:]
linear_model = Model(Linear)
data = RealData(V_temp, I_temp, sx=dV_temp, sy=dI_temp)
odr = ODR(data, linear_model, beta0=[0., 1.])
out = odr.run()
m_temp = out.beta[0]
#b_temp = out.beta[1]
m_err_temp = out.sd_beta[0]
m.append(m_temp)
chi2.append(out.res_var)
Rq_err_temp.append((celdas/m_temp**2) * m_err_temp)
index = ClosestToOne(chi2)
Rq.append(celdas/m[index])
chi2_out.append(chi2[index])
Rq_err.append(Rq_err_temp[index])
return R, R_err, Rq, Rq_err, chi2_out, array
y = beta[0] + beta[1] * x
return y
def linear_beta(x, beta_0, beta_1):
return beta_0 + beta_1 * x
intensity = np.array(As).reshape(9, 5).T[[0, 3, 4, 1, 2]].T # move peak 2 & 3 to the correct position
A_lambda = intensity[:-1].T # entries of each peak put together
peaks = ['558.3', '563.5', '564.7', '570.8', '576.7']
intersects = []
x = np.array(concentration[:-1])
sx = 4
for i, peak, A in enum(peaks, A_lambda):
y = un.nominal_values(A)
sy = un.std_devs(A)
data = RealData(x, y, sx=sx, sy=sy)
model = Model(func)
odr = ODR(data, model, [6, 0])
odr.set_job(fit_type=0)
output = odr.run()
beta = uc.correlated_values(output.beta, output.cov_beta)
fit = linear_beta(x_fit, *beta)
#Intersects
y0 = intensity[-1][i] # peak intensities of the original sample
x0 = (y0 - beta[0]) / beta[1]
intersects.append(x0)
lamb_all = np.array(x0s).reshape(9, 5)
d_lin = np.load(npy_dir + 'ccd_calibration.npy')
lamb_all = linear(lamb_all, *d_lin) # Integrate error on wavelength of CCD
lamb_mean = np.mean(lamb_all, 0)
lamb_laser = np.load(npy_dir + 'ccd_lamb_laser.npy')[0]
dnu = (1 / lamb_mean - 1 / lamb_laser) * 10**7
lamb_mean = np.sort(lamb_mean)
def fit_initial_param(bins_data, plt_data):
'''
This function computes the inizial parameter for the fit
from the data. It search V_floating, alpha and I ionic saturation.
It returns a tuple with (V_floating, alpha, i_ionic, figure, output_str).
-Figure is the plot of the linear fit
-output_str is a preformatted output string result.
'''
#1) Extracting V_floating
#selecting the data from the right zone (where V_floating is located)
right_range = bins_data['means'][1]
#we found the min I value, but we interested
#in the corresponding x value that is V_floating
i_floating = right_range['i'].abs().min()
#searching data row corresponding to i_floating and accessing the v index
v_floating = right_range[right_range['i'].abs()==i_floating]['v_probe'].values[0]
#2) Fitting the ionization current
# We start by defining our fit function
def ionic_saturation_function(p, v_probe):
return p[0] + p[1]*(v_probe -v_floating)
#we select the data from the first range (the left one)
x_left = bins_data['means'][0].v_probe.values
y_left = bins_data['means'][0].i.values
sx_left = bins_data['stds'][0].v_probe.values
sy_left = bins_data['stds'][0].i.values
#next, we define our type of model
linear_fit = Model(ionic_saturation_function)
odr_data = RealData(x_left, y_left, sx=sx_left, sy=sy_left)
#beta0 represents and array with the parameter guessing
odr = ODR(odr_data, linear_fit, beta0=[0.0002, 0.001])
#running fit and getting result
output = odr.run()
#saving the parameter obtained by the fit
q = output.beta[0]
sigma_q = output.sd_beta[0]
alpha = - output.beta[1] / q
#the ionic current is equal to q with this fitting function
i_ionic_sat = q
#creating the figure for linear fitting
y_fit = q - q*alpha*(x_left -v_floating)
fig1 ,ax = plt.subplots(1,1)
ax.plot(x_left, y_fit, 'r', label='Fit curve', linewidth=2.)
#fmt doesn't plot the line between data points
ax.errorbar(x_left,y_left, xerr=sx_left,
yerr=sy_left, fmt='o', label='Data')
ax.set_xlabel('$ V\;(V) $', fontsize=18)
ax.set_ylabel('$ I\;(mA) $', fontsize=18)
ax.legend(bbox_to_anchor=(0.15,1))
fig1.set_size_inches(18,12)
ax.grid()
output = []
output.append('Ionic saturation current = {0} A'.format(round(i_ionic_sat,5)))
output.append('Alpha = {0}'.format(round(alpha, 5)))
output.append('V floating = {0} V'.format(round(v_floating,5)))
output_str = '\n'.join(output)
return (v_floating, i_ionic_sat, alpha, fig1, output_str)
# Compute line of best fit
Y = future_life_expectancy
X = age
from scipy.odr import Model, Data, ODR
from scipy.stats import linregress
def f(p, x):
return (p[0] * x) + p[1]
linreg = linregress(X, Y)
mod = Model(f)
dat = Data(X, Y)
od = ODR(dat, mod, beta0=[1., 2.])
out = od.run()
TLSbeta = out.beta[0]
# In[15]:
# Plot chart
plt.plot(X, Y, '.')
plt.plot(X, out.beta[1] + np.multiply(X, out.beta[0]), '.')
plt.xlabel('Current Age')
plt.ylabel('Future Life')
plt.title('Future Life Expectancy')
noise_x = np.random.normal(loc = 0, scale=4, size = len(x_true))
x_error = np.random.normal(loc = 0, scale=4, size = len(x_true))
x_obs = x_true + noise_x
def quad_func(p, x):
m, c = p
return m*x + c
# Create a model for fitting.
quad_model = Model(quad_func)
# Create a RealData object using our initiated data from above.
data = RealData(x_obs, y_obs, sx=x_error, sy=y_error)
# Set up ODR with the model and data.
odr = ODR(data, quad_model, beta0=[0.15, 1.0])
# Run the regression.
out = odr.run()
# define the model/function to be fitted.
def model(x_obs, y_obs):
m = pm.Uniform('m', 0, 10, value= 0.15)
n = pm.Uniform('n', -10, 10, value= 1.0)
x_pred = pm.Normal('x_true', mu=x_obs, tau=(x_error)**-2) # this allows error in x_obs
#Theoretical values
@pm.deterministic(plot=False)
def linearD(x_true=x_pred, m=m, n=n):
return m * x_true + n
def test_total_least_squares():
dim = 10 + int(30 * np.random.rand())
x = np.arange(dim) + np.random.normal(0.0, 0.15, dim)
xerr = 0.1 + 0.1 * np.random.rand(dim)
y = 2 * np.exp(-0.06 * x) + np.random.normal(0.0, 0.15, dim)
yerr = 0.1 + 0.1 * np.random.rand(dim)
ox = []
for i, item in enumerate(x):
ox.append(pe.pseudo_Obs(x[i], xerr[i], str(i)))
oy = []
for i, item in enumerate(x):
oy.append(pe.pseudo_Obs(y[i], yerr[i], str(i)))
def f(x, a, b):
return a * np.exp(-b * x)
def func(a, x):
y = a[0] * np.exp(-a[1] * x)
return y
data = RealData([o.value for o in ox], [o.value for o in oy],
sx=[o.dvalue for o in ox],
sy=[o.dvalue for o in oy])
model = Model(func)
odr = ODR(data, model, [0, 0], partol=np.finfo(np.float64).eps)
odr.set_job(fit_type=0, deriv=1)
output = odr.run()
out = pe.total_least_squares(ox, oy, func, expected_chisquare=True)
beta = out.fit_parameters
str(out)
repr(out)
len(out)
for i in range(2):
beta[i].gamma_method(S=1.0)
assert math.isclose(beta[i].value, output.beta[i], rel_tol=1e-5)
assert math.isclose(
output.cov_beta[i, i], beta[i].dvalue**2, rel_tol=2.5e-1), str(
output.cov_beta[i, i]) + ' ' + str(beta[i].dvalue**2)
assert math.isclose(pe.covariance(beta[0], beta[1]),
output.cov_beta[0, 1],
rel_tol=2.5e-1)
out = pe.total_least_squares(ox, oy, func, const_par=[beta[1]])
diff = out.fit_parameters[0] - beta[0]
assert (diff / beta[0] < 1e-3 * beta[0].dvalue)
assert ((out.fit_parameters[1] - beta[1]).is_zero())
oxc = []
for i, item in enumerate(x):
oxc.append(pe.cov_Obs(x[i], xerr[i]**2, 'covx' + str(i)))
oyc = []
for i, item in enumerate(x):
oyc.append(pe.cov_Obs(y[i], yerr[i]**2, 'covy' + str(i)))
outc = pe.total_least_squares(oxc, oyc, func)
betac = outc.fit_parameters
for i in range(2):
betac[i].gamma_method(S=1.0)
assert math.isclose(betac[i].value, output.beta[i], rel_tol=1e-3)
assert math.isclose(
output.cov_beta[i, i], betac[i].dvalue**2, rel_tol=2.5e-1), str(
output.cov_beta[i, i]) + ' ' + str(betac[i].dvalue**2)
assert math.isclose(pe.covariance(betac[0], betac[1]),
output.cov_beta[0, 1],
rel_tol=2.5e-1)
outc = pe.total_least_squares(oxc, oyc, func, const_par=[betac[1]])
diffc = outc.fit_parameters[0] - betac[0]
assert (diffc / betac[0] < 1e-3 * betac[0].dvalue)
assert ((outc.fit_parameters[1] - betac[1]).is_zero())
outc = pe.total_least_squares(oxc, oy, func)
betac = outc.fit_parameters
for i in range(2):
betac[i].gamma_method(S=1.0)
assert math.isclose(betac[i].value, output.beta[i], rel_tol=1e-3)
assert math.isclose(
output.cov_beta[i, i], betac[i].dvalue**2, rel_tol=2.5e-1), str(
output.cov_beta[i, i]) + ' ' + str(betac[i].dvalue**2)
assert math.isclose(pe.covariance(betac[0], betac[1]),
output.cov_beta[0, 1],
rel_tol=2.5e-1)
outc = pe.total_least_squares(oxc, oy, func, const_par=[betac[1]])
diffc = outc.fit_parameters[0] - betac[0]
assert (diffc / betac[0] < 1e-3 * betac[0].dvalue)
assert ((outc.fit_parameters[1] - betac[1]).is_zero())
#reg_pars2 = linregress(data[:,2], data[:,3])
# now do ODR
# Define a function (quadratic in our case) to fit the data with.
def linear_func(p, x):
m, c = p
return m * x + c
# Create a model for fitting.
linear_model = Model(linear_func)
# Create a RealData object using our initiated data from above.
data = RealData(data[:, 2], data[:, 3])
# Set up ODR with the model and data.
odr = ODR(data, linear_model, beta0=[1., 0.])
odr.set_job(fit_type=0) #if set fit_type=2, returns the same as leastsq
# Run the regression.
out = odr.run()
# Use the in-built pprint method to give us results.
out.pprint()
modr = out.beta[0]
codr = out.beta[1]
yplt_odr = modr * xplt + codr
plt.plot(xplt, yplt_odr, 'b-', lw=2, label='ODR')
plt.xlim([1.8, 6.5])
plt.ylim([1.8, 6.5])
plt.legend(numpoints=1, loc=2)
plt.grid(which='both')
# Define a function (quadratic in our case) to fit the data with.
def linear_func(p, x):
m, c = p
return m * x + c
# Create a model for fitting.
linear_model = Model(linear_func)
# Create a RealData object using our initiated data from above.
data = RealData(x, y)
# Set up ODR with the model and data.
odr = ODR(data, linear_model, beta0=[0., 1.])
# Run the regression.
out = odr.run()
# Use the in-built pprint method to give us results.
out.pprint() # pretty print
slope = out.beta[0]
intercept = out.beta[1]
# xx holds the x limits of the line to draw. The graph is a straight line,
# so we only need the endpoints to draw it.
xx = np.array([0, 6])
yy = linear_func(out.beta, xx)
plt.plot(x, y, 'ro')
def match_copies(self,
matrix1, taxa1,
matrix2, taxa2,
force_single_copy=False):
"""Select best pairing copies between assessed gene families
:parameter matrix1: DataFrame with distances from gene1
:parameter matrix2: DataFrame with distances from gene2
:parameter taxa1: taxon table from gene1 (pd.DataFrame)
:parameter taxa2: taxon table from gene2 (pd.DataFrame)
Return paired copies of input DataFrames"""
#
# create a single DataFrame matching taxa from both gene families, and
# remove "|<num>" identification for added copies
all_taxon_pairs = pd.DataFrame()
all_taxon_pairs['gene1'] = taxa1.gene
all_taxon_pairs['gene2'] = taxa2.gene
all_taxon_pairs['genome'] = taxa1.genome.tolist()
all_taxon_pairs['pairs'] = all_taxon_pairs[['gene1', 'gene2']].apply(lambda x: frozenset(x), axis=1)
#
# summarize distances matrices by using only its upper triangle (triu)
triu_indices = np.triu_indices_from(matrix1, k=1)
condensed1 = matrix1.values[triu_indices]
condensed2 = matrix2.values[triu_indices]
#
# run ODR with no weights...
model = Model(self.line)
data = Data(condensed1,
condensed2)
odr = ODR(data,
model,
beta0=[np.std(condensed2) / # Geometric Mean slope estimate
np.std(condensed1)] #
)
regression = odr.run()
############################################### new code...
#
# create DataFrame with all residuals from the preliminary ODR with all
# possible combinations of gene within the same genome
residual_df = pd.DataFrame(columns=['matrix1_gene',
'matrix2_gene',
'genome',
'to_drop',
'combined_residual'],
data =zip(taxa1.iloc[triu_indices[0], 0].values,
taxa2.iloc[triu_indices[0], 0].values,
taxa1.iloc[triu_indices[0], 1].values,
taxa1.iloc[triu_indices[0], 1].values == taxa1.iloc[triu_indices[1], 1].values,
abs(regression.delta)+abs(regression.eps))
)
residual_df = residual_df.append(
pd.DataFrame(columns=['matrix1_gene',
'matrix2_gene',
'genome',
'to_drop',
'combined_residual'],
data =zip(taxa1.iloc[triu_indices[1], 0].values,
taxa2.iloc[triu_indices[1], 0].values,
taxa1.iloc[triu_indices[1], 1].values,
taxa1.iloc[triu_indices[0], 1].values == taxa1.iloc[triu_indices[1], 1].values,
abs(regression.delta)+abs(regression.eps))
),
sort =True,
ignore_index=True
)
residual_df.drop(index =residual_df.index[residual_df.to_drop],
inplace=True)
sum_paired_residuals = residual_df.groupby(
['matrix1_gene', 'matrix2_gene']
).agg(
residual_sum=pd.NamedAgg(column ="combined_residual",
aggfunc=sum),
genome =pd.NamedAgg(column='genome', aggfunc=lambda x: x.iloc[0])
).reset_index()
sum_paired_residuals.sort_values('residual_sum',
inplace=True)
sum_paired_residuals.reset_index(inplace=True,
drop =True)
best_pairs = pd.DataFrame(columns=['gene1', 'gene2', 'genome'])
for genome, indices in sum_paired_residuals.groupby('genome').groups.items():
pairing_possibilities = sum_paired_residuals.loc[indices].copy()
while pairing_possibilities.shape[0]:
first_row = pairing_possibilities.iloc[0]
best_pairs = best_pairs.append(
pd.Series(index=['gene1',
'gene2',
'genome'],
data =[first_row.matrix1_gene,
first_row.matrix2_gene,
genome]),
ignore_index=True
)
if force_single_copy:
break
pairing_possibilities.drop(
index=pairing_possibilities.query(
'(matrix1_gene == @first_row.matrix1_gene) | '
'(matrix2_gene == @first_row.matrix2_gene)'
).index,
inplace=True)
best_pairs['pairs'] = best_pairs[['gene1', 'gene2']].apply(lambda x: frozenset(x),
axis=1)
all_taxon_pairs = all_taxon_pairs.query('pairs.isin(@best_pairs.pairs)').copy()
taxa1 = taxa1.reindex(index=all_taxon_pairs.index)
taxa2 = taxa2.reindex(index=all_taxon_pairs.index)
taxa1.sort_values('genome', kind='mergesort', inplace=True)
taxa2.sort_values('genome', kind='mergesort', inplace=True)
taxa1.reset_index(drop=True, inplace=True)
taxa2.reset_index(drop=True, inplace=True)
############################################### ...up to here
if not all(taxa1.genome == taxa2.genome):
raise Exception('**Wow, taxa order is wrong! ABORT!!!')
matrix1 = matrix1.reindex(index =taxa1.taxon,
columns=taxa1.taxon,
copy =True)
matrix2 = matrix2.reindex(index =taxa2.taxon,
columns=taxa2.taxon,
copy =True)
return(matrix1, taxa1, matrix2, taxa2)
def fitDoubleGaussians(charges, charge_err, data, data_err):
# some simple peakfinding to find the first and second peaks
peak1_value = np.max(data)
peak1_bin = [x for x in range(len(data)) if data[x] == peak1_value][0]
peak1_charge = charges[peak1_bin]
# Preliminary fits to get better estimates of parameters...
first_peak_data = RealData(
charges[peak1_bin - 30:peak1_bin + 30],
data[peak1_bin - 30:peak1_bin + 30],
charge_err[peak1_bin - 30:peak1_bin + 30],
data_err[peak1_bin - 30:peak1_bin + 30],
)
first_peak_model = Model(gauss_fit)
first_peak_odr = ODR(first_peak_data, first_peak_model,
[peak1_value, peak1_charge, 0.1 * peak1_charge])
first_peak_odr.set_job(fit_type=2)
first_peak_output = first_peak_odr.run()
first_peak_params = first_peak_output.beta
#second_peak_params, covariance = curve_fit(gauss_fit2,
# data[peak1_bin-30:peak1_bin+30],
# data[peak1_bin-30:peak1_bin+30],
# p0 = [peak1_value, peak1_charge, 0.1*peak1_charge])
# subtract the largest peak so we can search for the other one
updated_data = data - gauss_fit(first_peak_params, charges)
#updated_data = data[:int(len(data)*2.0/3.0)]
peak2_value = np.max(updated_data)
peak2_bin = [
x for x in range(len(updated_data)) if updated_data[x] == peak2_value
][0]
peak2_charge = charges[peak2_bin]
#first_peak_params, covariance = curve_fit(gauss_fit2,
# updated_data[peak2_bin-30:peak2_bin+30],
# updated_data[peak2_bin-30:peak2_bin+30],
# p0 = [peak2_value, peak2_charge, 0.1*peak2_charge])
# and the second peak...
second_peak_data = RealData(
charges[peak2_bin - 30:peak2_bin + 30],
data[peak2_bin - 30:peak2_bin + 30],
charge_err[peak2_bin - 30:peak2_bin + 30],
data_err[peak2_bin - 30:peak2_bin + 30],
)
second_peak_model = Model(gauss_fit)
second_peak_odr = ODR(second_peak_data, second_peak_model,
[peak2_value, peak2_charge, first_peak_params[2]])
second_peak_odr.set_job(fit_type=2)
second_peak_output = second_peak_odr.run()
second_peak_params = second_peak_output.beta
# Now fit both gaussians simultaneously
double_peak_data = RealData(charges, data, charge_err, data_err)
double_peak_model = Model(double_gauss_fit)
double_peak_odr = ODR(double_peak_data, double_peak_model, [
first_peak_params[0], first_peak_params[1], first_peak_params[2],
second_peak_params[0], second_peak_params[1], second_peak_params[2]
])
double_peak_odr.set_job(fit_type=0)
double_peak_output = double_peak_odr.run()
double_peak_params = double_peak_output.beta
double_peak_sigmas = double_peak_output.sd_beta
#double_peak_params = [first_peak_params[0], first_peak_params[1], first_peak_params[2],
# second_peak_params[0], second_peak_params[1], second_peak_params[2]]
return double_peak_params, double_peak_sigmas
def source_colour(ifile, vfile, plotfile='source_colour.png', VIoffset=0.0):
# Define a function (quadratic in our case) to fit the data with.
def linear_func1(p, x):
m, c = p
return m * x + c
Idata = np.loadtxt(ifile)
Vdata = np.loadtxt(vfile)
qI = np.where(Idata[:, 5] < 1.0)
qV = np.where(Vdata[:, 5] < 1.0)
Idata = Idata[qI]
Vdata = Vdata[qV]
intervals = [0.025, 0.05, 0.1, 0.2]
colour = []
delta_colour = []
plt.figure(figsize=(12, 12))
for inter, interval in enumerate(intervals):
start = np.floor(np.min(Idata[:, 0]))
end = np.ceil(np.max(Idata[:, 0]))
time = np.arange(start, end, interval)
flux1 = np.zeros_like(time) + np.nan
flux2 = np.zeros_like(time) + np.nan
flux1_err = np.zeros_like(time) + np.nan
flux2_err = np.zeros_like(time) + np.nan
for i in range(len(time)):
q = np.where(np.abs(Idata[:, 0] - time[i]) < interval / 2.0)[0]
if q.any():
flux1[i] = np.sum(Idata[q, 1] / Idata[q, 2]**2) / np.sum(
1.0 / Idata[q, 2]**2)
flux1_err[i] = np.sqrt(1.0 / np.sum(1.0 / Idata[q, 2]**2))
p = np.where(np.abs(Vdata[:, 0] - time[i]) < interval / 2.0)[0]
if p.any():
flux2[i] = np.sum(Vdata[p, 1] / Vdata[p, 2]**2) / np.sum(
1.0 / Vdata[p, 2]**2)
flux2_err[i] = np.sqrt(1.0 / np.sum(1.0 / Vdata[p, 2]**2))
plt.subplot(2, 2, inter + 1)
plt.errorbar(flux1 / 1000.0,
flux2 / 1000.0,
xerr=flux1_err / 1000.0,
yerr=flux2_err / 1000.0,
fmt='.')
plt.xlabel(r'$\delta F_I (000)$')
plt.ylabel(r'$\delta F_V (000)$')
plt.grid()
# Create a model for fitting.
linear_model = Model(linear_func1)
good_data = np.where(
np.logical_and(np.isfinite(flux1), np.isfinite(flux2)))[0]
offset = np.mean(flux2[good_data] - flux1[good_data])
# Create a RealData object using our initiated data from above.
data = RealData(flux1[good_data],
flux2[good_data],
sx=flux1_err[good_data],
sy=flux2_err[good_data])
# Set up ODR with the model and data.
odr = ODR(data, linear_model, beta0=[1.0, offset])
# Run the regression.
out = odr.run()
# Use the in-built pprint method to give us results.
out.pprint()
x1, x2 = plt.gca().get_xlim()
x_fit = np.linspace(x1 * 1000, x2 * 1000, 1000)
y_fit = linear_func1(out.beta, x_fit)
plt.plot(x_fit / 1000.0,
y_fit / 1000.0,
'r-',
label=r"$\delta F_V = %5.3f \delta F_I + %5.3f$" %
(out.beta[0], out.beta[1]))
colour.append(VIoffset - 2.5 * np.log10(out.beta[0]))
delta_colour.append(VIoffset -
2.5 * np.log10(out.beta[0] - out.sd_beta[0]) -
colour[inter])
plt.title(r'$\Delta t = %5.3f \quad (V-I)_S = %8.3f \pm %8.3f$' %
(interval, colour[inter], delta_colour[inter]))
plt.legend()
plt.savefig(plotfile)
return colour, delta_colour
def zeroth_fit_2var(Nml,Eth,nu_des_ini,t_start,t_stop):
deca=0
sizeOfFont = 16
fontProperties = {'family':'sans-serif',
'weight' : 'normal', 'size' : sizeOfFont}
#rc('text', usetex=True)
rc('font',**fontProperties)
ticks_font = font_manager.FontProperties(family='cm', style='normal',
size=sizeOfFont, weight='normal', stretch='normal')
filenm='12CO_13CO_1_1_fit.txt'
#if molec =='n2':
# m_mol=30.0e-3
# xtex=0.042
#if molec =='co':
m_mol=28.0e-3
xtex=0.0375
#if molec =='co2':
# m_mol=44.0e-3
#if molec =='h2o':
# m_mol=18.0e-3
errort=0.1
T_tot, N_tot,N_13co = np.loadtxt(filenm,unpack=True,skiprows=1)
N_tot=N_13co
T_ini=T_tot
T_tot=T_tot
T_tot_err=T_tot*0.0+errort
N_tot=np.clip(N_tot,1e-12,1e30)
N_tot_err=1e12+N_tot*0
h_rate=1/60.0 #Kmin-1
T=T_tot[np.where((T_tot>t_start) & (T_tot<t_stop))]
N=N_tot[np.where((T_tot>t_start) & (T_tot<t_stop))]
N_err=1e10+N*0
T_err=T*0.0+errort
x=1.0/T
x_err=T_err/(T**2)
#print x_err
y=np.log(N/(1e15))
y_err=N_err/N
#y=np.log(N)
p=[0.,0.]
#nu_des_ini=7e11#np.sqrt(2.0*1e19*8.31*Eth/(np.pi*np.pi*m_mol))
#slope, intercept, r_value, p_value, std_err=scipy.stats.linregress(x, y)
def func(p, t):
return p[1] +t*p[0]
#def func (p,t):
# return np.log(1e12/h_rate)+t*p[0]
#def func(p, t):
# return np.log(np.sqrt(2.0*1e19*8.31*-1*p[0]/(np.pi*np.pi*m_mol))/h_rate) +t*p[0]
#p[1]=np.log(1e12/h_rate)
#p,pcov=curve_fit(func, x, y)
#err = np.sqrt(np.diag(pcov))
data = RealData(x, y, sx=x_err,sy=y_err)#, sy=np.log(0.001))
model = Model(func)
#odr = ODR(data, model, beta0=[-800,1e12])
odr = ODR(data, model, beta0=[-750,7e11])
odr.set_job(fit_type=2)
output = odr.run()
p=output.beta
err=output.sd_beta
E_des=-p[0]
nu_des=h_rate*np.exp(p[1])
nu_theo=h_rate*np.exp(np.log(np.sqrt(2.0*1e19*8.31*-1*p[0]/(np.pi*np.pi*m_mol))/h_rate))
#p[1]=np.log(np.sqrt(2.0*1e19*8.31*-1*p[0]/(np.pi*np.pi*m_mol))/h_rate)
#err[1]=0.0
#nu_theo=np.sqrt(2.0*1e19*8.31*Eth/(np.pi*np.pi*m_mol))
print 'E_des fit', p[0]*-1, '+-',err[0]
print 'E_des litt', Eth
print 'nu fit e12', 1e-12*h_rate*np.exp(p[1]), '+-',err[1]*1e-12*h_rate*np.exp(p[1])
print 'nu litt e12', 1e-12*nu_des_ini
print 'nu theo e12', 1e-12*nu_theo
#nu_des=2e11
#print "nu e12n", nu_des*1e-12
# def funct(x,t):
# if x[0]>0.0:
# f_ml = -(nu_des/h_rate)*np.exp(-E_des/t)
# f_g= f_ml-x[1]
# if x[0]<0.0:
# f_ml=0
# f_g= f_ml-x[1]
# return [f_ml,f_g]
# F_pure = odeint(funct,[Nml,0.00],T_tot)
# N_ml = np.clip(F_pure[:,0],0.0,10)
# N_g = -F_pure[:,1]
# k_d_ml = (nu_des/h_rate)*np.exp(-E_des/T_tot)
# k_d_mode=(nu_des_ini/h_rate)*np.exp(-Eth/T_tot)
#print "Ns",N_ml
#if order==0:
# k_d_ml[np.where(k_d_ml>np.max(N))]=0.0
#A[:,i] = k_d_ml
#A[:,i] = np.clip(k_d_ml,-1e15,1e15)
#x,resid = optimize.nnls(A,b)
#slope, intercept, r_value, p_value, std_err=scipy.stats.linregress(x, y)
plot_tpd='12CO_reglinfit_2var.pdf'
plt.errorbar(x,y,xerr=x_err,yerr=y_err,color='k')
#plt.errorbar(a,b,yerr=c, linestyle="None")
plt.plot(x,p[1]+p[0]*x,color='r',linestyle='--',linewidth=3.0)
#plt.plot(x,p[1]+p[0]*x,color='r',linestyle='--',linewidth=3.0)
plt.text(xtex,0.0,'-Slope: '+str(round(E_des))+' +- '+str(round(err[0])))
plt.text(xtex,-0.5,'exp(Intercept)*h_rate: '+ str(round(1e-12*h_rate*np.exp(p[1]),2))+ '+/-'+str(round(err[1]*1e-12*h_rate*np.exp(p[1]),2))+ 'e12')
#plt.plot(x,p[0]+(p[1]-err[1])*x,color='r',linestyle='--',linewidth=3.0)
#plt.plot(x,p[0]+(p[1]+err[1])*x,color='r',linestyle='--',linewidth=3.0)
#plt.plot(x,np.log(1e12/h_rate)-Eth*x,color='b',linestyle='--',linewidth=3.0)
#plt.xlim(1/(t_stop+5),1/(t_start-5))
plt.savefig(plot_tpd)
plt.close('all')
#ind=np.where(N_tot<np.max(N_tot))
#plot_tpd='12CO_tpdfit_2var.pdf'
#plt.errorbar(T_tot[ind],N_tot[ind]/(1e15),xerr=T_tot_err[ind],yerr=N_tot_err[ind]/1e15,color='k')
#plt.plot([t_start-deca,t_start-deca],[0,np.max(N_tot)/1e15],linestyle='--',color='g')
#plt.plot([t_stop-deca,t_stop-deca],[0,np.max(N_tot)/1e15],linestyle='--',color='g')
#plt.plot(T,-np.exp(intercept)*np.exp(-1*slope/T),color='r',linestyle='--',linewidth=3.0)
#plt.plot(T,(60*1.5e12/5.3e15)*np.exp(-950.0/T),color='b',linestyle='--',linewidth=3.0)
#plt.plot(T_tot,np.clip(k_d_ml,0,2*np.max(N_tot/1e15)),color='r')
#plt.plot(T_tot,np.clip(k_d_mode,0,2*np.max(N_tot/1e15)),color='r')
#plt.xlim(15,50)
#plt.ylim(-0.005,np.max(N_tot)/1e15+0.1)
#plt.savefig(plot_tpd)
plt.close('all')
class OrthogonalLeastSquares(CorrelBase):
"""
Accepts two series with timestamps as indexes and averaging period.
:param ref_spd: Series containing reference speed as a column, timestamp as the index
:param target_spd: Series containing target speed as a column, timestamp as the index
:param averaging_prd: Groups data by the period specified by period.
* 2T, 2 min for minutely average
* Set period to 1D for a daily average, 3D for three hourly average, similarly 5D, 7D, 15D etc.
* Set period to 1H for hourly average, 3H for three hourly average and so on for 5H, 6H etc.
* Set period to 1MS for monthly average
* Set period to 1AS fo annual average
:param coverage_threshold: Minimum coverage to include for correlation
:param preprocess: To average and check for coverage before correlating
:returns: Returns an object representing the model
"""
def linear_func(p, x):
return p[0] * x + p[1]
def __init__(self,
ref_spd,
target_spd,
averaging_prd,
coverage_threshold=0.9,
preprocess=True):
CorrelBase.__init__(self,
ref_spd,
target_spd,
averaging_prd,
coverage_threshold,
preprocess=preprocess)
self.params = 'not run yet'
def __repr__(self):
return 'Orthogonal Least Squares Model ' + str(self.params)
def run(self):
fit_data = RealData(self.data['ref_spd'].values.flatten(),
self.data['target_spd'].values.flatten())
p, res = lstsq(np.nan_to_num(fit_data.x[:, np.newaxis]**[1, 0]),
np.nan_to_num(np.asarray(fit_data.y)[:,
np.newaxis]))[0:2]
self._model = ODR(fit_data,
Model(OrthogonalLeastSquares.linear_func),
beta0=[p[0][0], p[1][0]])
self.out = self._model.run()
self.params = {'slope': self.out.beta[0], 'offset': self.out.beta[1]}
self.params['r2'] = self.get_r2()
self.params['Num data points'] = self.num_data_pts
# print("Model output:", self.out.pprint())
self.show_params()
def _predict(self, x):
def linear_func_inverted(x, p):
return OrthogonalLeastSquares.linear_func(p, x)
return x.transform(linear_func_inverted,
p=[self.params['slope'], self.params['offset']])
def fit_data(plt_data, initial_data, n_iterations, m_i):
t1 = time.time()
#unpacking variables
x, y, xerr, yerr = plt_data
v_floating, i_ionic_sat, alpha = initial_data
#splitting data for iterations
iterations_data = split_data(n_iterations, plt_data, v_floating)
# we need to save every fit result in order
# to use it with the next fitting.
# We define our first guessing parameters as fit_results.
#We define 5 the initial electronic Temperature
fit_results = [(i_ionic_sat, alpha, v_floating, 10.)]
#separate list for the errors results from fitting
error_results = [(0,0,0,0)]
#list for reduced chi2 results
chi2r_results = []
#iterating o
for i in range(n_iterations):
#getting data for i index
x_fit, y_fit, sx_fit, sy_fit = iterations_data[i]
#next, we define our type of model
langmuir_fit = Model(fpff)
odr_data = RealData(x_fit, y_fit, sx=sx_fit, sy=sy_fit)
#every cycle we use the previous fit
#result as beta 0 parameters
odr = ODR(odr_data, langmuir_fit,
beta0=fit_results[i], maxit=1000)
#running fit and getting result
output = odr.run()
#saving data
fit_results.append(tuple(output.beta))
error_results.append(tuple(output.sd_beta))
chi2r_results.append(output.res_var)
#now we have to search for the best fit
#using the principle of min temperature
#unpacking results for temperature
temp_fit = [b[3] for b in fit_results]
# discard the first temp value because it has been arbitrarly
# set
temp_fit = temp_fit[1:]
iterations_data = iterations_data[1:]
fit_results = fit_results[1:]
error_results = error_results[1:]
chi2r_results = chi2r_results[1:]
#find the minimum of the temperature to get all associated
#parameters and use them as the best parameters for fitting
min_temp = min(temp_fit)
min_index = temp_fit.index(min_temp)
last_v_probe = iterations_data[min_index][0][-1]
best_fit_results = fit_results[min_index]
best_fit_errors = error_results[min_index]
chi2r_best = chi2r_results[min_index]
#creating text output
output = []
output.append('Minimum temperature found: {0}'.format(min_temp))
output.append('Correspondent index of iteration: {0}'.format(min_index))
output.append('V_probe limit of fit: {} V'.format(last_v_probe))
output.append('\nParameters associated with the lowest temperature:')
output.append('Ionic saturation current = {0} ± {1} mA'.format(
best_fit_results[0], best_fit_errors[0]))
output.append('Alpha = {0} ± {1}'.format(
best_fit_results[1], best_fit_errors[1]))
output.append('V floating = {0} ± {1} V'.format(
best_fit_results[2], best_fit_errors[2]))
output.append('Temp = {0} ± {1}'.format(
best_fit_results[3], best_fit_errors[3]))
output.append('Reduced Chi2 = {0}'.format(
round(chi2r_best,3)))
output_str = '\n'.join(output)
#Creating plots
#1) Plot of temperatures
#plot of temperatures with respect to the number of iterations.
temp_fig, temp_plot = plt.subplots(1,1)
temp_plot.grid(True)
temp_plot.set_xlabel('Number of iterations', fontsize=18)
temp_plot.set_ylabel('Temperature (eV)', fontsize=18)
temp_plot.plot(temp_fit,'go', label='Temperature')
temp_plot.axhline(y= best_fit_results[3],linewidth=2, color = 'r')
temp_fig.set_size_inches(18,12)
#2) Final plot of the fit
plot_text = ('Parameters obtained with the \nminimum'+
' temperature fitting: \n\n' +
'last $V_{probe} = %.3f \, V $\n' +
'$ I_{i,sat} = %.5f \pm %.5f\, A $\n' +
'$\\alpha = %.5f \pm %.5f $\n' +
'$V_{floating} = %.3f \pm %.3f \,V $\n' +
'$T_e = %.3f \pm %.3f \,eV $\n' +
"$\\chi^2 reduced = %.3f $") % (last_v_probe,
best_fit_results[0], best_fit_errors[0],
best_fit_results[1], best_fit_errors[1],
best_fit_results[2], best_fit_errors[2],
best_fit_results[3], best_fit_errors[3] ,chi2r_best)
#calculating i from fitted data
#We use x because we want the i in all the x range
i_fit = fpff(best_fit_results, x)
fit_fig, fit_plot = plt.subplots(1,1)
fit_plot.grid(True)
fit_plot.plot(x, i_fit, 'r', label='Fit curve', linewidth=2.)
#fmt is the key to avoid line that connects markers
fit_plot.errorbar(x,y, xerr=xerr, yerr=yerr,
fmt='o', label='Data')
fit_plot.set_xlabel('$ V\;(V) $', fontsize=18)
fit_plot.set_ylabel('$ I\;(A) $', fontsize=18)
fit_plot.set_xlim([x[0]-5, x[-1]+5 ])
fit_plot.set_ylim([-0.003, 0.035])
fit_plot.legend(bbox_to_anchor=(0.11,1))
fit_plot.text(-50., 0.02, plot_text, fontsize=18,
bbox=dict(facecolor='white', edgecolor='black', ))
fit_plot.axvspan(last_v_probe, x[-1]+5 ,
facecolor='blue', alpha=0.2)
fit_fig.set_size_inches(18,12)
t2 = time.time()
#calculate addictional data
ne_data = calculate_ne(
best_fit_results[0],best_fit_errors[0],
best_fit_results[3], best_fit_errors[3], m_i)
vplasma_data = calculate_vplasma(
best_fit_results[2], best_fit_errors[2],
best_fit_results[3], best_fit_errors[3])
f_ep_data = calculate_fep(ne_data[0], ne_data[1])
#creating dictionary with data
data = {
"i_ionic_sat": best_fit_results[0],
"err_i_ionic_sat" : best_fit_errors[0],
"alpha" : best_fit_results[1],
"err_alpha" : best_fit_errors[1],
"v_floating" : best_fit_results[2],
"err_v_floating" : best_fit_errors[2],
"T_e" : best_fit_results[3],
"err_T_e": best_fit_errors[3],
"n_e": ne_data[0],
"err_n_e": ne_data[1],
"v_plasma": vplasma_data[0],
"err_v_plasma": vplasma_data[1],
"f_ep": f_ep_data[0],
"err_f_ep": f_ep_data[1]}
#returning a dictionary with all data
return {
'best_iteration': min_index,
"data": data,
'chi2r_results': chi2r_results,
'last_v_probe': last_v_probe,
'output_str': output_str,
'temp_fig': temp_fig,
'fit_fig': fit_fig,
'exec_time': t2-t1
}
def monte_carlo_odr(x_data, y_data, x_err, y_err, new_x_data, new_y_data, new_x_err, new_y_err):
"""
1) Randomises the data (i = 1000) based on values (x, y) and associated errors (x_err, y_err).
2) Constructs a standard logged OLS regression (used for ODR beta estimates).
3) Detects outliers using internally studentised residuals from the OLS. Those > 2 sigma (95%) are rejected.
4) Constructs an ODR and saves model coefficients (beta, covariance matrix, errors)
5) Takes the median coefficients for final ODR model construction
"""
# Generates results files
betas = []
covariances = []
eps = []
# make function into Model instance (logarithmic)
model = Model(log_func)
# Sets seed for reproducible results
np.random.seed(214)
# 1000 iterations
for i in range(1000):
# Randomises the data (mean, sd)
x = np.random.normal(x_data, x_err)
y = np.random.normal(y_data, y_err)
# Logs the data first
logX = log10(x)
# Adds constant for stats model intercept
X = sm.add_constant(logX)
# 10 iterations (should be much less, but "just in case")
for i in range(10):
# runs a simple OLS model (log)
linear_model = sm.OLS(y, X)
results = linear_model.fit()
# creates instance of influence
influence = results.get_influence()
# calculates internally standardized residuals
St_Res = influence.resid_studentized_internal
# Finds max residual
M = np.max(abs(St_Res))
# If any are larger than 2 standard deviations
if M > 2:
# Find their index
res = [idx for idx, val in enumerate(St_Res) if val > 2 or val < -2]
# Delete these data points
x = np.delete(x, res)
X = np.delete(X, res, axis = 0)
y = np.delete(y, res)
# If none are larger than 2 sd, continue using this dataset. Slope and intercept used for ODR fit.
else:
slope = results.params[1]
intercept = results.params[0]
continue
# New data and model
data = Data(x, y)
# Job = 0, explicit orthogonal
out = ODR(data, model, beta0=[slope, intercept], job=0).run()
# Appends model coefficients to results file (for EPS, only the maximum value is recorded)
betas.append(out.beta)
eps.append(max(out.eps))
covariances.append(out.cov_beta)
# Takes the median of the model estimates
Beta = np.median(betas, axis = 0)
Eps = np.median(eps, axis = 0)
Covariance = np.median(covariances, axis = 0)
# fit model using new beta, original x scale for consistency
xn = linspace(min(x_data), max(x_data), 1000)
yn = log_func(Beta, xn)
# 1 and 2 sigma prediction intervals
pl1 = prediction_interval([log10(xn), 1], 54, len(xn), Eps, 68., Beta, Covariance)
pl2 = prediction_interval([log10(xn), 1], 54, len(xn), Eps, 95., Beta, Covariance)
# create a figure to draw on and add a subplot
fig, ax = subplots(1)
# plot y calculated from px against de-logged x (and 1 and 2 sigma prediction intervals)
ax.plot(xn, yn, '#EC472F', label='Logarithmic ODR')
ax.plot(xn, yn + pl1, '#0076D4', dashes=[9, 4.5], label='1σ Prediction limit (~68%)', linewidth=0.8)
ax.plot(xn, yn - pl1, '#0076D4', dashes=[9, 4.5], linewidth=0.8)
ax.plot(xn, yn + pl2, '#BFBFBF', dashes=[9, 4.5], label='2σ Prediction limit (~95%)', linewidth=0.5)
ax.plot(xn, yn - pl2, '#BFBFBF', dashes=[9, 4.5], linewidth=0.5)
# plot points and error bars
ax.plot(x_data, y_data, 'k.', markerfacecolor= '#4495F3',
markeredgewidth=.5, markeredgecolor = 'k',
label='Calibration data (n = 54)', markersize=5)
ax.errorbar(x_data, y_data, ecolor='k', xerr=x_err, yerr=y_err, fmt=" ", linewidth=0.5, capsize=0)
# adds new data and errors bars
ax.plot(new_x_data, new_y_data,'k.', markerfacecolor= '#FF8130',
markeredgewidth=.5, markeredgecolor = 'k',
label = 'New data (n = 15)', markersize = 5)
ax.errorbar(new_x_data, new_y_data, ecolor='k', xerr=new_x_err, yerr=new_y_err, fmt=" ", linewidth=0.5, capsize=0)
# labels, extents etc.
ax.set_ylim(0, 60)
ax.set_xlabel('Mean R-value')
ax.set_ylabel('Age (ka)')
ax.tick_params(direction = 'in')
ax.tick_params(bottom=True, top=True, left=True, right=True)
ax.tick_params(labelbottom=True, labeltop=False, labelleft=True, labelright=False)
# configure legend
ax.legend(frameon=False,
fontsize=7)
# Sets axis ratio to 1
ratio = 1
ax.set_aspect(1.0/ax.get_data_ratio()*ratio)
# export the figure
fig.set_size_inches(3.2, 3.2)
savefig('Pyrenees_Monte_Carlo_ODR.png', dpi = 900, bbox_inches='tight')
#savefig('Pyrenees_Monte_Carlo_ODR.svg')
# Return final model coefficients
return Beta, Eps, Covariance
def fit(f,
xdata,
ydata,
xerr=None,
yerr=None,
beta0=None,
bf_res=500,
maxit=50):
"""
Fits a general function f(params, x) to data (xdata, ydata),
accounting for uncertainties in both the independent and dependent
variables.
Parameters
----------
f:
Function of the form f(param, x), where param is an array of size M
and x is an array containing values of the independent variable.
xdata:
Array of size N containing independent variable.
ydata:
Array of size N containing dependent variable.
xerr:
Uncertainty in the independent variable, must be a float or an array of
size N.
yerr:
Uncertainty in the dependent variable, must be a float or an array of
size N.
beta0:
Array of size M containing initial guesses for parameter values.
bf_res:
Int defining the number of points in the best fit array.
maxit:
Int specifying the maximum number of iterations to perform
Returns
-------
params:
Array of size M containing parameter estimates.
param_error:
Array of size M containing the standard errors of the estimated
parameters.
bestfit:
Array of shape (2, bf_res) containing x and y values for the best fit curve.
chi_vals:
List containing chi-squared and reduced chi-squared values respectively
[chi, red_chi]
"""
# create the model using the general function and fit the data using scipy.odr
mod = Model(f)
data = RealData(xdata, ydata, xerr, yerr)
result = ODR(data, mod, beta0, maxit=maxit).run()
# extract the estimated parameters and parameter errors
params = result.beta
param_error = result.sd_beta
# calculate chi-squared and reduced chi-squared values
chi_vals = _chi_values(f, xdata, ydata, yerr, params)
# create the best fit array by evaluating f using the estimated parameters
resx = np.linspace(np.min(xdata), np.max(xdata), bf_res)
bestfit = [resx, f(params, resx)]
return params, param_error, bestfit, chi_vals
def main():
try:
legend = ['curvature', 'neural-network', 'hough transform', 'SVM-Preliminary']
data_filename = ['number.txt', 'skflow.txt', 'curvature.txt', 'SVM.txt']
colors = ['blue', 'red', 'black', 'green']
annotation_text = "function: y = kx + b";
fig, ax = plt.subplots(ncols = 1)
for i in range(len(data_filename)):
temp = np.loadtxt(data_filename[i],skiprows=0)
data = np.reshape(temp,(41,len(temp)/41))
#print data
data[:,1:3] = np.true_divide(data[:,1:3], np.amax(data[:,1]))
data[:,1] = data[:,1]+i
#print data
#emperical error
xerr = np.sqrt(data[:,0])/3
yerr = data[:,2]
data_fit = Data(data[:,0].T, data[:,1].T, \
we = 1/(np.power(xerr.T,2)+np.spacing(1)),\
wd = 1/(np.power(yerr.T,2)+np.spacing(1)))
model = Model(ord_function)
odr = ODR(data_fit, model, beta0=[0, 0])
odr.set_job(fit_type=2)
output = odr.run()
popt = output.beta
perr= output.sd_beta
popt, pcov = curve_fit(linear_fit_function,
data[:,0], data[:,1], [1, 0], data[:, 2])
perr = np.sqrt(np.diag(pcov))
A = popt[0]/np.sqrt(popt[0]*popt[0]+1)
B = -1/np.sqrt(popt[0]*popt[0]+1)
C = popt[1]/np.sqrt(popt[0]*popt[0]+1)
fitting_error= np.mean(np.abs(A*data[:,0]+B*data[:,1]+C))
ax.errorbar(data[:,0], data[:,1], xerr = xerr,\
yerr = yerr, fmt='o',color=colors[i])
### not using error bar in fitting #########
popt[0],popt[1] = np.polyfit(data[:,0], data[:,1], 1)
###
ax.plot(data[:,0], popt[0]*data[:,0]+popt[1], colors[i], linewidth = 2)
annotation_text = annotation_text + "\n" +\
legend[i] + "(" + \
colors[i] + ") \n" + \
"k = %.2f b = %.2f Error = %.2f"%( \
popt[0], popt[1], fitting_error)
bbox_props = dict(boxstyle="square,pad=0.3", fc="white", ec="black", lw=2)
#ax.text(0, 4.15, annotation_text, ha="left", va="top", \
# rotation=0, size=14, bbox=bbox_props)
ax.set_title('Algorithom Performance')
ax.set_xlabel('Bubble Number Counted Manually')
ax.set_ylabel('Normalized Bubbble Number Counted by Algorithom')
plt.grid()
plt.xlim((np.amin(data[:,0])-5,np.amax(data[:,0])+5))
plt.ylim((0,np.amax(data[:,1])+0.2))
plt.show()
except KeyboardInterrupt:
print "Shutdown requested... exiting"
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def ks_iterative(x, y, x_err, y_err, Foward = True):
"""
Esta funcion agarra un eje X (k), un eje Y (k_nn) y busca los parametros para
ajustar la mejor recta, buscando el regimen lineal de la curva. Esto lo hace
sacando puntos de la curva, ajustando la curva resultante, y luego comparando
los parametros de los distintos ajustes con el metodo de Kolmogorov Smirnoff.
Si Foward=True entonces la funcion va a ir sacando puntos del final para
encontrar kmax. Si Foward=False, la funcion va a sacar puntos del principio para
calcular kmin. El punto va a estar dado por k[index].
Returns: m, b, ks_stat, index
m: pendiente de la recta resultante
b: ordenada de la recta resultante
ks_stat: estadistico de KS de la recta resultante
index: indice del elemento donde empieza/termina el regimen lineal.
.
.
"""
KS_list = []
pvalue_list = []
m_list = []
b_list = []
m_err_list = []
b_err_list = []
if Foward==True:
for j in range(0, len(x)-3):
y_temp = y[:len(y)-j]
x_temp = x[:len(x)-j]
x_err_temp = x_err[:len(x_err)-j]
y_err_temp = y_err[:len(y_err)-j]
linear_model = Model(Linear)
data = RealData(x_temp, y_temp, sx=x_err_temp, sy=y_err_temp)
odr = ODR(data, linear_model, beta0=[0., 1.])
out = odr.run()
modelo = [j*out.beta[0]+out.beta[1] for j in x_temp]
KS_list.append(stats.ks_2samp(y_temp, modelo)[0])
pvalue_list.append(stats.ks_2samp(y_temp, modelo)[1])
m_list.append(out.beta[0])
b_list.append(out.beta[1])
m_err_list.append(out.sd_beta[0])
b_err_list.append(out.sd_beta[1])
else:
for j in range(0, len(x)-3):
y_temp = y[:len(y)-j]
x_temp = x[:len(x)-j]
x_err_temp = x_err[:len(x_err)-j]
y_err_temp = y_err[:len(y_err)-j]
linear_model = Model(Linear)
data = RealData(x_temp, y_temp, sx=x_err_temp, sy=y_err_temp)
odr = ODR(data, linear_model, beta0=[0., 1.])
out = odr.run()
modelo = [j*out.beta[0]+out.beta[1] for j in x_temp]
KS_list.append(stats.ks_2samp(y_temp, modelo)[0])
pvalue_list.append(stats.ks_2samp(y_temp, modelo)[1])
m_list.append(out.beta[0])
b_list.append(out.beta[1])
m_err_list.append(out.sd_beta[0])
b_err_list.append(out.sd_beta[1])
index = KS_list.index(min(KS_list))
m = m_list[index]
b = b_list[index]
m_err = m_err_list[index]
b_err = b_err_list[index]
ks_stat = KS_list[index]
return m, b, m_err, b_err, ks_stat, index
def test_multi(self):
multi_mod = Model(
self.multi_fcn,
meta=dict(name='Sample Multi-Response Model',
ref='ODRPACK UG, pg. 56'),
)
multi_x = np.array([
30.0, 50.0, 70.0, 100.0, 150.0, 200.0, 300.0, 500.0, 700.0, 1000.0,
1500.0, 2000.0, 3000.0, 5000.0, 7000.0, 10000.0, 15000.0, 20000.0,
30000.0, 50000.0, 70000.0, 100000.0, 150000.0
])
multi_y = np.array([
[
4.22, 4.167, 4.132, 4.038, 4.019, 3.956, 3.884, 3.784, 3.713,
3.633, 3.54, 3.433, 3.358, 3.258, 3.193, 3.128, 3.059, 2.984,
2.934, 2.876, 2.838, 2.798, 2.759
],
[
0.136, 0.167, 0.188, 0.212, 0.236, 0.257, 0.276, 0.297, 0.309,
0.311, 0.314, 0.311, 0.305, 0.289, 0.277, 0.255, 0.24, 0.218,
0.202, 0.182, 0.168, 0.153, 0.139
],
])
n = len(multi_x)
multi_we = np.zeros((2, 2, n), dtype=float)
multi_ifixx = np.ones(n, dtype=int)
multi_delta = np.zeros(n, dtype=float)
multi_we[0, 0, :] = 559.6
multi_we[1, 0, :] = multi_we[0, 1, :] = -1634.0
multi_we[1, 1, :] = 8397.0
for i in range(n):
if multi_x[i] < 100.0:
multi_ifixx[i] = 0
elif multi_x[i] <= 150.0:
pass # defaults are fine
elif multi_x[i] <= 1000.0:
multi_delta[i] = 25.0
elif multi_x[i] <= 10000.0:
multi_delta[i] = 560.0
elif multi_x[i] <= 100000.0:
multi_delta[i] = 9500.0
else:
multi_delta[i] = 144000.0
if multi_x[i] == 100.0 or multi_x[i] == 150.0:
multi_we[:, :, i] = 0.0
multi_dat = Data(multi_x,
multi_y,
wd=1e-4 / np.power(multi_x, 2),
we=multi_we)
multi_odr = ODR(multi_dat,
multi_mod,
beta0=[4., 2., 7., .4, .5],
delta0=multi_delta,
ifixx=multi_ifixx)
multi_odr.set_job(deriv=1, del_init=1)
out = multi_odr.run()
assert_array_almost_equal(
out.beta,
np.array([
4.3799880305938963, 2.4333057577497703, 8.0028845899503978,
0.5101147161764654, 0.5173902330489161
]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([
0.0130625231081944, 0.0130499785273277, 0.1167085962217757,
0.0132642749596149, 0.0288529201353984
]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[
0.0064918418231375, 0.0036159705923791, 0.0438637051470406,
-0.0058700836512467, 0.011281212888768
],
[
0.0036159705923791, 0.0064793789429006,
0.0517610978353126, -0.0051181304940204,
0.0130726943624117
],
[
0.0438637051470406, 0.0517610978353126,
0.5182263323095322, -0.0563083340093696,
0.1269490939468611
],
[
-0.0058700836512467, -0.0051181304940204,
-0.0563083340093696, 0.0066939246261263,
-0.0140184391377962
],
[
0.011281212888768, 0.0130726943624117,
0.1269490939468611, -0.0140184391377962,
0.0316733013820852
]]),
)
selection = new_selection_dictionary['Mon R2'] + new_selection_dictionary[
'Orion A'] + new_selection_dictionary['Orion B']
selection_ids = [struct.idx for struct in selection]
s_size = catalog_orion[selection_ids]['size']
s_linewidth = catalog_orion[selection_ids]['v_rms']
mydata = RealData(s_size, s_linewidth)
def powerlaw(B, x):
return B[0] * (x**B[1])
powerlaw_model = Model(powerlaw)
myodr = ODR(mydata, powerlaw_model, beta0=[1., 1.])
myoutput = myodr.run()
orion_fit_constant = myoutput.beta[0]
orion_fit_power = myoutput.beta[1]
# the plot
xs = np.logspace(-1, 4, 20)
ys = orion_fit_constant * xs**orion_fit_power
dv_orion = d_orion.viewer()
dv_orion.hub.select(1, selection)
dsl_orion = Scatter(d_orion, dv_orion.hub, catalog_orion, 'size', 'v_rms')
I_err = error_I(I, '2612')
chi_2 = []
beta = []
sd_beta = []
t = (np.mean(R) - R0)/m
t_err = np.sqrt((np.std(R)/m)**2 * 1/len(R) + 1/m**2)
for h in range(len(V) - 2):
V_fit = V[h:]
I_fit = I[h:]
I_err_fit = I_err[h:]
V_err_fit = V_err[h:]
model = Model(Linear)
data = RealData(V_fit, I_fit, sx=V_err_fit, sy=I_err_fit)
odr = ODR(data, model, beta0=[0., 0.], maxit=1000000)
out = odr.run()
beta.append(out.beta[0])
sd_beta.append(out.sd_beta[0])
chi_2.append(out.res_var)
index = ClosestToOne(chi_2)
Rq_temp.append(pixels/beta[index])
Rq_err_temp.append(pixels/(beta[index]**2)*sd_beta[index])
T_temp.append(t)
T_err_temp.append(t_err)
print("%s/%s" % (j, end))
a0 = 2
b0 = 1
c0 = 1
x = numpy.linspace(-3, 7.0, N)
y = model((a0, b0, c0), x) + normal(0.0, 0.3, N)
errx = normal(0.1, 0.2, N)
erry = normal(0.1, 0.3, N)
# It is important to start with realistic initial estimates
beta0 = [1.8, 0.9, 0.9]
print("\nODR:")
print("==========")
from scipy.odr import Data, Model, ODR, RealData, odr_stop
linear = Model(model)
mydata = RealData(x, y, sx=errx, sy=erry)
myodr = ODR(mydata, linear, beta0=beta0, maxit=5000)
myoutput = myodr.run()
print("Fitted parameters: ", myoutput.beta)
print("Covariance errors: ", numpy.sqrt(myoutput.cov_beta.diagonal()))
print("Standard errors: ", myoutput.sd_beta)
print("Minimum chi^2: ", myoutput.sum_square)
print("Minimum (reduced)chi^2: ", myoutput.res_var)
beta = myoutput.beta
# Prepare fit routine
fitobj = kmpfit.Fitter(residuals=residuals, data=(x, y, errx, erry))
fitobj.fit(params0=beta0)
print("\n\n======== Results kmpfit with effective variance =========")
print("Fitted parameters: ", fitobj.params)
print("Covariance errors: ", fitobj.xerror)
print("Standard errors: ", fitobj.stderr)
def main():
try:
if (len(sys.argv) > 1 and sys.argv[1] == '1'):
data_filename = 'number.txt'
start_time = time.time()
number = np.zeros((41,3))
index = -1
for det in range(1,8):
if (det!=6):
num_n = 6
else:
num_n = 5
for n in range(0,num_n):
index = index + 1
temp = np.zeros(3)
for angle in range(1,4):
filename = 'detector_' + str(det) + '_no_' + str(n) \
+ '_angle_' + str(angle) + '.jpg'
refname = 'detector_' + str(det) + '_no_' + str(n) \
+ '_background.jpg'
elapse_time = (time.time()-start_time)
if(elapse_time >= 1):
remain_time = elapse_time/(index*3+angle-1)*41*3-elapse_time
print 'Processing .. ' + filename \
+ time.strftime(" %H:%M:%S", time.gmtime(elapse_time)) \
+ ' has past. ' + 'Remaining time: ' \
+ time.strftime(" %H:%M:%S", time.gmtime(remain_time))
else:
print 'Processing .. ' + filename \
+ time.strftime(" %H:%M:%S", time.gmtime(elapse_time)) \
+ ' has past'
image = rgb2gray(io.imread(filename))
ref = rgb2gray(io.imread(refname))
temp[angle-1] = ellipse.count_bubble(image,ref)
#temp[angle-1] = ellipse.count_bubble(image)
number[index,1] = np.mean(temp)
number[index,2] = np.std(temp)
manual_count = np.array([1,27,40,79,122,160,1,18,28,42,121,223,0,11,24,46,\
142,173,3,19,23,76,191,197,0,15,24,45,91,152,0,\
16,27,34,88,0,9,12,69,104,123])
number[:,0] = manual_count.T
number.tofile(data_filename,sep=" ")
elif(len(sys.argv) > 1):
data_filename = sys.argv[1]
data = np.loadtxt(data_filename,skiprows=0)
number = np.reshape(data,(41,3))
else:
data_filename = 'number.txt'
data = np.loadtxt(data_filename,skiprows=0)
number = np.reshape(data,(41,3))
#emperical error
xerr = np.sqrt(number[:,0])/3
yerr = number[:,2]
data = Data(number[:,0].T, number[:,1].T, we = 1/(np.power(xerr.T,2)+np.spacing(1)), wd = 1/(np.power(yerr.T,2)+np.spacing(1)))
model = Model(ord_function)
odr = ODR(data, model, beta0=[0, 0])
odr.set_job(fit_type=2)
output = odr.run()
popt = output.beta
perr = output.sd_beta
output.pprint()
# popt, pcov = curve_fit(linear_fit_function, number[:,0], number[:,1], [0, 0], number[:, 2])
# perr = np.sqrt(np.diag(pcov))
fitting_error = np.mean(np.sqrt(np.power(popt[0]*number[:,0]+popt[1] - number[:,1],2)))
labels = np.array([[1,1,1,1,1,1,\
2,2,2,2,2,2,\
3,3,3,3,3,3,\
4,4,4,4,4,4,\
5,5,5,5,5,5,\
6,6,6,6,6,\
7,7,7,7,7,7],
[0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,
0,1,2,3,4,5]])
fig, ax = plt.subplots(ncols = 1)
ax.errorbar(number[:,0], number[:,1], xerr = xerr, yerr = yerr, fmt='o')
ax.plot(number[:,0], popt[0]*number[:,0]+popt[1], '-r')
bbox_props = dict(boxstyle="square,pad=0.3", fc="white", ec="black", lw=2)
annotation_text = "function: y = kx + b \n" \
"k = %.2f" % popt[0] + " +/- %.2f" % perr[0] + '\n' \
"b = %.2f" % popt[1] + " +/- %.2f" % perr[1] + '\n' \
"Error: %.2f" % fitting_error
ax.text(10, np.amax(number[:,1])+10, annotation_text, ha="left", va="top", rotation=0,
size=15, bbox=bbox_props)
for i in range(0, len(number[:,0])): # <--
ax.annotate('(%s, %s)' % (labels[0,i], labels[1,i]),\
(number[i,0]+1, number[i,1]-1)) #
ax.set_title('Algorithom Performance')
ax.set_xlabel('Bubble Number Counted Manually')
ax.set_ylabel('Bubbble Number Counted by Algorithom')
plt.grid()
plt.xlim((np.amin(number[:,0])-5,np.amax(number[:,0])+5))
plt.ylim((0,np.amax(number[:,1])+20))
plt.show()
except KeyboardInterrupt:
print "Shutdown requested... exiting"
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def total_least_squares(data1, data2, data1err=None, data2err=None,
print_results=False, ignore_nans=True, intercept=True,
return_error=False, inf=1e10):
"""
Use Singular Value Decomposition to determine the Total Least Squares linear fit to the data.
(e.g. http://en.wikipedia.org/wiki/Total_least_squares)
data1 - x array
data2 - y array
if intercept:
returns m,b in the equation y = m x + b
else:
returns m
print tells you some information about what fraction of the variance is accounted for
ignore_nans will remove NAN values from BOTH arrays before computing
Parameters
----------
data1,data2 : np.ndarray
Vectors of the same length indicating the 'x' and 'y' vectors to fit
data1err,data2err : np.ndarray or None
Vectors of the same length as data1,data2 holding the 1-sigma error values
"""
if ignore_nans:
badvals = numpy.isnan(data1) + numpy.isnan(data2)
if data1err is not None:
badvals += numpy.isnan(data1err)
if data2err is not None:
badvals += numpy.isnan(data2err)
goodvals = True-badvals
if goodvals.sum() < 2:
if intercept:
return 0,0
else:
return 0
if badvals.sum():
data1 = data1[goodvals]
data2 = data2[goodvals]
if intercept:
dm1 = data1.mean()
dm2 = data2.mean()
else:
dm1,dm2 = 0,0
arr = numpy.array([data1-dm1,data2-dm2]).T
U,S,V = numpy.linalg.svd(arr, full_matrices=False)
# v should be sorted.
# this solution should be equivalent to v[1,0] / -v[1,1]
# but I'm using this: http://stackoverflow.com/questions/5879986/pseudo-inverse-of-sparse-matrix-in-python
M = V[-1,0]/-V[-1,-1]
varfrac = S[0]/S.sum()*100
if varfrac < 50:
raise ValueError("ERROR: SVD/TLS Linear Fit accounts for less than half the variance; this is impossible by definition.")
# this is performed after so that TLS gives a "guess"
if data1err is not None or data2err is not None:
try:
from scipy.odr import RealData,Model,ODR
except ImportError:
raise ImportError("Could not import scipy; cannot run Total Least Squares")
def linmodel(B,x):
if intercept:
return B[0]*x + B[1]
else:
return B[0]*x
if data1err is not None:
data1err = data1err[goodvals]
data1err[data1err<=0] = inf
if data2err is not None:
data2err = data2err[goodvals]
data2err[data2err<=0] = inf
if any([data1.shape != other.shape for other in (data2,data1err,data2err)]):
raise ValueError("Data shapes do not match")
linear = Model(linmodel)
data = RealData(data1,data2,sx=data1err,sy=data2err)
B = data2.mean() - M*data1.mean()
beta0 = [M,B] if intercept else [M]
myodr = ODR(data,linear,beta0=beta0)
output = myodr.run()
if print_results:
output.pprint()
if return_error:
return numpy.concatenate([output.beta,output.sd_beta])
else:
return output.beta
if intercept:
B = data2.mean() - M*data1.mean()
if print_results:
print "TLS Best fit y = %g x + %g" % (M,B)
print "The fit accounts for %0.3g%% of the variance." % (varfrac)
print "Chi^2 = %g, N = %i" % (((data2-(data1*M+B))**2).sum(),data1.shape[0]-2)
return M,B
else:
if print_results:
print "TLS Best fit y = %g x" % (M)
print "The fit accounts for %0.3g%% of the variance." % (varfrac)
print "Chi^2 = %g, N = %i" % (((data2-(data1*M))**2).sum(),data1.shape[0]-1)
return M
orion_output = load_orion_catalog_from_template()
d_orion, catalog_orion, header, metadata, template_cube, template_header, new_selection_dictionary, selection_ID_dictionary = orion_output
# the fit
selection = new_selection_dictionary['Mon R2'] + new_selection_dictionary['Orion A'] + new_selection_dictionary['Orion B']
selection_ids = [struct.idx for struct in selection]
s_size = catalog_orion[selection_ids]['size']
s_linewidth = catalog_orion[selection_ids]['v_rms']
mydata = RealData(s_size, s_linewidth)
def powerlaw(B, x):
return B[0] * (x**B[1])
powerlaw_model = Model(powerlaw)
myodr = ODR(mydata, powerlaw_model, beta0=[1.,1.])
myoutput = myodr.run()
orion_fit_constant = myoutput.beta[0]
orion_fit_power = myoutput.beta[1]
# the plot
xs = np.logspace(-1,4,20)
ys = orion_fit_constant * xs**orion_fit_power
dv_orion = d_orion.viewer()
dv_orion.hub.select(1, selection)
dsl_orion = Scatter(d_orion, dv_orion.hub, catalog_orion, 'size', 'v_rms')
def test_implicit(self):
implicit_mod = Model(
self.implicit_fcn,
implicit=1,
meta=dict(name='Sample Implicit Model', ref='ODRPACK UG, pg. 49'),
)
implicit_dat = Data(
[[
0.5, 1.2, 1.6, 1.86, 2.12, 2.36, 2.44, 2.36, 2.06, 1.74, 1.34,
0.9, -0.28, -0.78, -1.36, -1.9, -2.5, -2.88, -3.18, -3.44
],
[
-0.12, -0.6, -1., -1.4, -2.54, -3.36, -4., -4.75, -5.25,
-5.64, -5.97, -6.32, -6.44, -6.44, -6.41, -6.25, -5.88, -5.5,
-5.24, -4.86
]],
1,
)
implicit_odr = ODR(implicit_dat,
implicit_mod,
beta0=[-1.0, -3.0, 0.09, 0.02, 0.08])
out = implicit_odr.run()
assert_array_almost_equal(
out.beta,
np.array([
-0.9993809167281279, -2.9310484652026476, 0.0875730502693354,
0.0162299708984738, 0.0797537982976416
]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([
0.1113840353364371, 0.1097673310686467, 0.0041060738314314,
0.0027500347539902, 0.0034962501532468
]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[
2.1089274602333052e+00, -1.9437686411979040e+00,
7.0263550868344446e-02, -4.7175267373474862e-02,
5.2515575927380355e-02
],
[
-1.9437686411979040e+00, 2.0481509222414456e+00,
-6.1600515853057307e-02, 4.6268827806232933e-02,
-5.8822307501391467e-02
],
[
7.0263550868344446e-02, -6.1600515853057307e-02,
2.8659542561579308e-03, -1.4628662260014491e-03,
1.4528860663055824e-03
],
[
-4.7175267373474862e-02, 4.6268827806232933e-02,
-1.4628662260014491e-03, 1.2855592885514335e-03,
-1.2692942951415293e-03
],
[
5.2515575927380355e-02, -5.8822307501391467e-02,
1.4528860663055824e-03, -1.2692942951415293e-03,
2.0778813389755596e-03
]]),
)
def test_ticket_11800(self):
# parameters
beta_true = np.array([1.0, 2.3, 1.1, -1.0, 1.3, 0.5])
nr_measurements = 10
std_dev_x = 0.01
x_error = np.array([[
0.00063445, 0.00515731, 0.00162719, 0.01022866, -0.01624845,
0.00482652, 0.00275988, -0.00714734, -0.00929201, -0.00687301
],
[
-0.00831623, -0.00821211, -0.00203459,
0.00938266, -0.00701829, 0.0032169, 0.00259194,
-0.00581017, -0.0030283, 0.01014164
]])
std_dev_y = 0.05
y_error = np.array([[
0.05275304, 0.04519563, -0.07524086, 0.03575642, 0.04745194,
0.03806645, 0.07061601, -0.00753604, -0.02592543, -0.02394929
],
[
0.03632366, 0.06642266, 0.08373122, 0.03988822,
-0.0092536, -0.03750469, -0.03198903,
0.01642066, 0.01293648, -0.05627085
]])
beta_solution = np.array([
2.62920235756665876536e+00, -1.26608484996299608838e+02,
1.29703572775403074502e+02, -1.88560985401185465804e+00,
7.83834160771274923718e+01, -7.64124076838087091801e+01
])
# model's function and Jacobians
def func(beta, x):
y0 = beta[0] + beta[1] * x[0, :] + beta[2] * x[1, :]
y1 = beta[3] + beta[4] * x[0, :] + beta[5] * x[1, :]
return np.vstack((y0, y1))
def df_dbeta_odr(beta, x):
nr_meas = np.shape(x)[1]
zeros = np.zeros(nr_meas)
ones = np.ones(nr_meas)
dy0 = np.array([ones, x[0, :], x[1, :], zeros, zeros, zeros])
dy1 = np.array([zeros, zeros, zeros, ones, x[0, :], x[1, :]])
return np.stack((dy0, dy1))
def df_dx_odr(beta, x):
nr_meas = np.shape(x)[1]
ones = np.ones(nr_meas)
dy0 = np.array([beta[1] * ones, beta[2] * ones])
dy1 = np.array([beta[4] * ones, beta[5] * ones])
return np.stack((dy0, dy1))
# do measurements with errors in independent and dependent variables
x0_true = np.linspace(1, 10, nr_measurements)
x1_true = np.linspace(1, 10, nr_measurements)
x_true = np.array([x0_true, x1_true])
y_true = func(beta_true, x_true)
x_meas = x_true + x_error
y_meas = y_true + y_error
# estimate model's parameters
model_f = Model(func, fjacb=df_dbeta_odr, fjacd=df_dx_odr)
data = RealData(x_meas, y_meas, sx=std_dev_x, sy=std_dev_y)
odr_obj = ODR(data, model_f, beta0=0.9 * beta_true, maxit=100)
#odr_obj.set_iprint(init=2, iter=0, iter_step=1, final=1)
odr_obj.set_job(deriv=3)
odr_out = odr_obj.run()
# check results
assert_equal(odr_out.info, 1)
assert_array_almost_equal(odr_out.beta, beta_solution)
def total_least_squares(data1, data2, data1err=None, data2err=None,
print_results=False, ignore_nans=True, intercept=True,
return_error=False, inf=1e10):
"""
Use Singular Value Decomposition to determine the Total Least Squares linear fit to the data.
(e.g. http://en.wikipedia.org/wiki/Total_least_squares)
data1 - x array
data2 - y array
if intercept:
returns m,b in the equation y = m x + b
else:
returns m
print tells you some information about what fraction of the variance is accounted for
ignore_nans will remove NAN values from BOTH arrays before computing
Parameters
----------
data1,data2 : np.ndarray
Vectors of the same length indicating the 'x' and 'y' vectors to fit
data1err,data2err : np.ndarray or None
Vectors of the same length as data1,data2 holding the 1-sigma error values
"""
if ignore_nans:
badvals = numpy.isnan(data1) + numpy.isnan(data2)
if data1err is not None:
badvals += numpy.isnan(data1err)
if data2err is not None:
badvals += numpy.isnan(data2err)
goodvals = True-badvals
if goodvals.sum() < 2:
if intercept:
return 0,0
else:
return 0
if badvals.sum():
data1 = data1[goodvals]
data2 = data2[goodvals]
if intercept:
dm1 = data1.mean()
dm2 = data2.mean()
else:
dm1,dm2 = 0,0
arr = numpy.array([data1-dm1,data2-dm2]).T
U,S,V = numpy.linalg.svd(arr, full_matrices=False)
# v should be sorted.
# this solution should be equivalent to v[1,0] / -v[1,1]
# but I'm using this: http://stackoverflow.com/questions/5879986/pseudo-inverse-of-sparse-matrix-in-python
M = V[-1,0]/-V[-1,-1]
varfrac = S[0]/S.sum()*100
if varfrac < 50:
raise ValueError("ERROR: SVD/TLS Linear Fit accounts for less than half the variance; this is impossible by definition.")
# this is performed after so that TLS gives a "guess"
if data1err is not None or data2err is not None:
try:
from scipy.odr import RealData,Model,ODR
except ImportError:
raise ImportError("Could not import scipy; cannot run Total Least Squares")
def linmodel(B,x):
if intercept:
return B[0]*x + B[1]
else:
return B[0]*x
if data1err is not None:
data1err = data1err[goodvals]
data1err[data1err<=0] = inf
if data2err is not None:
data2err = data2err[goodvals]
data2err[data2err<=0] = inf
if any([data1.shape != other.shape for other in (data2,data1err,data2err)]):
raise ValueError("Data shapes do not match")
linear = Model(linmodel)
data = RealData(data1,data2,sx=data1err,sy=data2err)
B = data2.mean() - M*data1.mean()
beta0 = [M,B] if intercept else [M]
myodr = ODR(data,linear,beta0=beta0)
output = myodr.run()
if print_results:
output.pprint()
if return_error:
return numpy.concatenate([output.beta,output.sd_beta])
else:
return output.beta
if intercept:
B = data2.mean() - M*data1.mean()
if print_results:
print "TLS Best fit y = %g x + %g" % (M,B)
print "The fit accounts for %0.3g%% of the variance." % (varfrac)
print "Chi^2 = %g, N = %i" % (((data2-(data1*M+B))**2).sum(),data1.shape[0]-2)
return M,B
else:
if print_results:
print "TLS Best fit y = %g x" % (M)
print "The fit accounts for %0.3g%% of the variance." % (varfrac)
print "Chi^2 = %g, N = %i" % (((data2-(data1*M))**2).sum(),data1.shape[0]-1)
return M
def show():
data_filename = 'number.txt'
data = np.loadtxt(data_filename,skiprows=0)
number = np.reshape(data,(41,3))
#emperical error
xerr = np.sqrt(number[:,0])/3
yerr = number[:,2]
data = Data(number[:,0].T, number[:,1].T, we = 1/(np.power(xerr.T,2)+np.spacing(1)), wd = 1/(np.power(yerr.T,2)+np.spacing(1)))
model = Model(ord_function)
odr = ODR(data, model, beta0=[0, 0])
odr.set_job(fit_type=2)
output = odr.run()
popt = output.beta
perr = output.sd_beta
output.pprint()
fitting_error = np.mean(np.sqrt(np.power(popt[0]*number[:,0]+popt[1] - number[:,1],2)))
labels = np.array([[1,1,1,1,1,1,\
2,2,2,2,2,2,\
3,3,3,3,3,3,\
4,4,4,4,4,4,\
5,5,5,5,5,5,\
6,6,6,6,6,\
7,7,7,7,7,7],
[0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,5,\
0,1,2,3,4,
0,1,2,3,4,5]])
fig, ax = plt.subplots(ncols = 1)
ax.errorbar(number[:,0], number[:,1], xerr = xerr, yerr = yerr, fmt='o')
ax.plot(number[:,0], popt[0]*number[:,0]+popt[1], '-r')
bbox_props = dict(boxstyle="square,pad=0.3", fc="white", ec="black", lw=2)
annotation_text = "function: y = kx + b \n" \
"k = %.2f" % popt[0] + " +/- %.2f" % perr[0] + '\n' \
"b = %.2f" % popt[1] + " +/- %.2f" % perr[1] + '\n' \
"Error: %.2f" % fitting_error
ax.text(10, np.amax(number[:,1])+10, annotation_text, ha="left", va="top", rotation=0,
size=15, bbox=bbox_props)
for i in range(0, len(number[:,0])): # <--
ax.annotate('(%s, %s)' % (labels[0,i], labels[1,i]),\
(number[i,0]+1, number[i,1]-1)) #
ax.set_title('Algorithom Performance')
ax.set_xlabel('Bubble Number Counted Manually')
ax.set_ylabel('Bubbble Number Counted by Algorithom')
plt.grid()
plt.xlim((np.amin(number[:,0])-5,np.amax(number[:,0])+5))
plt.ylim((0,np.amax(number[:,1])+20))
plt.show()
def total_least_squares(x, y, func, silent=False, **kwargs):
r'''Performs a non-linear fit to y = func(x) and returns a list of Obs corresponding to the fit parameters.
Parameters
----------
x : list
list of Obs, or a tuple of lists of Obs
y : list
list of Obs. The dvalues of the Obs are used as x- and yerror for the fit.
func : object
func has to be of the form
```python
import autograd.numpy as anp
def func(a, x):
return a[0] + a[1] * x + a[2] * anp.sinh(x)
```
For multiple x values func can be of the form
```python
def func(a, x):
(x1, x2) = x
return a[0] * x1 ** 2 + a[1] * x2
```
It is important that all numpy functions refer to autograd.numpy, otherwise the differentiation
will not work.
silent : bool, optional
If true all output to the console is omitted (default False).
initial_guess : list
can provide an initial guess for the input parameters. Relevant for non-linear
fits with many parameters.
expected_chisquare : bool
If true prints the expected chisquare which is
corrected by effects caused by correlated input data.
This can take a while as the full correlation matrix
has to be calculated (default False).
Based on the orthogonal distance regression module of scipy
'''
output = Fit_result()
output.fit_function = func
x = np.array(x)
x_shape = x.shape
if not callable(func):
raise TypeError('func has to be a function.')
for i in range(25):
try:
func(np.arange(i), x.T[0])
except Exception:
pass
else:
break
n_parms = i
if not silent:
print('Fit with', n_parms, 'parameter' + 's' * (n_parms > 1))
x_f = np.vectorize(lambda o: o.value)(x)
dx_f = np.vectorize(lambda o: o.dvalue)(x)
y_f = np.array([o.value for o in y])
dy_f = np.array([o.dvalue for o in y])
if np.any(np.asarray(dx_f) <= 0.0):
raise Exception('No x errors available, run the gamma method first.')
if np.any(np.asarray(dy_f) <= 0.0):
raise Exception('No y errors available, run the gamma method first.')
if 'initial_guess' in kwargs:
x0 = kwargs.get('initial_guess')
if len(x0) != n_parms:
raise Exception(
'Initial guess does not have the correct length: %d vs. %d' %
(len(x0), n_parms))
else:
x0 = [1] * n_parms
data = RealData(x_f, y_f, sx=dx_f, sy=dy_f)
model = Model(func)
odr = ODR(data, model, x0, partol=np.finfo(np.float64).eps)
odr.set_job(fit_type=0, deriv=1)
out = odr.run()
output.residual_variance = out.res_var
output.method = 'ODR'
output.message = out.stopreason
output.xplus = out.xplus
if not silent:
print('Method: ODR')
print(*out.stopreason)
print('Residual variance:', output.residual_variance)
if out.info > 3:
raise Exception('The minimization procedure did not converge.')
m = x_f.size
def odr_chisquare(p):
model = func(p[:n_parms], p[n_parms:].reshape(x_shape))
chisq = anp.sum(((y_f - model) / dy_f)**2) + anp.sum(
((x_f - p[n_parms:].reshape(x_shape)) / dx_f)**2)
return chisq
if kwargs.get('expected_chisquare') is True:
W = np.diag(1 / np.asarray(np.concatenate(
(dy_f.ravel(), dx_f.ravel()))))
if kwargs.get('covariance') is not None:
cov = kwargs.get('covariance')
else:
cov = covariance_matrix(np.concatenate((y, x.ravel())))
number_of_x_parameters = int(m / x_f.shape[-1])
old_jac = jacobian(func)(out.beta, out.xplus)
fused_row1 = np.concatenate(
(old_jac,
np.concatenate(
(number_of_x_parameters * [np.zeros(old_jac.shape)]),
axis=0)))
fused_row2 = np.concatenate(
(jacobian(lambda x, y: func(y, x))(out.xplus, out.beta).reshape(
x_f.shape[-1], x_f.shape[-1] * number_of_x_parameters),
np.identity(number_of_x_parameters * old_jac.shape[0])))
new_jac = np.concatenate((fused_row1, fused_row2), axis=1)
A = W @ new_jac
P_phi = A @ np.linalg.inv(A.T @ A) @ A.T
expected_chisquare = np.trace(
(np.identity(P_phi.shape[0]) - P_phi) @ W @ cov @ W)
if expected_chisquare <= 0.0:
warnings.warn("Negative expected_chisquare.", RuntimeWarning)
expected_chisquare = np.abs(expected_chisquare)
output.chisquare_by_expected_chisquare = odr_chisquare(
np.concatenate((out.beta, out.xplus.ravel()))) / expected_chisquare
if not silent:
print('chisquare/expected_chisquare:',
output.chisquare_by_expected_chisquare)
fitp = out.beta
hess_inv = np.linalg.pinv(
jacobian(jacobian(odr_chisquare))(np.concatenate(
(fitp, out.xplus.ravel()))))
def odr_chisquare_compact_x(d):
model = func(d[:n_parms], d[n_parms:n_parms + m].reshape(x_shape))
chisq = anp.sum(((y_f - model) / dy_f)**2) + anp.sum(
((d[n_parms + m:].reshape(x_shape) -
d[n_parms:n_parms + m].reshape(x_shape)) / dx_f)**2)
return chisq
jac_jac_x = jacobian(jacobian(odr_chisquare_compact_x))(np.concatenate(
(fitp, out.xplus.ravel(), x_f.ravel())))
deriv_x = -hess_inv @ jac_jac_x[:n_parms + m, n_parms + m:]
def odr_chisquare_compact_y(d):
model = func(d[:n_parms], d[n_parms:n_parms + m].reshape(x_shape))
chisq = anp.sum(((d[n_parms + m:] - model) / dy_f)**2) + anp.sum(
((x_f - d[n_parms:n_parms + m].reshape(x_shape)) / dx_f)**2)
return chisq
jac_jac_y = jacobian(jacobian(odr_chisquare_compact_y))(np.concatenate(
(fitp, out.xplus.ravel(), y_f)))
deriv_y = -hess_inv @ jac_jac_y[:n_parms + m, n_parms + m:]
result = []
for i in range(n_parms):
result.append(
derived_observable(
lambda my_var, **kwargs:
(my_var[0] + np.finfo(np.float64).eps) /
(x.ravel()[0].value + np.finfo(np.float64).eps) * out.beta[i],
list(x.ravel()) + list(y),
man_grad=list(deriv_x[i]) + list(deriv_y[i])))
output.fit_parameters = result
output.odr_chisquare = odr_chisquare(
np.concatenate((out.beta, out.xplus.ravel())))
output.dof = x.shape[-1] - n_parms
output.p_value = 1 - chi2.cdf(output.odr_chisquare, output.dof)
return output
return A[0] * numpy.exp(-numpy.power(x, 2) / numpy.power(0.017453, 2)) xdata = numpy.array([.174, .2618, .3491, .5236, .6981, .8727, 1.047]) xdata = xdata - 0.0314 ydata = numpy.array([9.64, 0.66, 0.12, 0.02, 0.005, 0.002156, 0.0009]) x_err = numpy.array([0.0087, 0.0087, 0.0087, 0.0087, 0.0087, 0.0087, 0.0087]) y_err = numpy.array([1.79, 0.03, 0.005, 0.00049, 0.000253, 0.00009, 0.00004]) """xdata = numpy.array([.3491, .5236, .6981, .8727, 1.047]) ydata = numpy.array([ 0.12, 0.02, 0.005, 0.002156, 0.0009]) x_err = numpy.array([ 0.0087, 0.0087, 0.0087, 0.0087, 0.0087])""" #x_err = x_err*2 #y_err = numpy.array([0.005, 0.00049, 0.000253, 0.00009, 0.00004]) higherconv_model = Model(higherfitFunc) higherdata = RealData(xdata, ydata, sx=x_err, sy=y_err) higherodr = ODR(higherdata, higherconv_model, beta0=[0.005]) higherout = higherodr.run() higherout.pprint() lowerconv_model = Model(lowerfitFunc) lowerdata = RealData(xdata, ydata, sx=x_err, sy=y_err) lowerodr = ODR(lowerdata, lowerconv_model, beta0=[0.005]) lowerout = lowerodr.run() lowerout.pprint() lowerplum_conv_model = Model(lowerplum_fitFunc) lowerplum_data = RealData(xdata, ydata, sx=x_err, sy=y_err) lowerplum_odr = ODR(lowerplum_data, lowerplum_conv_model, beta0=[0.05, 10])
def test_multi(self):
multi_mod = Model(
self.multi_fcn,
meta=dict(name='Sample Multi-Response Model',
ref='ODRPACK UG, pg. 56'),
)
multi_x = np.array([30.0, 50.0, 70.0, 100.0, 150.0, 200.0, 300.0, 500.0,
700.0, 1000.0, 1500.0, 2000.0, 3000.0, 5000.0, 7000.0, 10000.0,
15000.0, 20000.0, 30000.0, 50000.0, 70000.0, 100000.0, 150000.0])
multi_y = np.array([
[4.22, 4.167, 4.132, 4.038, 4.019, 3.956, 3.884, 3.784, 3.713,
3.633, 3.54, 3.433, 3.358, 3.258, 3.193, 3.128, 3.059, 2.984,
2.934, 2.876, 2.838, 2.798, 2.759],
[0.136, 0.167, 0.188, 0.212, 0.236, 0.257, 0.276, 0.297, 0.309,
0.311, 0.314, 0.311, 0.305, 0.289, 0.277, 0.255, 0.24, 0.218,
0.202, 0.182, 0.168, 0.153, 0.139],
])
n = len(multi_x)
multi_we = np.zeros((2, 2, n), dtype=float)
multi_ifixx = np.ones(n, dtype=int)
multi_delta = np.zeros(n, dtype=float)
multi_we[0,0,:] = 559.6
multi_we[1,0,:] = multi_we[0,1,:] = -1634.0
multi_we[1,1,:] = 8397.0
for i in range(n):
if multi_x[i] < 100.0:
multi_ifixx[i] = 0
elif multi_x[i] <= 150.0:
pass # defaults are fine
elif multi_x[i] <= 1000.0:
multi_delta[i] = 25.0
elif multi_x[i] <= 10000.0:
multi_delta[i] = 560.0
elif multi_x[i] <= 100000.0:
multi_delta[i] = 9500.0
else:
multi_delta[i] = 144000.0
if multi_x[i] == 100.0 or multi_x[i] == 150.0:
multi_we[:,:,i] = 0.0
multi_dat = Data(multi_x, multi_y, wd=1e-4/np.power(multi_x, 2),
we=multi_we)
multi_odr = ODR(multi_dat, multi_mod, beta0=[4.,2.,7.,.4,.5],
delta0=multi_delta, ifixx=multi_ifixx)
multi_odr.set_job(deriv=1, del_init=1)
out = multi_odr.run()
assert_array_almost_equal(
out.beta,
np.array([4.3799880305938963, 2.4333057577497703, 8.0028845899503978,
0.5101147161764654, 0.5173902330489161]),
)
assert_array_almost_equal(
out.sd_beta,
np.array([0.0130625231081944, 0.0130499785273277, 0.1167085962217757,
0.0132642749596149, 0.0288529201353984]),
)
assert_array_almost_equal(
out.cov_beta,
np.array([[0.0064918418231375, 0.0036159705923791, 0.0438637051470406,
-0.0058700836512467, 0.011281212888768],
[0.0036159705923791, 0.0064793789429006, 0.0517610978353126,
-0.0051181304940204, 0.0130726943624117],
[0.0438637051470406, 0.0517610978353126, 0.5182263323095322,
-0.0563083340093696, 0.1269490939468611],
[-0.0058700836512467, -0.0051181304940204, -0.0563083340093696,
0.0066939246261263, -0.0140184391377962],
[0.011281212888768, 0.0130726943624117, 0.1269490939468611,
-0.0140184391377962, 0.0316733013820852]]),
)
return a/np.linalg.norm(a) #x=(x-np.min(x))/np.ptp(x) #y=(x-np.min(y))/np.ptp(y) #x = normalize(x) #y = normalize(y) linear = Model(f) mydata = RealData(x, y) myodr = ODR(mydata, linear, beta0=[1., 0.]) myoutput = myodr.run() myoutput.pprint() #print myoutput.delta #print myoutput.eps print myoutput.beta print myoutput.sum_square #print myoutput.sum_square_delta #print myoutput.sum_square_eps x_d = myoutput.xplus y_d = myoutput.y print myoutput.res_var/len(x)
def vertprofile(datafnme=(u'', ),
work_dir=(u'', ),
header=(1, ),
struct=([1, 2, 3, 4], ),
labelx=u'to be completed',
labely=u'to be completed',
labeldata=(u'Data', ),
rangex=None,
rangey=None,
statstypes=[0, 1, 2, 3],
confprob=95.0,
fontsz=10,
fontleg=9,
output='graph'):
"""
This code is to compute regressions for points with their error
using different regression methods
USAGE
>>>from vertprof import vertprofile
>>>vertprofile(datafnme = u'', work_dir = u'', header = 1, struct = [1,2,3,4],
labelx = 'to be completed', labely = 'to be completed', rangex = None, rangey = None,
statstypes = [0,1,2,3], confprob = 95.0,
fontsz = 10, fontleg = 9,
output = 'graph')
INPUTS
To use options or inputs, you need to set them as
vertprof(option_name = option_value, [...])
1. work_dir: tuple (i.e. (u'data1 workdir',u'data2 workdir',...) of the Working directory(-ies) (local path)
for each dataset
ex: work_dir = (u'Purgatorio3/',) (note the (,) structure if there is only one dataset to plot
Default None
2. datafnme: tuple (i.e. (u'data1 name ',u'data2 name',...) of the name(s) of text data file
should be in the form : Alt - Age - Age_Error
ex: datafnme = 'Purgatorio3.txt'
3. header: tuple of the number of lines of the header in the data for each dataset
ex: header = (1,) (Default)
4. struct: tuple of the structure of the data for each dataset
Struct = ([Xdata, Xerr, Ydata, Yerr],)
where Xdata, Xerr, Ydata, Yerr give their respective columns in the data file
If one of the data type do not exist, set the corresponding column to NONE
ex : struct = ([1,2,3,4],) (Default)
5. output: name of output
ex: output = 'graph' (Default)
6. labelx/labely: label x-axis/y-axis
ex: labelx = 'Exposure age (ka)'
labely ='Distance to the top of the scarp (m)'
7. labeldata: tuple (i.e. (u'data1 label',u'data2 label',...) with the name of the type of data
for each dataset plotted
default = (u'Data',)
8. rangex/rangey: Set the x- and y-range
Depends on type of data
ex: rangex = [0,8]
rangey = [10,4] (in that case, the axis is inverted)
9. statstypes: Type(s) of the stats to plot
0 = kmpfit effective variance
1 = kmpfit unweighted
2 = Williamson
3 = Cl relative weighting in X &/or Y
ex: statstype = [0,1,2,3] (Default)
statstype = [1,3]
10. fontsz: labels fontsize
ex: fontsz = 10 (Default)
11. fontleg: legend fontsize
ex: fontleg = 9 (Default)
12. confprob: the confidence interval probabilty (in %)
ex: confprob = 95.0 (Default)
OUTPUTS
For each dataset, the module outputs a pdf graph, and a text file with the stats outputs for each method asked
CONTACT
If needed, do not hesitate to contact the author.
Please, use https://isterre.fr/spip.php?page=contact&id_auteur=303
LICENCE
This package is licenced with `CCby-nc-sa` (https://creativecommons.org/licenses/by-nc-sa/2.0/)
"""
# set colors
# For the moment, only for 6 datasets, it should be enough
colors = ['b', 'darkorange', 'peru', 'black', 'darkviolet', 'green']
# initiate variables
erry = errx = a_will = b_will = verts = fitobj = fitobj2 = fitobj3 = None
# Tuples to lists
datafnme = list(datafnme)
work_dir = list(work_dir)
header = list(header)
struct = list(struct)
labeldata = list(labeldata)
# Initiate the plotting
plt.rc(u'font', size=fontsz)
#plt.rc(u'title', size = fontsz)
plt.rc(u'legend', fontsize=fontleg)
# Open figure
plt.figure(1).clear
#plt.add_subplot(1,1,1, aspect=1, adjustable=u'datalim')
# If the minima input data are not given print the help file
if not datafnme:
help(vertprofile)
raise TypeError(color.RED + u'ERROR : ' + color.END +
u'Name of Input file not given...')
# check if the length of work_dir, struct and labeldata are equal to the length of datafnme
# If not, copy the first record to the others, with a warning
for item in [work_dir, header, struct, labeldata]:
if len(datafnme) != len(item):
warnings.warn(
color.YELLOW + u"\nWARNING : " + color.END +
u"work_dir, header, struct and/or labeldata may be the same for all dataset"
)
#item = (item[0])
for i_data in range(1, len(datafnme)):
item = item.append(item[0])
#item[i_data] = item[0]
# do a loop on the number of dataset to plot
for i_data in range(0, len(datafnme)):
# check if the working directory exists, if not, create it
if work_dir[i_data].strip():
if work_dir[i_data][-1:] != u'/':
work_dir[i_data] = work_dir[i_data] + u'/'
if path.exists(work_dir[i_data]) == False:
os.mkdir(work_dir[i_data])
else:
warnings.warn(
color.YELLOW + u"\nWARNING : " + color.END +
u"Working directory already exists and will be erased\n")
# Check if the file exists in the main folder
# if not, ckeck in the working folder
# if only in the working folder, copy it to the main folder
if not path.isfile(datafnme[i_data]):
if not path.isfile(work_dir[i_data] + datafnme[i_data]):
#sys.exit(color.RED + u"ERROR : " + color.END + u"Input file %s does not exist..." % datafnme)
raise NameError(color.RED + u"ERROR : " + color.END +
u"Input file %s does not exist..." %
datafnme[i_data])
else:
copyfile(work_dir[i_data] + datafnme[i_data],
datafnme[i_data])
if not path.isfile(work_dir[i_data] + datafnme[i_data]):
copyfile(datafnme[i_data], work_dir[i_data] + datafnme[i_data])
datapath = work_dir[i_data] + datafnme[i_data]
else:
if not path.isfile(work_dir[i_data] + datafnme[i_data]):
#sys.exit(color.RED + u"ERROR : " + color.END + u"Input file %s does not exist..." % datafnme)
raise NameError(color.RED + u"ERROR : " + color.END +
u"Input file %s does not exist..." %
datafnme[i_data])
else:
datapath = datafnme[i_data]
work_dir[i_data] = u''
# read data
data = np.loadtxt(datapath, skiprows=header[i_data])
y = data[:, struct[i_data][0] - 1]
if struct[i_data][1] != 0 and struct[i_data][1] != None:
erry = data[:, struct[i_data][1] - 1]
else:
erry = 0
x = data[:, struct[i_data][2] - 1]
if struct[i_data][3] != 0 and struct[i_data][3] != None:
errx = data[:, struct[i_data][3] - 1]
else:
errx = 0
N = len(x)
# Open new file to print the results
f1w = open(work_dir[i_data] + u'results_' + datafnme[i_data], 'w')
string = ' '
print(string)
f1w.write(string + u"\n")
string = ' '
print(string)
f1w.write(string + u"\n")
string = ' '
print(string)
f1w.write(string + u"\n")
string = u' Dataset ' + labeldata[i_data]
print(color.RED + color.BOLD + string + color.END)
f1w.write(string + u"\n")
string = u"Results of weigthed least square from file " + datafnme[
i_data] + u"\n"
print(color.RED + color.BOLD + string + color.END)
f1w.write(string + u"\n")
string = u"xmax/min = " + str(x.max()) + u", " + str(x.min())
print(string)
f1w.write(string + u"\n")
string = u"ymax/min = " + str(y.max()) + u", " + str(y.min())
print(string)
f1w.write(string + u"\n")
string = u"x/y mean = " + str(x.mean()) + u", " + str(y.mean())
print(string)
f1w.write(string + u"\n")
string = (
u"\nMethods used are: \n" +
u" - kmpfit: kapteyn method, from " +
u"https://www.astro.rug.nl/software/kapteyn/kmpfittutorial.html \n"
+ u" with error on X and Y or Y only or none \n" +
u" - ODR: Orthogonal Distance Regression \n" +
u" - Williamson: least square fitting with errors in X and Y " +
u"according to \n" +
u" Williamson (Canadian Journal of Physics, 46, 1845-1847, 1968) \n"
)
print(color.CYAN + string + color.END)
f1w.write(string)
beta0 = [5.0, 1.0] # Initial estimates
if struct[i_data][3] != 0 and struct[i_data][3] != None:
if struct[i_data][1] != 0 or struct[i_data][1] != None:
# Prepare fit routine
fitobj = kmpfit.Fitter(residuals=residuals,
data=(x, y, errx, erry))
fitobj.fit(params0=beta0)
string = (
u"\n\n======== Results kmpfit: weights for both coordinates ========="
)
print(color.BLUE + color.BOLD + string + color.END)
f1w.write(string + "\n")
string = (
u"Fitted parameters: " + str(fitobj.params) + u"\n"
u"Covariance errors: " + str(fitobj.xerror) + u"\n"
u"Standard errors " + str(fitobj.stderr) + u"\n"
u"Chi^2 min: " + str(fitobj.chi2_min) + u"\n"
u"Reduced Chi^2: " + str(fitobj.rchi2_min) + u"\n"
u"Iterations: " + str(fitobj.niter))
print(string)
f1w.write(string + u"\n")
else:
# Prepare fit routine
fitobj2 = kmpfit.Fitter(residuals=residuals2,
data=(x, y, erry))
fitobj2.fit(params0=beta0)
string = (
u"\n\n======== Results kmpfit errors in Y only =========")
print(color.BLUE + color.BOLD + string + color.END)
f1w.write(string + u"\n")
string = (
u"Fitted parameters: " + str(fitobj2.params) + u"\n"
u"Covariance errors: " + str(fitobj2.xerror) + u"\n"
u"Standard errors " + str(fitobj2.stderr) + u"\n"
u"Chi^2 min: " + str(fitobj2.chi2_min) + u"\n"
u"Reduced Chi^2: " + str(fitobj2.rchi2_min))
print(string)
f1w.write(string)
# Unweighted (unit weighting)
fitobj3 = kmpfit.Fitter(residuals=residuals2, data=(x, y, np.ones(N)))
fitobj3.fit(params0=beta0)
string = (u"\n\n======== Results kmpfit unit weighting =========")
print(color.BLUE + color.BOLD + string + color.END)
f1w.write(string + "\n")
string = (u"Fitted parameters: " + str(fitobj3.params) + u"\n"
u"Covariance errors: " + str(fitobj3.xerror) + u"\n"
u"Standard errors " + str(fitobj3.stderr) + u"\n"
u"Chi^2 min: " + str(fitobj3.chi2_min) + u"\n"
u"Reduced Chi^2: " + str(fitobj3.rchi2_min))
print(string)
f1w.write(string)
# Compare result with ODR
# Create the linear model from statsfuncs
linear = Model(model)
if struct[i_data][3] != 0 and struct[i_data][3] != None:
if struct[i_data][1] != 0 and struct[i_data][1] != None:
mydata = RealData(x, y, sx=errx, sy=erry)
else:
mydata = RealData(x, y, sy=erry)
else:
mydata = RealData(x, y)
myodr = ODR(mydata, linear, beta0=beta0, maxit=5000, sstol=1e-14)
myoutput = myodr.run()
string = (u"\n\n======== Results ODR =========")
print(color.BLUE + color.BOLD + string + color.END)
f1w.write(string + u"\n")
string = (u"Fitted parameters: " + str(myoutput.beta) + u"\n"
u"Covariance errors: " +
str(np.sqrt(myoutput.cov_beta.diagonal())) + u"\n"
u"Standard errors: " + str(myoutput.sd_beta) + u"\n"
u"Minimum chi^2: " + str(myoutput.sum_square) +
u"\n"
u"Minimum (reduced)chi^2: " + str(myoutput.res_var))
print(string)
f1w.write(string)
# Compute Williamson regression
a_will, b_will, siga, sigb, beta, x_av, x__mean = williamson(
x, y, errx, erry, struct[i_data], myoutput)
if struct[i_data][3] != 0 and struct[i_data][3] != None:
if struct[i_data][1] != 0 and struct[i_data][1] != None:
chi2 = (residuals(fitobj.params, (x, y, errx, erry))**2).sum()
else:
chi2 = (residuals2(fitobj2.params, (x, y, erry))**2).sum()
else:
chi2 = (residuals2(fitobj3.params, (x, y, np.ones(N)))**2).sum()
string = (u"\n\n======== Results Williamson =========")
print(color.BLUE + color.BOLD + string + color.END)
f1w.write(string + u"\n")
string = (u"Average x weighted: " + str(x_av) + u"\n"
u"Average x unweighted: " + str(x__mean) + u"\n"
u"Fitted parameters: " + u"[" + str(a_will) + u"," +
str(b_will) + u"]\n"
u"Covariance errors: " + u"[" + str(siga) + u"," +
str(sigb) + u"]\n"
u"Minimum chi^2: " + str(chi2))
print(string)
f1w.write(string)
string = (
u"\n====================================="
u"\nPractical results: a + b * x"
)
print(color.BLUE + color.BOLD + string + color.END)
f1w.write(u"\n" + string + u"\n")
string = (u"kmpfit unweighted: %13.4f %13.4f" %
(fitobj3.params[0], fitobj3.params[1]))
print(string)
f1w.write(string + u"\n")
string = (u" Exhumation Rate: %13.4f" % (1 / fitobj3.params[1]))
print(string)
f1w.write(string + u"\n")
string = (u" Min. Exh. Rate: %13.4f" %
(1 / (fitobj3.params[1] - np.sqrt(fitobj3.stderr[1]))))
print(string)
f1w.write(string + u"\n")
string = (u" Max. Exh. Rate: %13.4f" %
(1 / (fitobj3.params[1] + np.sqrt(fitobj3.stderr[1]))))
print(string)
f1w.write(string + "\n")
if struct[i_data][3] != 0 and struct[i_data][3] != None:
if struct[i_data][1] != 0 and struct[i_data][1] != None:
string = u"kmpfit effective variance: %13.4f %13.4f" % (
fitobj.params[0], fitobj.params[1])
print(string)
f1w.write(string + u"\n")
string = (u" Exhumation Rate: %13.4f" %
(1 / fitobj.params[1]))
print(string)
f1w.write(string + u"\n")
string = (u" Min. Exh. Rate: %13.4f" %
(1 / (fitobj.params[1] - np.sqrt(fitobj.stderr[1]))))
print(string)
f1w.write(string + u"\n")
string = (u" Max. Exh. Rate: %13.4f" %
(1 / (fitobj.params[1] + np.sqrt(fitobj.stderr[1]))))
print(string)
f1w.write(string + u"\n")
else:
string = u"kmpfit weights in Y only: %13.4f %13.4f" % (
fitobj2.params[0], fitobj2.params[1])
print(string)
f1w.write(string + u"\n")
string = (u" Exhumation Rate: %13.4f" %
(1 / fitobj2.params[1]))
print(string)
f1w.write(string + u"\n")
string = (u" Min. Exh. Rate: %13.4f" %
(1 /
(fitobj2.params[1] - np.sqrt(fitobj2.stderr[1]))))
print(string)
f1w.write(string + u"\n")
string = (u" Max. Exh. Rate: %13.4f" %
(1 /
(fitobj2.params[1] + np.sqrt(fitobj2.stderr[1]))))
print(string)
f1w.write(string + u"\n")
string = u"ODR: %13.4f %13.4f" % (beta[0],
beta[1])
print(string)
f1w.write(string + u"\n")
string = (u" Exhumation Rate: %13.4f" % (1 / beta[1]))
print(string)
f1w.write(string + u"\n")
string = (u" Min. Exh. Rate: %13.4f" %
(1 / (beta[1] - np.sqrt(myoutput.sd_beta[1]))))
print(string)
f1w.write(string + u"\n")
string = (u" Max. Exh. Rate: %13.4f" %
(1 / (beta[1] + np.sqrt(myoutput.sd_beta[1]))))
print(string)
f1w.write(string + u"\n")
string = u"Williamson: %13.4f %13.4f" % (a_will,
b_will)
print(string)
f1w.write(string + u"\n")
string = (u" Exhumation Rate: %13.4f" % (1 / b_will))
print(string)
f1w.write(string + u"\n")
# For the next lines, not sure that we have to use the chi2 as error...
string = (u" Min. Exh. Rate: %13.4f" % (1 / (b_will - chi2)))
print(string)
f1w.write(string + u"\n")
string = (u" Max. Exh. Rate: %13.4f" % (1 / (b_will + chi2)))
print(string)
f1w.write(string + u"\n")
print(u"\n")
# calcul of confidence intervals
dfdp = [1, x, x**2]
if struct[i_data][3] != 0 and struct[i_data][3] != None:
if struct[i_data][1] != 0 and struct[i_data][1] != None:
ydummy, upperband, lowerband = confidence_band(
x, dfdp, confprob, fitobj, model)
labelconf = u"CI (" + str(
confprob) + u"%) relative weighting in X & Y"
if i_data == 0:
#string = (u"y = a + b * x with a = %6.4f +/- %6.4f and b = %6.4f +/- %6.4f "
#stringl = (u"y_" + labeldata[i_data] + u" = a + b * x with a = %.2f +/- %.2f and b = %.2f +/- %.2f "
#stringl = (u" y_" + labeldata[i_data] + u" = a + b * x with a = %.2f +/- %.2f and b = %.2f +/- %.2f "
# %(fitobj.params[0], np.sqrt(fitobj.stderr[0]),
# fitobj.params[1], np.sqrt(fitobj.stderr[1])))
stringl = (
r" $\mathbf{Y_{%s}} = \mathbf{a} + \mathbf{b} \times \mathbf{X}$ with $\mathbf{a} = %.2f \pm %.2f$ and $\mathbf{b} = %.2f \pm %.2f$ "
% (labeldata[i_data], fitobj.params[0],
np.sqrt(fitobj.stderr[0]), fitobj.params[1],
np.sqrt(fitobj.stderr[1])))
else:
#string = (u"y = a + b * x with a = %6.4f +/- %6.4f and b = %6.4f +/- %6.4f "
stringl = stringl + "\n" + (
r"$\mathbf{Y_{%s}} = \mathbf{a} + \mathbf{b} \times \mathbf{X}$ with $\mathbf{a} = %.2f \pm %.2f$ and $\mathbf{b} = %.2f \pm %.2f$ "
% (labeldata[i_data], fitobj.params[0],
np.sqrt(fitobj.stderr[0]), fitobj.params[1],
np.sqrt(fitobj.stderr[1])))
else:
ydummy, upperband, lowerband = confidence_band(
x, dfdp, confprob, fitobj2, model)
labelconf = u"CI (" + str(
confprob) + u"%) relative weighting in Y"
if i_data == 0:
#string = (u"y = a + b * x with a = %6.4f +/- %6.4f and b = %6.4f +/- %6.4f "
stringl = (
r"$\mathbf{Y_{%s}} = \mathbf{a} + \mathbf{b} \times \mathbf{X}$ with $\mathbf{a} = %.2f \pm %.2f$ and $\mathbf{b} = %.2f \pm %.2f$ "
% (labeldata[i_data], fitobj2.params[0],
np.sqrt(fitobj2.stderr[0]), fitobj2.params[1],
np.sqrt(fitobj2.stderr[1])))
else:
#string = (u"y = a + b * x with a = %6.4f +/- %6.4f and b = %6.4f +/- %6.4f "
stringl = stringl + "\n" + (
r"$\mathbf{Y_{%s}} = \mathbf{a} + \mathbf{b} \times \mathbf{X}$ with $\mathbf{a} = %.2f \pm %.2f$ and $\mathbf{b} = %.2f \pm %.2f$ "
% (labeldata[i_data], fitobj2.params[0],
np.sqrt(fitobj2.stderr[0]), fitobj2.params[1],
np.sqrt(fitobj2.stderr[1])))
else:
ydummy, upperband, lowerband = confidence_band(
x, dfdp, confprob, fitobj3, model)
labelconf = u"CI (" + str(confprob) + u"%) with no weighting"
if i_data == 0:
#string = (u"y = a + b * x with a = %6.4f +/- %6.4f and b = %6.4f +/- %6.4f "
stringl = (
r"$\mathbf{y_{%s}} = \mathbf{a} + \mathbf{b} \times x$ with $\mathbf{a} = %.2f \pm %.2f$ and $\mathbf{b} = %.2f \pm %.2f$ "
% (labeldata[i_data], fitobj3.params[0],
np.sqrt(fitobj3.stderr[0]), fitobj3.params[1],
np.sqrt(fitobj3.stderr[1])))
else:
#string = (u"y = a + b * x with a = %6.4f +/- %6.4f and b = %6.4f +/- %6.4f "
stringl = stringl + "\n" + (
r"$\mathbf{Y_{%s}} = \mathbf{a} + \mathbf{b} \times \mathbf{X}$ with $\mathbf{a} = %.2f \pm %.2f$ and $\mathbf{b} = %.2f \pm %.2f$ "
% (labeldata[i_data], fitobj3.params[0],
np.sqrt(fitobj3.stderr[0]), fitobj3.params[1],
np.sqrt(fitobj3.stderr[1])))
#verts = zip(x, lowerband) + zip(x[::-1], upperband[::-1])
# Because we need to invert X and Y for the graph
verts = list(zip(lowerband, x)) + list(zip(upperband[::-1], x[::-1]))
# Close output file
f1w.close()
#if rangex:
# X = np.linspace(rangex[0], rangex[1], 50)
#else:
# d = (x.max() - x.min())/10
# X = np.linspace(x.min()-d, x.max()+d, 50)
#if rangey:
# Y = np.linspace(rangey[0], rangey[1], 50)
#else:
# d = (y.max() - y.min())/10
# Y = np.linspace(y.min()-d, y.max()+d, 50)
# Call the plotting function
#axes = plotgraphreg(struct[i_data], x, y, errx, erry, X, Y,
axes = plotgraphreg(struct[i_data], x, y, errx, erry, colors[i_data],
labeldata[i_data], statstypes, a_will, b_will,
verts, confprob, fitobj, fitobj2, fitobj3)
# Set graph characteristics/attributes
plt.xlabel(labelx)
plt.ylabel(labely)
plt.grid(True)
plt.title(stringl, size=fontsz)
# clean the legend from duplicates
handles, labels = plt.gca().get_legend_handles_labels()
by_label = OrderedDict(zip(labels, handles))
plt.legend(by_label.values(), by_label.keys(), loc="best")
# set the X and Y limits
if rangex:
axes.set_xlim([rangex[0], rangex[1]])
if rangey:
axes.set_ylim(bottom=rangey[0], top=rangey[1])
plt.savefig(work_dir[0] + output + u'.pdf')
plt.close()
Bin_I = binfrac[index]
BinErr_I = binfracerr[index]
Ro_I = ro[index]
### And now to calculate error in log-space
Ro_Log = np.log10(Ro_I)
Bin_Log = np.log10(Bin_I)
BinErr_Log = np.log10(Bin_I + BinErr_I) - np.log10(Bin_I)
xn = np.linspace(min(Bin_Log - BinErr_Log), max(Bin_Log + BinErr_Log), 100)
#%% Setting up the ODR process, the plot arrays and chi squared ###
data = RealData(Bin_Log[0:14], Ro_Log[0:14], sx=BinErr_Log[0:14])
modelLM = Model(funcLin)
odrLM = ODR(data, modelLM, [0.70, -1.95])
odrLM.set_job(fit_type=0)
outLM = odrLM.run()
yn = funcLin(outLM.beta, xn)
#%% Plotting binary fraction vs density cleaned up
plt.figure(2)
plt.clf()
plt.plot(Bin_Log, Ro_Log, 'k.')
plt.errorbar(Bin_Log,
Ro_Log,
xerr=BinErr_Log,
ecolor='k',
ls='none',
elinewidth=0.5,
N = 40
a0 = -6; b0 = 1; c0 = 0.5
x = numpy.linspace(-4, 3.0, N)
y = model((a0,b0,c0),x) + normal(0.0, 0.5, N) # Mean 0, sigma 1
errx = normal(0.3, 0.3, N)
erry = normal(0.2, 0.4, N)
print("\nTrue model values [a0,b0,c0]:", [a0,b0,c0])
beta0 = [1,1,1]
#beta0 = [1.8,-0.5,0.1]
print("\nODR:")
print("==========")
from scipy.odr import Data, Model, ODR, RealData, odr_stop
linear = Model(model)
mydata = RealData(x, y, sx=errx, sy=erry)
myodr = ODR(mydata, linear, beta0=beta0, maxit=5000)
#myodr.set_job(2)
myoutput = myodr.run()
print("Fitted parameters: ", myoutput.beta)
print("Covariance errors: ", numpy.sqrt(myoutput.cov_beta.diagonal()))
print("Standard errors: ", myoutput.sd_beta)
print("Minimum chi^2: ", myoutput.sum_square)
print("Minimum (reduced)chi^2: ", myoutput.res_var)
beta = myoutput.beta
# Prepare fit routine
fitobj = kmpfit.Fitter(residuals=residuals, data=(x, y, errx, erry),
xtol=1e-12, gtol=1e-12)
fitobj.fit(params0=beta0)
print("\n\n======== Results kmpfit with effective variance =========")
def fitDoubleGaussians(charges, charge_err, data, data_err):
# some simple peakfinding to find the first and second peaks
peak1_value = np.max(data)
peak1_bin = [x for x in range(len(data)) if data[x]==peak1_value][0]
peak1_charge = charges[peak1_bin]
# Preliminary fits to get better estimates of parameters...
first_peak_data = RealData(charges[peak1_bin-30:peak1_bin+30],
data[peak1_bin-30:peak1_bin+30],
charge_err[peak1_bin-30:peak1_bin+30],
data_err[peak1_bin-30:peak1_bin+30],)
first_peak_model = Model(gauss_fit)
first_peak_odr = ODR(first_peak_data, first_peak_model,
[peak1_value, peak1_charge, 0.1*peak1_charge])
first_peak_odr.set_job(fit_type=2)
first_peak_output = first_peak_odr.run()
first_peak_params = first_peak_output.beta
#second_peak_params, covariance = curve_fit(gauss_fit2,
# data[peak1_bin-30:peak1_bin+30],
# data[peak1_bin-30:peak1_bin+30],
# p0 = [peak1_value, peak1_charge, 0.1*peak1_charge])
# subtract the largest peak so we can search for the other one
updated_data = data-gauss_fit(first_peak_params, charges)
#updated_data = data[:int(len(data)*2.0/3.0)]
peak2_value = np.max(updated_data)
peak2_bin = [x for x in range(len(updated_data)) if updated_data[x]==peak2_value][0]
peak2_charge = charges[peak2_bin]
#first_peak_params, covariance = curve_fit(gauss_fit2,
# updated_data[peak2_bin-30:peak2_bin+30],
# updated_data[peak2_bin-30:peak2_bin+30],
# p0 = [peak2_value, peak2_charge, 0.1*peak2_charge])
# and the second peak...
second_peak_data = RealData(charges[peak2_bin-30:peak2_bin+30],
data[peak2_bin-30:peak2_bin+30],
charge_err[peak2_bin-30:peak2_bin+30],
data_err[peak2_bin-30:peak2_bin+30],)
second_peak_model = Model(gauss_fit)
second_peak_odr = ODR(second_peak_data, second_peak_model,
[peak2_value, peak2_charge, first_peak_params[2]])
second_peak_odr.set_job(fit_type=2)
second_peak_output = second_peak_odr.run()
second_peak_params = second_peak_output.beta
# Now fit both gaussians simultaneously
double_peak_data = RealData(charges, data,
charge_err, data_err)
double_peak_model = Model(double_gauss_fit)
double_peak_odr = ODR(double_peak_data, double_peak_model,
[first_peak_params[0], first_peak_params[1], first_peak_params[2],
second_peak_params[0], second_peak_params[1], second_peak_params[2]])
double_peak_odr.set_job(fit_type=0)
double_peak_output = double_peak_odr.run()
double_peak_params = double_peak_output.beta
double_peak_sigmas = double_peak_output.sd_beta
#double_peak_params = [first_peak_params[0], first_peak_params[1], first_peak_params[2],
# second_peak_params[0], second_peak_params[1], second_peak_params[2]]
return double_peak_params, double_peak_sigmas
