def regressor(self):
# print self.name, self.fit, self.__class__.__name__
is_mean = 'average' in self.fit.lower()
reg = self._regressor
if reg is None:
if is_mean:
reg = self._mean_regressor_factory()
else:
# print 'doing import of regressor {}'.format(self.__class__)
# st=time.time()
from pychron.core.regression.ols_regressor import PolynomialRegressor
# print 'doing import of regressor {}'.format(time.time()-st)
reg = PolynomialRegressor(tag=self.name,
xs=self.offset_xs,
ys=self.ys,
# fit=self.fit,
# filter_outliers_dict=self.filter_outliers_dict,
error_calc_type=self.error_type)
elif is_mean and not isinstance(reg, MeanRegressor):
reg = self._mean_regressor_factory()
if not is_mean:
reg.set_degree(fit_to_degree(self.fit), refresh=False)
reg.filter_outliers_dict = self.filter_outliers_dict
reg.trait_set(xs=self.offset_xs, ys=self.ys)
reg.calculate()
self._regressor = reg
return reg
def setUp(self):
self.xs = np.linspace(0, 10, 10)
# self.ys = np.random.normal(self.xs, 1)
# print self.ys
self.ys = [
-1.8593967, 3.15506254, 1.82144207, 4.58729807, 4.95813564,
5.71229382, 7.04611731, 8.14459843, 10.27429285, 10.10989719
]
'''
draper and smith p.8
'''
self.xs = [
35.3, 29.7, 30.8, 58.8, 61.4, 71.3, 74.4, 76.7, 70.7, 57.5, 46.4,
28.9, 28.1, 39.1, 46.8, 48.5, 59.3, 70, 70, 74.5, 72.1, 58.1, 44.6,
33.4, 28.6
]
self.ys = [
10.98, 11.13, 12.51, 8.4, 9.27, 8.73, 6.36, 8.50, 7.82, 9.14, 8.24,
12.19, 11.88, 9.57, 10.94, 9.58, 10.09, 8.11, 6.83, 8.88, 7.68,
8.47, 8.86, 10.36, 11.08
]
self.slope = -0.0798
self.intercept = 13.623
self.Xk = 28.6
self.ypred_k = 0.3091
xs = self.xs
ys = self.ys
ols = PolynomialRegressor(xs=xs, ys=ys, fit='linear')
self.ols = ols
def _get_regressor(self):
# print '{} getting regerssior'.format(self.name)
# try:
if self.fit.lower() =='average':
reg = self._mean_regressor_factory()
else:
reg = PolynomialRegressor(xs=self.xs,
ys=self.ys,
degree=self.fit,
error_type=self.error_type,
filter_outliers_dict=self.filter_outliers_dict)
# except Exception, e:
# reg = PolynomialRegressor(xs=self.xs, ys=self.ys,
# degree=self.fit,
# filter_outliers_dict=self.filter_outliers_dict)
reg.calculate()
return reg
def regressor(self):
fit = self.fit
if fit is None:
fit = 'linear'
self.fit = fit
lfit = fit.lower()
is_mean = 'average' in lfit
is_expo = lfit == 'exponential'
is_poly = not (is_mean or is_expo)
reg = self._regressor
if reg is None:
if is_mean:
reg = MeanRegressor()
elif is_expo:
reg = ExponentialRegressor()
else:
reg = PolynomialRegressor()
elif is_poly and not isinstance(reg, PolynomialRegressor):
reg = PolynomialRegressor()
elif is_mean and not isinstance(reg, MeanRegressor):
reg = MeanRegressor()
elif is_expo and not isinstance(reg, ExponentialRegressor):
reg = ExponentialRegressor()
if is_poly:
reg.set_degree(fit_to_degree(fit), refresh=False)
reg.trait_set(xs=self.offset_xs,
ys=self.ys,
error_calc_type=self.error_type or 'SEM',
filter_outliers_dict=self.filter_outliers_dict,
tag=self.name)
if self.truncate:
reg.set_truncate(self.truncate)
reg.calculate()
self._regressor = reg
return reg
def _poly_regress(self, r, x, y, ox, oy, index, fit, fod, apply_filter,
scatter, selection):
if hasattr(scatter, 'yerror'):
es = scatter.yerror.get_data()
if selection:
es = delete(es, selection, 0)
if r is None or not isinstance(r, WeightedPolynomialRegressor):
r = WeightedPolynomialRegressor(yserr=es)
else:
if r is None or not isinstance(r, PolynomialRegressor):
r = PolynomialRegressor()
r.trait_set(xs=x, ys=y, degree=fit)
if apply_filter:
r = self._apply_outlier_filter(r, ox, oy, index, fod)
return r
def _poly_regress(self, scatter, r, fit):
if hasattr(scatter, 'yerror'):
if r is None or not isinstance(r, WeightedPolynomialRegressor):
r = WeightedPolynomialRegressor()
else:
if r is None or not isinstance(r, PolynomialRegressor):
r = PolynomialRegressor()
self._set_regressor(scatter, r)
r.trait_set(degree=fit)
r.set_truncate(scatter.truncate)
r.calculate()
if r.ys.shape[0] < fit + 1:
return
self._set_excluded(scatter, r)
return r
def _poly_regress(self, scatter, r, fit):
from pychron.core.regression.ols_regressor import PolynomialRegressor
from pychron.core.regression.wls_regressor import WeightedPolynomialRegressor
if hasattr(scatter, 'yerror') and any(scatter.yerror.get_data()):
if r is None or not isinstance(r, WeightedPolynomialRegressor):
r = WeightedPolynomialRegressor()
else:
if r is None or not isinstance(r, PolynomialRegressor):
r = PolynomialRegressor()
self._set_regressor(scatter, r)
r.trait_set(degree=fit)
r.set_truncate(scatter.truncate)
if r.ys.shape[0] < fit + 1:
return
r.calculate()
self._set_excluded(scatter, r)
return r
def _get_regressor(self):
if 'average' in self.fit.lower():
reg = self._mean_regressor_factory()
else:
# print 'doing import of regresor {}'.format(self.__class__)
# st=time.time()
from pychron.core.regression.ols_regressor import PolynomialRegressor
# print 'doing import of regresor {}'.format(time.time()-st)
reg = PolynomialRegressor(tag=self.name,
xs=self.offset_xs,
ys=self.ys,
# fit=self.fit,
# filter_outliers_dict=self.filter_outliers_dict,
error_calc_type=self.error_type)
reg.set_degree(self.fit, refresh=False)
reg.filter_outliers_dict = self.filter_outliers_dict
reg.calculate()
return reg
def results():
x, y, e = get_data()
reg = PolynomialRegressor(degree=1, xs=x, ys=y)
mi, ma = min(x), max(x)
pad = (ma - mi) * 0.1
fxs = linspace(mi - pad, ma + pad)
plt.errorbar(x, y, yerr=e,)
reg.calculate()
l, u = reg.calculate_error_envelope(fxs, error_calc='CI')
plt.plot(fxs, l, 'b')
plt.plot(fxs, u, 'b')
# plt.plot(fxs, reg.predict(fxs), 'b-')
print('Age={}, SD={} SEM={} CI={}'.format(reg.predict(328), reg.predict_error(328), reg.predict_error(328, 'SEM'),
reg.predict_error(328, 'CI')))
reg = WeightedPolynomialRegressor(degree=1, xs=x, ys=y, yserr=e)
reg.calculate()
plt.plot(fxs, reg.predict(fxs), 'g')
l,u=reg.calculate_error_envelope(fxs, error_calc='CI')
plt.plot(fxs, l, 'r')
plt.plot(fxs, u, 'r')
print('Weighted fit Age={}, SD={} SEM={} CI={}'.format(reg.predict(328),
reg.predict_error(328), reg.predict_error(328,'SEM'),
reg.predict_error(328, 'CI')))
plt.show()
def testSYX(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
self.assertAlmostEqual(reg.get_syx(), 0.297, delta=0.01)
def testSSX(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
self.assertAlmostEqual(reg.get_ssx(), 8301.389, delta=0.01)
def testSYX(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
self.assertAlmostEqual(reg.get_syx(), 0.297, delta=0.01)
def testLower(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
l, u = reg.calculate_ci([0])
self.assertAlmostEqual(l[0], 8.25, delta=0.01)
def testUpper(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
l, u = reg.calculate_ci([0, 10, 100])
for ui, ti in zip(u, [9.16, 8.56, 3.83]):
self.assertAlmostEqual(ui, ti, delta=0.01)
def _test_fired(self):
import numpy as np
self.db.connect()
xbase = np.linspace(430, 580, 150)
# ybase = np.zeros(150)
# cddybase = np.zeros(150)
ybase = np.random.random(150)
cddybase = np.random.random(150) * 0.001
base = [zip(xbase, ybase), zip(xbase, ybase), zip(xbase, ybase), zip(xbase, ybase), zip(xbase, cddybase)]
xsig = np.linspace(20, 420, 410)
# y40 = np.ones(410) * 680
# y39 = np.ones(410) * 107
# y38 = np.zeros(410) * 1.36
# y37 = np.zeros(410) * 0.5
# y36 = np.ones(410) * 0.001
y40 = 680 - 0.1 * xsig
y39 = 107 - 0.1 * xsig
y38 = np.zeros(410) * 1.36
y37 = np.zeros(410) * 0.5
y36 = 1 + 0.1 * xsig
sig = [zip(xsig, y40), zip(xsig, y39), zip(xsig, y38), zip(xsig, y37), zip(xsig, y36)]
regbs = MeanRegressor(xs=xbase, ys=ybase)
cddregbs = MeanRegressor(xs=xbase, ys=cddybase)
reg = PolynomialRegressor(xs=xsig, ys=y40, fit="linear")
reg1 = PolynomialRegressor(xs=xsig, ys=y39, fit="linear")
reg2 = PolynomialRegressor(xs=xsig, ys=y38, fit="linear")
reg3 = PolynomialRegressor(xs=xsig, ys=y37, fit="linear")
reg4 = PolynomialRegressor(xs=xsig, ys=y36, fit="linear")
keys = [("H1", "Ar40"), ("AX", "Ar39"), ("L1", "Ar38"), ("L2", "Ar37"), ("CDD", "Ar36")]
regresults = (
dict(
Ar40=ufloat(reg.predict(0), reg.predict_error(0)),
Ar39=ufloat(reg1.predict(0), reg1.predict_error(0)),
Ar38=ufloat(reg2.predict(0), reg2.predict_error(0)),
Ar37=ufloat(reg3.predict(0), reg3.predict_error(0)),
Ar36=ufloat(reg4.predict(0), reg4.predict_error(0)),
),
dict(
Ar40=ufloat(regbs.predict(0), regbs.predict_error(0)),
Ar39=ufloat(regbs.predict(0), regbs.predict_error(0)),
Ar38=ufloat(regbs.predict(0), regbs.predict_error(0)),
Ar37=ufloat(regbs.predict(0), regbs.predict_error(0)),
Ar36=ufloat(cddregbs.predict(0), cddregbs.predict_error(0)),
),
)
blanks = [ufloat(0, 0.1), ufloat(0.1, 0.001), ufloat(0.01, 0.001), ufloat(0.01, 0.001), ufloat(0.00001, 0.0001)]
fits = (
dict(zip(["Ar40", "Ar39", "Ar38", "Ar37", "Ar36"], ["Linear", "Linear", "Linear", "Linear", "Linear"])),
dict(
zip(
["Ar40", "Ar39", "Ar38", "Ar37", "Ar36"],
["Average Y", "Average Y", "Average Y", "Average Y", "Average Y"],
)
),
)
mass_spectrometer = "obama"
extract_device = "Laser Furnace"
extract_value = 10
position = 1
duration = 10
first_stage_delay = 0
second_stage_delay = 30
tray = "100-hole"
runscript_name = "Foo"
runscript_text = "this is a test script"
self.add_analysis(
"4318",
"500",
"",
"4318",
base,
sig,
blanks,
keys,
regresults,
fits,
mass_spectrometer,
extract_device,
tray,
position,
extract_value, # power requested
extract_value, # power achieved
duration, # total extraction
duration, # time at extract_value
first_stage_delay,
second_stage_delay,
runscript_name,
runscript_text,
)
def testUpper(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
l, u = reg.calculate_ci([0, 10, 100])
for ui, ti in zip(u, [9.16, 8.56, 3.83]):
self.assertAlmostEqual(ui, ti, delta=0.01)
def testLower(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
l, u = reg.calculate_ci([0])
self.assertAlmostEqual(l[0], 8.25, delta=0.01)
def _test_fired(self):
import numpy as np
self.db.connect()
xbase = np.linspace(430, 580, 150)
# ybase = np.zeros(150)
# cddybase = np.zeros(150)
ybase = np.random.random(150)
cddybase = np.random.random(150) * 0.001
base = [
zip(xbase, ybase),
zip(xbase, ybase),
zip(xbase, ybase),
zip(xbase, ybase),
zip(xbase, cddybase),
]
xsig = np.linspace(20, 420, 410)
# y40 = np.ones(410) * 680
# y39 = np.ones(410) * 107
# y38 = np.zeros(410) * 1.36
# y37 = np.zeros(410) * 0.5
# y36 = np.ones(410) * 0.001
y40 = 680 - 0.1 * xsig
y39 = 107 - 0.1 * xsig
y38 = np.zeros(410) * 1.36
y37 = np.zeros(410) * 0.5
y36 = 1 + 0.1 * xsig
sig = [
zip(xsig, y40),
zip(xsig, y39),
zip(xsig, y38),
zip(xsig, y37),
zip(xsig, y36),
]
regbs = MeanRegressor(xs=xbase, ys=ybase)
cddregbs = MeanRegressor(xs=xbase, ys=cddybase)
reg = PolynomialRegressor(xs=xsig, ys=y40, fit='linear')
reg1 = PolynomialRegressor(xs=xsig, ys=y39, fit='linear')
reg2 = PolynomialRegressor(xs=xsig, ys=y38, fit='linear')
reg3 = PolynomialRegressor(xs=xsig, ys=y37, fit='linear')
reg4 = PolynomialRegressor(xs=xsig, ys=y36, fit='linear')
keys = [
('H1', 'Ar40'),
('AX', 'Ar39'),
('L1', 'Ar38'),
('L2', 'Ar37'),
('CDD', 'Ar36'),
]
regresults = (dict(
Ar40=ufloat(reg.predict(0), reg.predict_error(0)),
Ar39=ufloat(reg1.predict(0), reg1.predict_error(0)),
Ar38=ufloat(reg2.predict(0), reg2.predict_error(0)),
Ar37=ufloat(reg3.predict(0), reg3.predict_error(0)),
Ar36=ufloat(reg4.predict(0), reg4.predict_error(0)),
),
dict(Ar40=ufloat(regbs.predict(0),
regbs.predict_error(0)),
Ar39=ufloat(regbs.predict(0),
regbs.predict_error(0)),
Ar38=ufloat(regbs.predict(0),
regbs.predict_error(0)),
Ar37=ufloat(regbs.predict(0),
regbs.predict_error(0)),
Ar36=ufloat(cddregbs.predict(0),
cddregbs.predict_error(0))))
blanks = [
ufloat(0, 0.1),
ufloat(0.1, 0.001),
ufloat(0.01, 0.001),
ufloat(0.01, 0.001),
ufloat(0.00001, 0.0001),
]
fits = (dict(
zip(['Ar40', 'Ar39', 'Ar38', 'Ar37', 'Ar36'],
['Linear', 'Linear', 'Linear', 'Linear', 'Linear'])),
dict(
zip(['Ar40', 'Ar39', 'Ar38', 'Ar37', 'Ar36'], [
'Average Y', 'Average Y', 'Average Y', 'Average Y',
'Average Y'
])))
mass_spectrometer = 'obama'
extract_device = 'Laser Furnace'
extract_value = 10
position = 1
duration = 10
first_stage_delay = 0
second_stage_delay = 30
tray = '100-hole'
runscript_name = 'Foo'
runscript_text = 'this is a test script'
self.add_analysis(
'4318',
'500',
'',
'4318',
base,
sig,
blanks,
keys,
regresults,
fits,
mass_spectrometer,
extract_device,
tray,
position,
extract_value, # power requested
extract_value, # power achieved
duration, # total extraction
duration, # time at extract_value
first_stage_delay,
second_stage_delay,
runscript_name,
runscript_text,
)
def testSSX(self):
reg = PolynomialRegressor(xs=self.x, ys=self.y, degree=1)
self.assertAlmostEqual(reg.get_ssx(), 8301.389, delta=0.01)
def results():
x, y, e = get_data()
reg = PolynomialRegressor(degree=1, xs=x, ys=y)
mi, ma = min(x), max(x)
pad = (ma - mi) * 0.1
fxs = linspace(mi - pad, ma + pad)
plt.errorbar(
x,
y,
yerr=e,
)
reg.calculate()
l, u = reg.calculate_error_envelope(fxs, error_calc='CI')
plt.plot(fxs, l, 'b')
plt.plot(fxs, u, 'b')
# plt.plot(fxs, reg.predict(fxs), 'b-')
print('Age={}, SD={} SEM={} CI={}'.format(reg.predict(328),
reg.predict_error(328),
reg.predict_error(328, 'SEM'),
reg.predict_error(328, 'CI')))
reg = WeightedPolynomialRegressor(degree=1, xs=x, ys=y, yserr=e)
reg.calculate()
plt.plot(fxs, reg.predict(fxs), 'g')
l, u = reg.calculate_error_envelope(fxs, error_calc='CI')
plt.plot(fxs, l, 'r')
plt.plot(fxs, u, 'r')
print('Weighted fit Age={}, SD={} SEM={} CI={}'.format(
reg.predict(328), reg.predict_error(328),
reg.predict_error(328, 'SEM'), reg.predict_error(328, 'CI')))
plt.show()