def test_thaw_model(string, clean_ui):
"""Can we thaw a model?
We use a model with an alwaysfrozen parameter.
"""
mdl = ui.create_model_component("logparabola", "mdl")
ui.set_source(mdl)
mdl.c1.freeze()
mdl.ampl.freeze()
assert mdl.ref.frozen
assert mdl.c1.frozen
assert not mdl.c2.frozen
assert mdl.ampl.frozen
assert ui.get_num_par_thawed() == 1
assert ui.get_num_par_frozen() == 3
if string:
ui.thaw("mdl")
else:
ui.thaw(mdl)
assert mdl.ref.frozen
assert not mdl.c1.frozen
assert not mdl.c2.frozen
assert not mdl.ampl.frozen
assert ui.get_num_par_thawed() == 3
assert ui.get_num_par_frozen() == 1
def fit_sbp():
ui.set_model(sbp)
ui.thaw(sbp)
ui.freeze(sbp.x_r)
ui.freeze(sbp.gamma1)
ui.fit()
def fit_pix_values(t_ccd, esec, id=1):
logger = logging.getLogger("sherpa")
logger.setLevel(logging.WARN)
data_id = id
ui.clean()
ui.set_method('simplex')
ui.load_user_model(dark_scale_model, 'model')
ui.add_user_pars('model', ['scale', 'dark_t_ref'])
ui.set_model(data_id, 'model')
ui.load_arrays(
data_id,
np.array(t_ccd),
np.array(esec),
)
ui.set_staterror(data_id, 30 * np.ones(len(t_ccd)))
model.scale.val = 0.588
model.scale.min = 0.3
model.scale.max = 1.0
model.dark_t_ref.val = 500
ui.freeze(model.scale)
# If more than 5 degrees in the temperature range,
# thaw and fit for model.scale. Else just use/return
# the fit of dark_t_ref
if np.max(t_ccd) - np.min(t_ccd) > 2:
# Fit first for dark_t_ref
ui.fit(data_id)
ui.thaw(model.scale)
ui.fit(data_id)
return ui.get_fit_results(), ui.get_model(data_id)
def test_thaw_no_arguments(clean_ui):
"""This is a no-op"""
mdl = ui.create_model_component("logparabola", "mdl")
ui.set_source(mdl)
mdl.c1.freeze()
assert ui.get_num_par_thawed() == 2
assert ui.get_num_par_frozen() == 2
ui.thaw()
assert ui.get_num_par_thawed() == 2
assert ui.get_num_par_frozen() == 2
def test_thaw_invalid_arguments(string, clean_ui):
"""We error out with an invalid argument"""
mdl = ui.create_model_component("logparabola", "mdl")
ui.set_source(mdl)
with pytest.raises(ArgumentTypeErr) as ae:
if string:
ui.thaw("1")
else:
ui.thaw(1)
assert str(
ae.value
) == "'par' must be a parameter or model object or expression string"
def test_thaw_thawed_parameter(string, clean_ui):
"""Can we thaw a thawed parameter? String argument"""
mdl = ui.create_model_component("polynom1d", "mdl")
ui.set_source(mdl)
assert not mdl.c0.frozen
assert ui.get_num_par_thawed() == 1
assert ui.get_num_par_frozen() == 9
if string:
ui.thaw("mdl.c0")
else:
ui.thaw(mdl.c0)
assert not mdl.c0.frozen
assert ui.get_num_par_thawed() == 1
assert ui.get_num_par_frozen() == 9
def fit_model(
model,
comm=None,
method='simplex',
config=None,
nofit=None,
freeze_pars=freeze_pars,
thaw_pars=[],
):
dummy_data = np.zeros(1)
dummy_times = np.arange(1)
ui.load_arrays(1, dummy_times, dummy_data)
ui.set_method(method)
ui.get_method().config.update(config or sherpa_configs.get(method, {}))
ui.load_user_model(CalcModel(model, comm), 'xijamod')
ui.add_user_pars('xijamod', model.parnames)
ui.set_model(1, 'xijamod')
fit_parnames = set()
for parname, parval in zip(model.parnames, model.parvals):
getattr(xijamod, parname).val = parval
fit_parnames.add(parname)
if any([re.match(x + '$', parname) for x in freeze_pars]):
fit_logger.info('Freezing ' + parname)
ui.freeze(getattr(xijamod, parname))
fit_parnames.remove(parname)
if any([re.match(x + '$', parname) for x in thaw_pars]):
fit_logger.info('Thawing ' + parname)
ui.thaw(getattr(xijamod, parname))
fit_parnames.add(parname)
if 'tau' in parname:
getattr(xijamod, parname).min = 0.1
calc_stat = CalcStat(model, comm)
ui.load_user_stat('xijastat', calc_stat, lambda x: np.ones_like(x))
ui.set_stat(xijastat)
if fit_parnames and not nofit:
ui.fit(1)
else:
model.calc()
def fit_model(model,
comm=None,
method='simplex',
config=None,
nofit=None,
freeze_pars=freeze_pars,
thaw_pars=[],
):
dummy_data = np.zeros(1)
dummy_times = np.arange(1)
ui.load_arrays(1, dummy_times, dummy_data)
ui.set_method(method)
ui.get_method().config.update(config or sherpa_configs.get(method, {}))
ui.load_user_model(CalcModel(model, comm), 'xijamod')
ui.add_user_pars('xijamod', model.parnames)
ui.set_model(1, 'xijamod')
fit_parnames = set()
for parname, parval in zip(model.parnames, model.parvals):
getattr(xijamod, parname).val = parval
fit_parnames.add(parname)
if any([re.match(x + '$', parname) for x in freeze_pars]):
fit_logger.info('Freezing ' + parname)
ui.freeze(getattr(xijamod, parname))
fit_parnames.remove(parname)
if any([re.match(x + '$', parname) for x in thaw_pars]):
fit_logger.info('Thawing ' + parname)
ui.thaw(getattr(xijamod, parname))
fit_parnames.add(parname)
if 'tau' in parname:
getattr(xijamod, parname).min = 0.1
calc_stat = CalcStat(model, comm)
ui.load_user_stat('xijastat', calc_stat, lambda x: np.ones_like(x))
ui.set_stat(xijastat)
if fit_parnames and not nofit:
ui.fit(1)
else:
model.calc()
def fit_pix_values(t_ccd, esec, id=1):
logger = logging.getLogger("sherpa")
logger.setLevel(logging.WARN)
data_id = id
ui.clean()
ui.set_method("simplex")
ui.load_user_model(dark_scale_model, "model")
ui.add_user_pars("model", ["scale", "dark_t_ref"])
ui.set_model(data_id, "model")
ui.load_arrays(data_id, np.array(t_ccd), np.array(esec), 0.1 * np.ones(len(t_ccd)))
model.scale.val = 0.70
model.dark_t_ref.val = 500
ui.freeze(model.scale)
# If more than 5 degrees in the temperature range,
# thaw and fit for model.scale. Else just use/return
# the fit of dark_t_ref
ui.fit(data_id)
ui.thaw(model.scale)
ui.fit(data_id)
return ui.get_fit_results(), ui.get_model(data_id)
def test_thaw_alwaysfrozen_parameter(string, clean_ui):
"""Can we thaw an always-frozen parameter?"""
mdl = ui.create_model_component("logparabola", "mdl")
ui.set_source(mdl)
assert mdl.ref.alwaysfrozen
assert mdl.ref.frozen
assert ui.get_num_par_thawed() == 3
assert ui.get_num_par_frozen() == 1
with pytest.raises(ParameterErr) as pe:
if string:
ui.thaw("mdl.ref")
else:
ui.thaw(mdl.ref)
assert mdl.ref.frozen
assert ui.get_num_par_thawed() == 3
assert ui.get_num_par_frozen() == 1
assert str(
pe.value) == "parameter mdl.ref is always frozen and cannot be thawed"
def test_thaw_multi_arguments(string, clean_ui):
"""Check we can combine model and parameters"""
mdl1 = ui.create_model_component("logparabola", "mdl1")
mdl2 = ui.create_model_component("polynom1d", "mdl2")
ui.set_source(mdl1 + mdl2)
mdl1.c1.freeze()
mdl1.ampl.freeze()
assert mdl1.c1.frozen
assert mdl1.ampl.frozen
assert mdl2.c2.frozen
assert ui.get_num_par_thawed() == 2
assert ui.get_num_par_frozen() == 12
if string:
ui.thaw("mdl1", "mdl2.c2")
else:
ui.thaw(mdl1, mdl2.c2)
assert not mdl1.c1.frozen
assert not mdl1.ampl.frozen
assert not mdl2.c2.frozen
assert ui.get_num_par_thawed() == 5
assert ui.get_num_par_frozen() == 9
def tst_ui(self, thaw_c1):
ui.load_arrays(1, self._x, self._y, self._e)
ui.set_source(1, ui.polynom1d.mdl)
if thaw_c1:
ui.thaw(mdl.c1)
ui.thaw(mdl.c2)
mdl.c2 = 1
ui.fit()
if not thaw_c1:
ui.thaw(mdl.c1)
ui.fit()
ui.conf()
result = ui.get_conf_results()
self.cmp_results(result)
def _fit_poly(fit_data, evt_times, degree, data_id=0):
"""
Given event data transformed into Y or Z angle positions, and a degree of the desired
fit polynomial, fit a polynomial to the data.
:param fit_data: event y or z angle position data
:param evt_times: times of event/fit_data
:param degree: degree of polynomial to use for the fit model
:param data_id: sherpa dataset id to use for the fit
:returns: (sherpa model plot, sherpa model)
"""
# Set initial value for fit data position error
init_error = 1
ui.clean()
ui.load_arrays(data_id, evt_times - evt_times[0], fit_data,
np.zeros_like(fit_data) + init_error)
v2("Fitting a line to the data to get reduced stat errors")
# First just fit a line to get reduced errors on this set
ui.polynom1d.line
ui.set_model(data_id, 'line')
ui.thaw('line.c1')
ui.fit(data_id)
fit = ui.get_fit_results()
calc_error = init_error * np.sqrt(fit.rstat)
ui.set_staterror(data_id, calc_error)
# Then fit the specified model
v2("Fitting a polynomial of degree {} to the data".format(degree))
ui.polynom1d.fitpoly
ui.freeze('fitpoly')
# Thaw the coefficients requested by the degree of the desired polynomial
ui.thaw('fitpoly.c0')
fitpoly.c0.val = 0
for deg in range(1, 1 + degree):
ui.thaw("fitpoly.c{}".format(deg))
ui.set_model(data_id, 'fitpoly')
ui.fit(data_id)
# Let's screw up Y on purpose
if data_id == 0:
fitpoly.c0.val = 0
fitpoly.c1.val = 7.5e-05
fitpoly.c2.val = -1.0e-09
fitpoly.c3.val = 0
fitpoly.c4.val = 0
mp = ui.get_model_plot(data_id)
model = ui.get_model(data_id)
return mp, model
def fit_gauss_sbp():
g1 = ui.gauss1d.g1
ui.set_model(sbp + g1)
ui.set_method('simplex')
g1.fwhm = 5.0
g1.pos = 7.0
g1.ampl = 30000.
ui.freeze(sbp.gamma1)
ui.freeze(sbp.gamma2)
ui.freeze(sbp.x_b)
ui.freeze(sbp.x_r)
ui.freeze(g1.fwhm)
ui.freeze(g1.pos)
ui.thaw(g1.ampl)
ui.fit()
ui.thaw(g1.fwhm)
ui.thaw(g1.pos)
ui.fit()
ui.thaw(sbp)
ui.freeze(sbp.x_r)
ui.fit()
def tst_ui(thaw_c1, setUp, clean_ui):
data, mdl = setUp
ui.load_arrays(1, data.x, data.y, data.staterror)
ui.set_source(1, ui.polynom1d.mdl)
if thaw_c1:
ui.thaw(mdl.c1)
ui.thaw(mdl.c2)
mdl.c2 = 1
ui.fit()
if not thaw_c1:
ui.thaw(mdl.c1)
ui.fit()
ui.conf()
result = ui.get_conf_results()
cmp_results(result)
def _fit_poly(fit_data, evt_times, degree, data_id=0):
"""
Given event data transformed into Y or Z angle positions, and a degree of the desired
fit polynomial, fit a polynomial to the data.
:param fit_data: event y or z angle position data
:param evt_times: times of event/fit_data
:param degree: degree of polynomial to use for the fit model
:param data_id: sherpa dataset id to use for the fit
:returns: (sherpa model plot, sherpa model)
"""
# Set initial value for fit data position error
init_error = 1
ui.clean()
ui.load_arrays(data_id, evt_times - evt_times[0], fit_data,
np.zeros_like(fit_data) + init_error)
v2("Fitting a line to the data to get reduced stat errors")
# First just fit a line to get reduced errors on this set
ui.polynom1d.line
ui.set_model(data_id, 'line')
ui.thaw('line.c1')
ui.fit(data_id)
fit = ui.get_fit_results()
calc_error = init_error * np.sqrt(fit.rstat)
ui.set_staterror(data_id, calc_error)
# Then fit the specified model
v2("Fitting a polynomial of degree {} to the data".format(degree))
ui.polynom1d.fitpoly
ui.freeze('fitpoly')
# Thaw the coefficients requested by the degree of the desired polynomial
ui.thaw('fitpoly.c0')
fitpoly.c0.val = 0
for deg in range(1, 1 + degree):
ui.thaw("fitpoly.c{}".format(deg))
ui.set_model(data_id, 'fitpoly')
ui.fit(data_id)
mp = ui.get_model_plot(data_id)
model = ui.get_model(data_id)
return mp, model
def run_fits(obsids, ax, user_pars=None,
fixed_pars=None, guess_pars=None, label='model',
per_obs_dir='per_obs_nfits',
outdir=None, redo=False):
if len(obsids) == 0:
print "No obsids, nothing to fit"
return None
if user_pars is None:
user_pars = USER_PARS
if not os.path.exists(per_obs_dir):
os.makedirs(per_obs_dir)
obsfits = []
for obsid in obsids:
outdir = os.path.join(per_obs_dir, 'obs{:05d}'.format(obsid))
if not os.path.exists(outdir):
os.makedirs(outdir)
model_file = os.path.join(outdir, '{}.pkl'.format(label))
if os.path.exists(model_file) and not redo:
#logger.warn('Using previous fit found in %s' % model_file)
print model_file
mod_pick = open(model_file, 'r')
modelfit = cPickle.load( mod_pick )
mod_pick.close()
obsfits.append(modelfit)
continue
modelfit = {'label': obsid}
ui.clean()
data_id = 0
obsdir = "%s/obs%05d" % (DATADIR, obsid)
tf = open(os.path.join(obsdir,'tilt.pkl'), 'r')
tilt = cPickle.load(tf)
tf.close()
pf = open(os.path.join(obsdir, 'pos.pkl'), 'r')
pos = cPickle.load(pf)
pf.close()
pos_data = pos[ax]
point_error = 5
pos_data_mean = np.mean(pos_data)
ui.set_method('simplex')
# Fit a line to get more reasonable errors
init_staterror = np.zeros(len(pos_data))+point_error
ui.load_arrays(data_id,
pos['time']-pos['time'][0],
pos_data-np.mean(pos_data),
init_staterror)
ui.polynom1d.ypoly
ui.set_model(data_id, 'ypoly')
ui.thaw(ypoly.c0, ypoly.c1)
ui.fit(data_id)
fit = ui.get_fit_results()
calc_staterror = init_staterror * np.sqrt(fit.rstat)
ui.set_staterror(data_id, calc_staterror)
# Confirm those errors
ui.fit(data_id)
fit = ui.get_fit_results()
if ( abs(fit.rstat-1) > .2):
raise ValueError('Reduced statistic not close to 1 for error calc')
# Load up data to do the real model fit
fit_times = pos['time']
tm_func = tilt_model(tilt,
fit_times,
user_pars=user_pars)
ui.get_data(data_id).name = str(obsid)
ui.load_user_model(tm_func, 'tiltm%d' % data_id)
ui.add_user_pars('tiltm%d' % data_id, user_pars)
ui.set_method('simplex')
ui.set_model(data_id, 'tiltm%d' % (data_id))
ui.set_par('tiltm%d.diam' % data_id, 0)
if fixed_pars is not None and ax in fixed_pars:
for par in fixed_pars[ax]:
ui.set_par('tiltm{}.{}'.format(0, par), fixed_pars[ax][par])
ui.freeze('tiltm{}.{}'.format(0, par))
if guess_pars is not None and ax in guess_pars:
for par in guess_pars[ax]:
ui.set_par('tiltm{}.{}'.format(0, par), guess_pars[ax][par])
ui.show_all()
# Fit the tilt model
ui.fit(data_id)
fitres = ui.get_fit_results()
ui.confidence(data_id)
myconf = ui.get_confidence_results()
# save_fits(ax=ax, fit=fitres, conf=myconf, outdir=outdir)
# plot_fits(ids,outdir=os.path.join(outdir,'fit_plots'))
axmod = dict(fit=fitres, conf=myconf)
for idx, modpar in enumerate(myconf.parnames):
par = modpar.lstrip('tiltm0.')
axmod[par] = ui.get_par('tiltm0.%s' % par).val
axmod["{}_parmax".format(par)] = myconf.parmaxes[idx]
axmod["{}_parmin".format(par)] = myconf.parmins[idx]
modelfit[ax] = axmod
mod_pick = open(model_file, 'w')
cPickle.dump( modelfit, mod_pick)
mod_pick.close()
obsfits.append(modelfit)
plot_fits([dict(obsid=obsid, data_id=data_id, ax=ax)],
posdir=obsdir,
outdir=outdir)
return obsfits
def run_fits(obsids,
ax,
user_pars=None,
fixed_pars=None,
guess_pars=None,
label='model',
per_obs_dir='per_obs_nfits',
outdir=None,
redo=False):
if len(obsids) == 0:
print "No obsids, nothing to fit"
return None
if user_pars is None:
user_pars = USER_PARS
if not os.path.exists(per_obs_dir):
os.makedirs(per_obs_dir)
obsfits = []
for obsid in obsids:
outdir = os.path.join(per_obs_dir, 'obs{:05d}'.format(obsid))
if not os.path.exists(outdir):
os.makedirs(outdir)
model_file = os.path.join(outdir, '{}.pkl'.format(label))
if os.path.exists(model_file) and not redo:
#logger.warn('Using previous fit found in %s' % model_file)
print model_file
mod_pick = open(model_file, 'r')
modelfit = cPickle.load(mod_pick)
mod_pick.close()
obsfits.append(modelfit)
continue
modelfit = {'label': obsid}
ui.clean()
data_id = 0
obsdir = "%s/obs%05d" % (DATADIR, obsid)
tf = open(os.path.join(obsdir, 'tilt.pkl'), 'r')
tilt = cPickle.load(tf)
tf.close()
pf = open(os.path.join(obsdir, 'pos.pkl'), 'r')
pos = cPickle.load(pf)
pf.close()
pos_data = pos[ax]
point_error = 5
pos_data_mean = np.mean(pos_data)
ui.set_method('simplex')
# Fit a line to get more reasonable errors
init_staterror = np.zeros(len(pos_data)) + point_error
ui.load_arrays(data_id, pos['time'] - pos['time'][0],
pos_data - np.mean(pos_data), init_staterror)
ui.polynom1d.ypoly
ui.set_model(data_id, 'ypoly')
ui.thaw(ypoly.c0, ypoly.c1)
ui.fit(data_id)
fit = ui.get_fit_results()
calc_staterror = init_staterror * np.sqrt(fit.rstat)
ui.set_staterror(data_id, calc_staterror)
# Confirm those errors
ui.fit(data_id)
fit = ui.get_fit_results()
if (abs(fit.rstat - 1) > .2):
raise ValueError('Reduced statistic not close to 1 for error calc')
# Load up data to do the real model fit
fit_times = pos['time']
tm_func = tilt_model(tilt, fit_times, user_pars=user_pars)
ui.get_data(data_id).name = str(obsid)
ui.load_user_model(tm_func, 'tiltm%d' % data_id)
ui.add_user_pars('tiltm%d' % data_id, user_pars)
ui.set_method('simplex')
ui.set_model(data_id, 'tiltm%d' % (data_id))
ui.set_par('tiltm%d.diam' % data_id, 0)
if fixed_pars is not None and ax in fixed_pars:
for par in fixed_pars[ax]:
ui.set_par('tiltm{}.{}'.format(0, par), fixed_pars[ax][par])
ui.freeze('tiltm{}.{}'.format(0, par))
if guess_pars is not None and ax in guess_pars:
for par in guess_pars[ax]:
ui.set_par('tiltm{}.{}'.format(0, par), guess_pars[ax][par])
ui.show_all()
# Fit the tilt model
ui.fit(data_id)
fitres = ui.get_fit_results()
ui.confidence(data_id)
myconf = ui.get_confidence_results()
# save_fits(ax=ax, fit=fitres, conf=myconf, outdir=outdir)
# plot_fits(ids,outdir=os.path.join(outdir,'fit_plots'))
axmod = dict(fit=fitres, conf=myconf)
for idx, modpar in enumerate(myconf.parnames):
par = modpar.lstrip('tiltm0.')
axmod[par] = ui.get_par('tiltm0.%s' % par).val
axmod["{}_parmax".format(par)] = myconf.parmaxes[idx]
axmod["{}_parmin".format(par)] = myconf.parmins[idx]
modelfit[ax] = axmod
mod_pick = open(model_file, 'w')
cPickle.dump(modelfit, mod_pick)
mod_pick.close()
obsfits.append(modelfit)
plot_fits([dict(obsid=obsid, data_id=data_id, ax=ax)],
posdir=obsdir,
outdir=outdir)
return obsfits
filename = "by%s_data_%s.txt" % (trend_type, ftype)
rates = asciitable.read(filename)
data_id = fail_types[ftype]
ui.set_method('simplex')
ui.load_arrays(data_id,
rates['time'],
rates['rate'])
ui.set_staterror(data_id,
rates['err'])
ftype_poly = ui.polynom1d(ftype)
ui.set_model(data_id, ftype_poly)
ui.thaw(ftype_poly.c0)
ui.thaw(ftype_poly.c1)
ui.notice(DateTime(trend_date_start).frac_year)
ui.fit(data_id)
ui.notice()
myfit = ui.get_fit_results()
axplot = ui.get_model_plot(data_id)
if myfit.succeeded:
b = ftype_poly.c1.val * DateTime(trend_date_start).frac_year + ftype_poly.c0.val
m = ftype_poly.c1.val
rep_file = open('%s_fitfile.json' % ftype, 'w')
rep_file.write(json.dumps(dict(time0=DateTime(trend_date_start).frac_year,
datestart=trend_date_start,
datestop=data_stop,
bin=trend_type,
m=m,