def test_emcee_custom_pool(self):
# tests use of a custom pool
if not HAS_EMCEE:
return True
global emcee_counter
emcee_counter = 0
class my_pool:
def map(self, f, arg):
global emcee_counter
emcee_counter += 1
return map(f, arg)
def cost_fun(params, **kwargs):
return rosen_der([params['a'], params['b']])
params = Parameters()
params.add('a', 1, min=-5, max=5, vary=True)
params.add('b', 1, min=-5, max=5, vary=True)
fitter = Minimizer(cost_fun, params)
fitter.emcee(workers=my_pool(), steps=1000, nwalkers=100)
assert emcee_counter > 500
def fit_single_line(self, x, y, zero_lev, err_continuum, fitting_parameters, bootstrap_iterations = 1000):
#Simple fit
if self.fit_dict['MC_iterations'] == 1:
fit_output = lmfit_minimize(residual_gauss, fitting_parameters, args=(x, y, zero_lev, err_continuum))
self.fit_dict['area_intg'] = simps(y, x) - simps(zero_lev, x)
self.fit_dict['area_intg_err'] = 0.0
#Bootstrap
else:
mini_posterior = Minimizer(lnprob_gaussCurve, fitting_parameters, fcn_args = ([x, y, zero_lev, err_continuum]))
fit_output = mini_posterior.emcee(steps=200, params = fitting_parameters)
#Bootstrap for the area of the lines
area_array = empty(bootstrap_iterations)
len_x_array = len(x)
for i in range(bootstrap_iterations):
y_new = y + np_normal_dist(0.0, err_continuum, len_x_array)
area_array[i] = simps(y_new, x) - simps(zero_lev, x)
self.fit_dict['area_intg'] = mean(area_array)
self.fit_dict['area_intg_err'] = std(area_array)
#Store the fitting parameters
output_params = fit_output.params
for key in self.fit_dict['parameters_list']:
self.fit_dict[key + '_norm'] = output_params[key].value
self.fit_dict[key + '_norm_er'] = output_params[key].stderr
return
def fitter(model, params, args, mcmc=False, pos=None, nwalkers=100,
steps=1000, burn=0.2, progress=True, get_ci=False,
nan_policy='raise', max_nfev=None, thin=10, is_weighted=True):
# Do fit
maxfev = [0 if max_nfev is None else max_nfev]
maxfev = int(maxfev[0])
func = Minimizer(model, params, fcn_args=args, nan_policy=nan_policy,
max_nfev=maxfev)
results = func.minimize()
if mcmc:
func = Minimizer(model, results.params, fcn_args=args)
mcmc_results = func.emcee(nwalkers=nwalkers, steps=steps,
burn=int(burn * steps), pos=pos,
is_weighted=is_weighted, progress=progress,
thin=thin)
results = mcmc_results
if get_ci:
if results.errorbars:
ci = conf_interval(func, results)
else:
ci = ''
return results, ci
else:
return results
def fit_isoturbHI_model(vels,
spec,
vcent,
delta_vcent=5 * u.km / u.s,
verbose=True,
plot_fit=True,
use_emcee=False,
emcee_kwargs={}):
vels = vels.copy().to(u.km / u.s)
vel_min = (vcent - delta_vcent).to(vels.unit).value
vel_max = (vcent + delta_vcent).to(vels.unit).value
# Create the parameter list.
pfit = Parameters()
pfit.add(name='log_Ts', value=2., min=1.2, max=3.2)
pfit.add(name='sigma_nt', value=15., min=2.0, max=31.6)
pfit.add(name='log_NH', value=21., min=20., max=23.5)
pfit.add(name='vcent',
value=vcent.to(vels.unit).value,
min=vel_min,
max=vel_max)
def residual(pars, x, data):
model = isoturbHI(x, pars['log_Ts'], pars['sigma_nt'], pars['log_NH'],
pars['vcent'])
return model - data
mini = Minimizer(residual, pfit, fcn_args=(vels.value, spec.value))
if use_emcee:
out = mini.emcee(**emcee_kwargs)
else:
out = mini.leastsq()
if verbose:
report_fit(out.params)
if plot_fit:
plt.plot(vels.value, spec.value, drawstyle='steps-mid')
pars = out.params
model = isoturbHI(vels.value, pars['log_Ts'].value,
pars['sigma_nt'].value, pars['log_NH'].value,
pars['vcent'].value)
plt.plot(vels.value, model)
return out
class MinimizerClassSuite:
"""
Benchmarks for the Minimizer class
"""
def setup(self):
self.x = np.linspace(1, 10, 250)
np.random.seed(0)
self.y = (3.0 * np.exp(-self.x / 2) -
5.0 * np.exp(-(self.x - 0.1) / 10.) +
0.1 * np.random.randn(len(self.x)))
self.p = Parameters()
self.p.add_many(('a1', 4., True, 0., 10.), ('a2', 4., True, -10., 10.),
('t1', 3., True, 0.01, 10.),
('t2', 3., True, 0.01, 20.))
self.p_emcee = deepcopy(self.p)
self.p_emcee.add('noise', 0.2, True, 0.001, 1.)
self.mini_de = Minimizer(Minimizer_Residual,
self.p,
fcn_args=(self.x, self.y),
kws={
'seed': 1,
'polish': False,
'maxiter': 100
})
self.mini_emcee = Minimizer(Minimizer_lnprob,
self.p_emcee,
fcn_args=(self.x, self.y))
def time_differential_evolution(self):
self.mini_de.minimize(method='differential_evolution')
def time_emcee(self):
self.mini_emcee.emcee(self.p_emcee, steps=100, seed=1)
def __call__(self):
#out = minimize(self.residual,
# self.params,
# scale_covar = False
# #method = 'cg'
# )
mini = Minimizer(self.residual, self.params)
out = mini.emcee(burn = 10000, steps = 60000, thin = 1, workers = 1, params = self.params)
self.H0 = 10**(out.params['a_nu'].value + 5 +
0.2 * (out.params['m04258'].value - 5*log10(out.params['mu_geometric'].value) - 25))
#print 5*log10(out.params['mu_geometric'].value) + 25
self.e_H0 = model.H0 * sqrt((out.params['a_nu'].stderr * log(10))**2
+ (log(10)/5 *out.params['m04258'].stderr )**2
+ (out.params['mu_geometric'].stderr/out.params['mu_geometric'].value)**2)
return out
class MinimizerClassSuite:
"""
Benchmarks for the Minimizer class
"""
def setup(self):
self.x = np.linspace(1, 10, 250)
np.random.seed(0)
self.y = (3.0 * np.exp(-self.x / 2)
- 5.0 * np.exp(-(self.x - 0.1) / 10.)
+ 0.1 * np.random.randn(len(self.x)))
self.p = Parameters()
self.p.add_many(('a1', 4., True, 0., 10.),
('a2', 4., True, -10., 10.),
('t1', 3., True, 0.01, 10.),
('t2', 3., True, 0.01, 20.))
self.p_emcee = deepcopy(self.p)
self.p_emcee.add('noise', 0.2, True, 0.001, 1.)
self.mini_de = Minimizer(Minimizer_Residual,
self.p,
fcn_args=(self.x, self.y),
kws={'seed': 1,
'polish': False,
'maxiter': 100})
self.mini_emcee = Minimizer(Minimizer_lnprob,
self.p_emcee,
fcn_args=(self.x, self.y))
def time_differential_evolution(self):
self.mini_de.minimize(method='differential_evolution')
def time_emcee(self):
self.mini_emcee.emcee(self.p_emcee, steps=100, seed=1)
def _sparam_iterator(self,params,L,freq_0,s11,s21,s12,s22):
"""
Perform the s-parameter fit using lmfit and emcee and produce the fit
report.
"""
# Fit data
minner = Minimizer(self._log_likelihood,\
params,fcn_args=(L,freq_0,s11,s21,s12),\
nan_policy='omit')
from timeit import default_timer as timer
start = timer()
result = minner.emcee(steps=self.nsteps,nwalkers=self.nwalkers,burn=self.nburn,thin=self.nthin,workers=self.nworkers)
end = timer()
m, s = divmod(end - start, 60)
h, m = divmod(m, 60)
time_str = "emcee took: %02d:%02d:%02d" % (h, m, s)
print(time_str)
report_fit(result)
return result, time_str
class CommonMinimizerTest(unittest.TestCase):
def setUp(self):
"""
test scale minimizers except newton-cg (needs jacobian) and
anneal (doesn't work out of the box).
"""
p_true = Parameters()
p_true.add('amp', value=14.0)
p_true.add('period', value=5.33)
p_true.add('shift', value=0.123)
p_true.add('decay', value=0.010)
self.p_true = p_true
n = 2500
xmin = 0.
xmax = 250.0
noise = np.random.normal(scale=0.7215, size=n)
self.x = np.linspace(xmin, xmax, n)
self.data = self.residual(p_true, self.x) + noise
fit_params = Parameters()
fit_params.add('amp', value=11.0, min=5, max=20)
fit_params.add('period', value=5., min=1., max=7)
fit_params.add('shift', value=.10, min=0.0, max=0.2)
fit_params.add('decay', value=6.e-3, min=0, max=0.1)
self.fit_params = fit_params
self.mini = Minimizer(self.residual, fit_params, [self.x, self.data])
def residual(self, pars, x, data=None):
amp = pars['amp'].value
per = pars['period'].value
shift = pars['shift'].value
decay = pars['decay'].value
if abs(shift) > pi / 2:
shift = shift - np.sign(shift) * pi
model = amp * np.sin(shift + x / per) * np.exp(-x * x * decay * decay)
if data is None:
return model
return model - data
def test_diffev_bounds_check(self):
# You need finite (min, max) for each parameter if you're using
# differential_evolution.
self.fit_params['decay'].min = None
self.minimizer = 'differential_evolution'
np.testing.assert_raises(ValueError, self.scalar_minimizer)
def test_scalar_minimizers(self):
# test all the scalar minimizers
for method in SCALAR_METHODS:
if method in ['newton', 'dogleg', 'trust-ncg']:
continue
self.minimizer = SCALAR_METHODS[method]
if method == 'Nelder-Mead':
sig = 0.2
else:
sig = 0.15
self.scalar_minimizer(sig=sig)
def scalar_minimizer(self, sig=0.15):
try:
from scipy.optimize import minimize as scipy_minimize
except ImportError:
raise SkipTest
print(self.minimizer)
out = self.mini.scalar_minimize(method=self.minimizer)
self.residual(out.params, self.x)
for name, par in out.params.items():
nout = "%s:%s" % (name, ' ' * (20 - len(name)))
print("%s: %s (%s) " % (nout, par.value, self.p_true[name].value))
for para, true_para in zip(out.params.values(), self.p_true.values()):
check_wo_stderr(para, true_para.value, sig=sig)
@decorators.slow
def test_emcee(self):
# test emcee
if not HAS_EMCEE:
return True
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200, burn=50, thin=10)
check_paras(out.params, self.p_true, sig=3)
@decorators.slow
def test_emcee_PT(self):
# test emcee with parallel tempering
if not HAS_EMCEE:
return True
np.random.seed(123456)
self.mini.userfcn = residual_for_multiprocessing
out = self.mini.emcee(ntemps=4,
nwalkers=50,
steps=200,
burn=100,
thin=10,
workers=2)
check_paras(out.params, self.p_true, sig=3)
@decorators.slow
def test_emcee_multiprocessing(self):
# test multiprocessing runs
if not HAS_EMCEE:
return True
np.random.seed(123456)
self.mini.userfcn = residual_for_multiprocessing
out = self.mini.emcee(steps=10, workers=4)
def test_emcee_bounds_length(self):
# the log-probability functions check if the parameters are
# inside the bounds. Check that the bounds and parameters
# are the right lengths for comparison. This can be done
# if nvarys != nparams
if not HAS_EMCEE:
return True
self.mini.params['amp'].vary = False
self.mini.params['period'].vary = False
self.mini.params['shift'].vary = False
out = self.mini.emcee(steps=10)
@decorators.slow
def test_emcee_partial_bounds(self):
# mcmc with partial bounds
if not HAS_EMCEE:
return True
np.random.seed(123456)
# test mcmc output vs lm, some parameters not bounded
self.fit_params['amp'].max = None
# self.fit_params['amp'].min = None
out = self.mini.emcee(nwalkers=100, steps=300, burn=100, thin=10)
check_paras(out.params, self.p_true, sig=3)
def test_emcee_init_with_chain(self):
# can you initialise with a previous chain
if not HAS_EMCEE:
return True
out = self.mini.emcee(nwalkers=100, steps=5)
# can initialise with a chain
out2 = self.mini.emcee(nwalkers=100, steps=1, pos=out.chain)
# can initialise with a correct subset of a chain
out3 = self.mini.emcee(nwalkers=100,
steps=1,
pos=out.chain[..., -1, :])
# but you can't initialise if the shape is wrong.
assert_raises(ValueError,
self.mini.emcee,
nwalkers=100,
steps=1,
pos=out.chain[..., -1, :-1])
def test_emcee_reuse_sampler(self):
if not HAS_EMCEE:
return True
self.mini.emcee(nwalkers=100, steps=5)
# if you've run the sampler the Minimizer object should have a _lastpos
# attribute
assert_(hasattr(self.mini, '_lastpos'))
# now try and re-use sampler
out2 = self.mini.emcee(steps=10, reuse_sampler=True)
assert_(out2.chain.shape[1] == 15)
# you shouldn't be able to reuse the sampler if nvarys has changed.
self.mini.params['amp'].vary = False
assert_raises(ValueError, self.mini.emcee, reuse_sampler=True)
def test_emcee_lnpost(self):
# check ln likelihood is calculated correctly. It should be
# -0.5 * chi**2.
result = self.mini.minimize()
# obtain the numeric values
# note - in this example all the parameters are varied
fvars = np.array([par.value for par in result.params.values()])
# calculate the cost function with scaled values (parameters all have
# lower and upper bounds.
scaled_fvars = []
for par, fvar in zip(result.params.values(), fvars):
par.value = fvar
scaled_fvars.append(par.setup_bounds())
val = self.mini.penalty(np.array(scaled_fvars))
# calculate the log-likelihood value
bounds = np.array([(par.min, par.max)
for par in result.params.values()])
val2 = _lnpost(fvars,
self.residual,
result.params,
result.var_names,
bounds,
userargs=(self.x, self.data))
assert_almost_equal(-0.5 * val, val2)
def test_emcee_output(self):
# test mcmc output
if not HAS_EMCEE:
return True
try:
from pandas import DataFrame
except ImportError:
return True
out = self.mini.emcee(nwalkers=10, steps=20, burn=5, thin=2)
assert_(isinstance(out, MinimizerResult))
assert_(isinstance(out.flatchain, DataFrame))
# check that we can access the chains via parameter name
assert_(out.flatchain['amp'].shape[0] == 80)
assert_(out.errorbars is True)
assert_(np.isfinite(out.params['amp'].correl['period']))
# the lnprob array should be the same as the chain size
assert_(np.size(out.chain) // 4 == np.size(out.lnprob))
@decorators.slow
def test_emcee_float(self):
# test that it works if the residuals returns a float, not a vector
if not HAS_EMCEE:
return True
def resid(pars, x, data=None):
return -0.5 * np.sum(self.residual(pars, x, data=data)**2)
# just return chi2
def resid2(pars, x, data=None):
return np.sum(self.residual(pars, x, data=data)**2)
self.mini.userfcn = resid
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200, burn=50, thin=10)
check_paras(out.params, self.p_true, sig=3)
self.mini.userfcn = resid2
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100,
steps=200,
burn=50,
thin=10,
float_behavior='chi2')
check_paras(out.params, self.p_true, sig=3)
params.add('slope', value=-1.0, min=-30.0, max=10.0)
params.add('intercept', value=0.6, min=-20.0, max=20.0)
x = data['met'][(np.isfinite(alpha_co)) & (data.mstq == 1)]
y = alpha_co[(np.isfinite(alpha_co)) & (data.mstq == 1)]
yerr = alpha_co_err[(np.isfinite(alpha_co)) & (data.mstq == 1)]
xerr = 0.1
c = data['zsp'][(np.isfinite(alpha_co)) & (data.mstq == 1)]
def residual(pars):
p = pars.valuesdict()
model = p['slope']*x + p['intercept']
return (model - y) / np.sqrt(yerr**2 + xerr**2)
mini = Minimizer(residual, params, nan_policy='omit')
res = mini.emcee(burn=100, steps=1000, thin=1, nwalkers=100, is_weighted=True)
corner.corner(res.flatchain, labels=res.var_names, truths=list(res.params.valuesdict().values()));
plt.savefig('Corner.pdf')
plt.close()
report_fit(res.params)
X = np.linspace(-0.55,0.55,100)
model = res.params.valuesdict()['slope'] * X + res.params.valuesdict()['intercept']
scatter = np.outer(res.flatchain.slope,X) + np.tile(res.flatchain.intercept,(len(X),1)).T
up = np.mean(scatter,axis=0) + np.std(scatter,axis=0)
down = np.mean(scatter,axis=0) - np.std(scatter,axis=0)
scatter_kwargs = {'zorder':100}
resid += np.log(2 * np.pi * s**2)
return -0.5 * np.sum(resid) + logprior
mini = Minimizer(lnprob, mle0.params)
start = time()
#import emcee
#res = emcee.sampler(lnlikelihood = lnprob, lnprior=logprior_func)
res = mini.emcee(params=mle0.params,
steps=100,
nwalkers=100,
burn=100,
thin=10,
ntemps=1,
pos=None,
reuse_sampler=False,
workers=1,
float_behavior='posterior',
is_weighted=True,
seed=None)
print("MCMC operation took {} seconds".format(time() - start))
emcee_save_name = save_header + '_emcee_sample_results.joblib.save'
print("Saving EMCEE results to {}".format(emcee_save_name))
joblib.dump(res, emcee_save_name)
res_var_names = np.array(res.var_names)
res_flatchain = np.array(res.flatchain)
res_df = DataFrame(res_flatchain, columns=res_var_names)
# res_df = res_df.drop(['u2','slope'], axis=1)
class CommonMinimizerTest(unittest.TestCase):
def setUp(self):
"""
test scale minimizers except newton-cg (needs jacobian) and
anneal (doesn't work out of the box).
"""
p_true = Parameters()
p_true.add('amp', value=14.0)
p_true.add('period', value=5.33)
p_true.add('shift', value=0.123)
p_true.add('decay', value=0.010)
self.p_true = p_true
n = 2500
xmin = 0.
xmax = 250.0
noise = np.random.normal(scale=0.7215, size=n)
self.x = np.linspace(xmin, xmax, n)
self.data = self.residual(p_true, self.x) + noise
fit_params = Parameters()
fit_params.add('amp', value=11.0, min=5, max=20)
fit_params.add('period', value=5., min=1., max=7)
fit_params.add('shift', value=.10, min=0.0, max=0.2)
fit_params.add('decay', value=6.e-3, min=0, max=0.1)
self.fit_params = fit_params
self.mini = Minimizer(self.residual, fit_params, [self.x, self.data])
def residual(self, pars, x, data=None):
amp = pars['amp'].value
per = pars['period'].value
shift = pars['shift'].value
decay = pars['decay'].value
if abs(shift) > pi/2:
shift = shift - np.sign(shift) * pi
model = amp*np.sin(shift + x/per) * np.exp(-x*x*decay*decay)
if data is None:
return model
return model - data
def test_diffev_bounds_check(self):
# You need finite (min, max) for each parameter if you're using
# differential_evolution.
self.fit_params['decay'].min = None
self.minimizer = 'differential_evolution'
np.testing.assert_raises(ValueError, self.scalar_minimizer)
def test_scalar_minimizers(self):
# test all the scalar minimizers
for method in SCALAR_METHODS:
if method in ['newton', 'dogleg', 'trust-ncg']:
continue
self.minimizer = SCALAR_METHODS[method]
if method == 'Nelder-Mead':
sig = 0.2
else:
sig = 0.15
self.scalar_minimizer(sig=sig)
def scalar_minimizer(self, sig=0.15):
try:
from scipy.optimize import minimize as scipy_minimize
except ImportError:
raise SkipTest
print(self.minimizer)
out = self.mini.scalar_minimize(method=self.minimizer)
self.residual(out.params, self.x)
for name, par in out.params.items():
nout = "%s:%s" % (name, ' '*(20-len(name)))
print("%s: %s (%s) " % (nout, par.value, self.p_true[name].value))
for para, true_para in zip(out.params.values(),
self.p_true.values()):
check_wo_stderr(para, true_para.value, sig=sig)
@decorators.slow
def test_emcee(self):
# test emcee
if not HAS_EMCEE:
return True
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200,
burn=50, thin=10)
check_paras(out.params, self.p_true, sig=3)
@decorators.slow
def test_emcee_PT(self):
# test emcee with parallel tempering
if not HAS_EMCEE:
return True
np.random.seed(123456)
self.mini.userfcn = residual_for_multiprocessing
out = self.mini.emcee(ntemps=4, nwalkers=50, steps=200,
burn=100, thin=10, workers=2)
check_paras(out.params, self.p_true, sig=3)
@decorators.slow
def test_emcee_multiprocessing(self):
# test multiprocessing runs
if not HAS_EMCEE:
return True
np.random.seed(123456)
self.mini.userfcn = residual_for_multiprocessing
out = self.mini.emcee(steps=10, workers=4)
def test_emcee_bounds_length(self):
# the log-probability functions check if the parameters are
# inside the bounds. Check that the bounds and parameters
# are the right lengths for comparison. This can be done
# if nvarys != nparams
if not HAS_EMCEE:
return True
self.mini.params['amp'].vary=False
self.mini.params['period'].vary=False
self.mini.params['shift'].vary=False
out = self.mini.emcee(steps=10)
@decorators.slow
def test_emcee_partial_bounds(self):
# mcmc with partial bounds
if not HAS_EMCEE:
return True
np.random.seed(123456)
# test mcmc output vs lm, some parameters not bounded
self.fit_params['amp'].max = None
# self.fit_params['amp'].min = None
out = self.mini.emcee(nwalkers=100, steps=300,
burn=100, thin=10)
check_paras(out.params, self.p_true, sig=3)
def test_emcee_init_with_chain(self):
# can you initialise with a previous chain
if not HAS_EMCEE:
return True
out = self.mini.emcee(nwalkers=100, steps=5)
# can initialise with a chain
out2 = self.mini.emcee(nwalkers=100, steps=1, pos=out.chain)
# can initialise with a correct subset of a chain
out3 = self.mini.emcee(nwalkers=100,
steps=1,
pos=out.chain[..., -1, :])
# but you can't initialise if the shape is wrong.
assert_raises(ValueError,
self.mini.emcee,
nwalkers=100,
steps=1,
pos=out.chain[..., -1, :-1])
def test_emcee_reuse_sampler(self):
if not HAS_EMCEE:
return True
self.mini.emcee(nwalkers=100, steps=5)
# if you've run the sampler the Minimizer object should have a _lastpos
# attribute
assert_(hasattr(self.mini, '_lastpos'))
# now try and re-use sampler
out2 = self.mini.emcee(steps=10, reuse_sampler=True)
assert_(out2.chain.shape[1] == 15)
# you shouldn't be able to reuse the sampler if nvarys has changed.
self.mini.params['amp'].vary = False
assert_raises(ValueError, self.mini.emcee, reuse_sampler=True)
def test_emcee_lnpost(self):
# check ln likelihood is calculated correctly. It should be
# -0.5 * chi**2.
result = self.mini.minimize()
# obtain the numeric values
# note - in this example all the parameters are varied
fvars = np.array([par.value for par in result.params.values()])
# calculate the cost function with scaled values (parameters all have
# lower and upper bounds.
scaled_fvars = []
for par, fvar in zip(result.params.values(), fvars):
par.value = fvar
scaled_fvars.append(par.setup_bounds())
val = self.mini.penalty(np.array(scaled_fvars))
# calculate the log-likelihood value
bounds = np.array([(par.min, par.max)
for par in result.params.values()])
val2 = _lnpost(fvars,
self.residual,
result.params,
result.var_names,
bounds,
userargs=(self.x, self.data))
assert_almost_equal(-0.5 * val, val2)
def test_emcee_output(self):
# test mcmc output
if not HAS_EMCEE:
return True
try:
from pandas import DataFrame
except ImportError:
return True
out = self.mini.emcee(nwalkers=10, steps=20, burn=5, thin=2)
assert_(isinstance(out, MinimizerResult))
assert_(isinstance(out.flatchain, DataFrame))
# check that we can access the chains via parameter name
assert_(out.flatchain['amp'].shape[0] == 80)
assert_(out.errorbars is True)
assert_(np.isfinite(out.params['amp'].correl['period']))
# the lnprob array should be the same as the chain size
assert_(np.size(out.chain)//4 == np.size(out.lnprob))
@decorators.slow
def test_emcee_float(self):
# test that it works if the residuals returns a float, not a vector
if not HAS_EMCEE:
return True
def resid(pars, x, data=None):
return -0.5 * np.sum(self.residual(pars, x, data=data)**2)
# just return chi2
def resid2(pars, x, data=None):
return np.sum(self.residual(pars, x, data=data)**2)
self.mini.userfcn = resid
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200,
burn=50, thin=10)
check_paras(out.params, self.p_true, sig=3)
self.mini.userfcn = resid2
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200,
burn=50, thin=10, float_behavior='chi2')
check_paras(out.params, self.p_true, sig=3)
(n_walkers, len(starting_values)))
reuse_sampler = False
n_workers = cpu_count() #-1
is_weighted = True
seed = None # 42
start = time()
print('MCMC routine in progress...')
mcmc_fit = mini.emcee(params=mle0.params,
steps=n_steps,
nwalkers=n_walkers,
burn=n_burn,
thin=n_thin,
ntemps=n_temps,
pos=pos,
reuse_sampler=reuse_sampler,
workers=n_workers,
is_weighted=is_weighted,
seed=seed)
nrg_ratio_mcmc, temp_night_mcmc, delta_T_mcmc = res.x
nrg_ratio_mcmc, temp_night_mcmc, delta_T_mcmc, err_mod_mcmc = np.median(
mcmc_fit.flatchain, axis=0)
mcmc_model = generate_spiderman_model(times, planet_info, nrg_ratio,
temp_night, delta_T, T_star,
spider_params)
plot_now = True
def main(self,
ID0,
PA0,
zgal,
flag_m,
zprev,
Cz0,
Cz1,
mcmcplot=True,
fzvis=1,
specplot=1,
fneld=0,
ntemp=5,
sigz=1.0,
ezmin=0.01,
f_move=False,
f_disp=False
): # flag_m related to redshift error in redshift check func.
#
# sigz (float): confidence interval for redshift fit.
# ezmin (float): minimum error in redshift.
#
print('########################')
print('### Fitting Function ###')
print('########################')
start = timeit.default_timer()
inputs = self.inputs
DIR_TMP = inputs['DIR_TEMP']
if os.path.exists(DIR_TMP) == False:
os.mkdir(DIR_TMP)
# For error parameter
ferr = 0
#
# Age
#
age = inputs['AGE']
age = [float(x.strip()) for x in age.split(',')]
nage = np.arange(0, len(age), 1)
#
# Metallicity
#
Zmax, Zmin = float(inputs['ZMAX']), float(inputs['ZMIN'])
delZ = float(inputs['DELZ'])
Zall = np.arange(Zmin, Zmax, delZ) # in logZsun
# For minimizer.
delZtmp = delZ
#delZtmp = 0.4 # to increase speed.
# For z prior.
delzz = 0.001
zlimu = 6.
snlim = 1
zliml = zgal - 0.5
agemax = cd.age(zgal, use_flat=True, **cosmo) / cc.Gyr_s
# N of param:
try:
ndim = int(inputs['NDIM'])
print('No of params are : %d' % (ndim))
except:
if int(inputs['ZEVOL']) == 1:
ndim = int(len(nage) * 2 + 1)
print('Metallicity evolution is on.')
if int(inputs['ZMC']) == 1:
ndim += 1
print('No of params are : %d' % (ndim))
else:
ndim = int(len(nage) + 1 + 1)
print('Metallicity evolution is off.')
if int(inputs['ZMC']) == 1:
ndim += 1
print('No of params are : %d' % (ndim))
pass
#
# Line
#
LW0 = inputs['LINE']
LW0 = [float(x.strip()) for x in LW0.split(',')]
#
# Params for MCMC
#
nmc = int(inputs['NMC'])
nwalk = int(inputs['NWALK'])
nmc_cz = int(inputs['NMCZ'])
nwalk_cz = int(inputs['NWALKZ'])
f_Zevol = int(inputs['ZEVOL'])
#f_zvis = int(inputs['ZVIS'])
try:
fzmc = int(inputs['ZMC'])
except:
fzmc = 0
#
# If FIR data;
#
try:
DT0 = float(inputs['TDUST_LOW'])
DT1 = float(inputs['TDUST_HIG'])
dDT = float(inputs['TDUST_DEL'])
Temp = np.arange(DT0, DT1, dDT)
f_dust = True
print('FIR fit is on.')
except:
f_dust = False
pass
#
# Tau for MCMC parameter; not as fitting parameters.
#
tau0 = inputs['TAU0']
tau0 = [float(x.strip()) for x in tau0.split(',')]
#
# Dust model specification;
#
try:
dust_model = int(inputs['DUST_MODEL'])
except:
dust_model = 0
from .function_class import Func
from .basic_func import Basic
fnc = Func(Zall, nage, dust_model=dust_model,
DIR_TMP=DIR_TMP) # Set up the number of Age/ZZ
bfnc = Basic(Zall)
# Open ascii file and stock to array.
#lib = open_spec(ID0, PA0)
lib = fnc.open_spec_fits(ID0, PA0, fall=0, tau0=tau0)
lib_all = fnc.open_spec_fits(ID0, PA0, fall=1, tau0=tau0)
if f_dust:
lib_dust = fnc.open_spec_dust_fits(ID0,
PA0,
Temp,
fall=0,
tau0=tau0)
lib_dust_all = fnc.open_spec_dust_fits(ID0,
PA0,
Temp,
fall=1,
tau0=tau0)
#################
# Observed Data
#################
##############
# Spectrum
##############
dat = np.loadtxt(DIR_TMP + 'spec_obs_' + ID0 + '_PA' + PA0 + '.cat',
comments='#')
NR = dat[:, 0]
x = dat[:, 1]
fy00 = dat[:, 2]
ey00 = dat[:, 3]
con0 = (NR < 1000)
fy0 = fy00[con0] * Cz0
ey0 = ey00[con0] * Cz0
con1 = (NR >= 1000) & (NR < 10000)
fy1 = fy00[con1] * Cz1
ey1 = ey00[con1] * Cz1
# BB data in spec_obs are not in use.
#con2 = (NR>=10000) # BB
#fy2 = fy00[con2]
#ey2 = ey00[con2]
##############
# Broadband
##############
dat = np.loadtxt(DIR_TMP + 'bb_obs_' + ID0 + '_PA' + PA0 + '.cat',
comments='#')
NRbb = dat[:, 0]
xbb = dat[:, 1]
fybb = dat[:, 2]
eybb = dat[:, 3]
exbb = dat[:, 4]
fy2 = fybb
ey2 = eybb
fy01 = np.append(fy0, fy1)
fy = np.append(fy01, fy2)
ey01 = np.append(ey0, ey1)
ey = np.append(ey01, ey2)
wht = 1. / np.square(ey)
wht2 = check_line_man(fy, x, wht, fy, zprev, LW0)
sn = fy / ey
#####################
# Function fo MCMC
#####################
def residual(
pars,
fy,
wht2,
f_fir,
out=False): # x, y, wht are taken from out of the definition.
#
# Returns: residual of model and data.
# out: model as second output. For lnprob func.
# f_fir: syntax. If dust component is on or off.
vals = pars.valuesdict()
model, x1 = fnc.tmp04(ID0, PA0, vals, zprev, lib, tau0=tau0)
if f_fir:
model_dust, x1_dust = fnc.tmp04_dust(ID0,
PA0,
vals,
zprev,
lib_dust,
tau0=tau0)
n_optir = len(model)
# Add dust flux to opt/IR grid.
model[:] += model_dust[:n_optir]
#print(model_dust)
# then append only FIR flux grid.
model = np.append(model, model_dust[n_optir:])
x1 = np.append(x1, x1_dust[n_optir:])
#plt.plot(x1,model,'r.')
#plt.show()
if ferr == 1:
f = vals['f']
else:
f = 0 # temporary... (if f is param, then take from vals dictionary.)
con_res = (model > 0) & (wht2 > 0) #& (fy>0)
sig = np.sqrt(1. / wht2 + f**2 * model**2)
'''
contmp = x1>1e6 & (wht2>0)
try:
print(x1[contmp],model[contmp],fy[contmp],np.log10(vals['MDUST']))
except:
pass
'''
if not out:
if fy is None:
print('Data is none')
return model[con_res]
else:
return (model - fy)[con_res] / sig[
con_res] # i.e. residual/sigma. Because is_weighted = True.
if out:
if fy is None:
print('Data is none')
return model[con_res], model
else:
return (model - fy)[con_res] / sig[
con_res], model # i.e. residual/sigma. Because is_weighted = True.
def lnprob(pars, fy, wht2, f_fir):
#
# Returns: posterior.
#
vals = pars.valuesdict()
if ferr == 1:
f = vals['f']
else:
f = 0 # temporary... (if f is param, then take from vals dictionary.)
resid, model = residual(pars, fy, wht2, f_fir, out=True)
con_res = (model > 0) & (wht2 > 0)
sig = np.sqrt(1. / wht2 + f**2 * model**2)
lnlike = -0.5 * np.sum(resid**2 +
np.log(2 * 3.14 * sig[con_res]**2))
#print(np.log(2 * 3.14 * 1) * len(sig[con_res]), np.sum(np.log(2 * 3.14 * sig[con_res]**2)))
#Av = vals['Av']
#if Av<0:
# return -np.inf
#else:
# respr = 0 #np.log(1)
respr = 0 # Flat prior...
return lnlike + respr
###############################
# Add parameters
###############################
fit_params = Parameters()
for aa in range(len(age)):
if age[aa] == 99 or age[aa] > agemax:
fit_params.add('A' + str(aa), value=0, min=0, max=1e-10)
else:
fit_params.add('A' + str(aa), value=1, min=0, max=1e3)
#####################
# Dust attenuation
#####################
try:
Avmin = float(inputs['AVMIN'])
Avmax = float(inputs['AVMAX'])
fit_params.add('Av', value=Avmin, min=Avmin, max=Avmax)
except:
fit_params.add('Av', value=0.2, min=0, max=4.0)
#####################
# Metallicity
#####################
if int(inputs['ZEVOL']) == 1:
for aa in range(len(age)):
if age[aa] == 99 or age[aa] > agemax:
fit_params.add('Z' + str(aa), value=0, min=0, max=1e-10)
else:
fit_params.add('Z' + str(aa),
value=0,
min=np.min(Zall),
max=np.max(Zall))
elif inputs['ZFIX']:
#print('Z is fixed')
ZFIX = float(inputs['ZFIX'])
aa = 0
fit_params.add('Z' + str(aa), value=0, min=ZFIX, max=ZFIX + 0.01)
else:
aa = 0
fit_params.add('Z' + str(aa),
value=0,
min=np.min(Zall),
max=np.max(Zall))
####################################
# Initial Metallicity Determination
####################################
chidef = 1e5
Zbest = 0
fwz = open('Z_' + ID0 + '_PA' + PA0 + '.cat', 'w')
fwz.write('# ID Zini chi/nu AA Av Zbest\n')
fwz.write('# FNELD = %d\n' % fneld)
nZtmp = int((Zmax - Zmin) / delZtmp)
ZZtmp = np.arange(Zmin, Zmax, delZtmp)
# How to get initial parameters?
# Nelder;
if fneld == 1:
fit_name = 'nelder'
for zz in range(len(ZZtmp)):
ZZ = ZZtmp[zz]
if int(inputs['ZEVOL']) == 1:
for aa in range(len(age)):
fit_params['Z' + str(aa)].value = ZZ
else:
aa = 0
fit_params['Z' + str(aa)].value = ZZ
out_tmp = minimize(
residual,
fit_params,
args=(fy, wht2, False),
method=fit_name) # nelder is the most efficient.
keys = fit_report(out_tmp).split('\n')
csq = 99999
rcsq = 99999
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc = [csq, rcsq] # Chi2, Reduced-chi2
fwz.write('%s %.2f %.5f' % (ID0, ZZ, fitc[1]))
AA_tmp = np.zeros(len(age), dtype='float32')
ZZ_tmp = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
AA_tmp[aa] = out_tmp.params['A' + str(aa)].value
fwz.write(' %.5f' % (AA_tmp[aa]))
Av_tmp = out_tmp.params['Av'].value
fwz.write(' %.5f' % (Av_tmp))
if int(inputs['ZEVOL']) == 1:
for aa in range(len(age)):
ZZ_tmp[aa] = out_tmp.params['Z' + str(aa)].value
fwz.write(' %.5f' % (ZZ_tmp[aa]))
else:
aa = 0
ZZ_tmp[aa] = out_tmp.params['Z' + str(aa)].value
fwz.write(' %.5f' % (ZZ_tmp[aa]))
fwz.write('\n')
if fitc[1] < chidef:
chidef = fitc[1]
out = out_tmp
# Or
# Powell;
else:
fit_name = 'powell'
for zz in range(0, nZtmp, 2):
ZZ = zz * delZtmp + np.min(Zall)
if int(inputs['ZEVOL']) == 1:
for aa in range(len(age)):
fit_params['Z' + str(aa)].value = ZZ
else:
aa = 0
fit_params['Z' + str(aa)].value = ZZ
out_tmp = minimize(
residual,
fit_params,
args=(fy, wht2, False),
method=fit_name) # powel is the more accurate.
keys = fit_report(out_tmp).split('\n')
csq = 99999
rcsq = 99999
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc = [csq, rcsq] # Chi2, Reduced-chi2
fwz.write('%s %.2f %.5f' % (ID0, ZZ, fitc[1]))
AA_tmp = np.zeros(len(age), dtype='float32')
ZZ_tmp = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
AA_tmp[aa] = out_tmp.params['A' + str(aa)].value
fwz.write(' %.5f' % (AA_tmp[aa]))
Av_tmp = out_tmp.params['Av'].value
fwz.write(' %.5f' % (Av_tmp))
if int(inputs['ZEVOL']) == 1:
for aa in range(len(age)):
ZZ_tmp[aa] = out_tmp.params['Z' + str(aa)].value
fwz.write(' %.5f' % (ZZ_tmp[aa]))
else:
aa = 0
fwz.write(' %.5f' % (ZZ_tmp[aa]))
fwz.write('\n')
if fitc[1] < chidef:
chidef = fitc[1]
out = out_tmp
#
# Best fit
#
keys = fit_report(out).split('\n')
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc = [csq, rcsq] # Chi2, Reduced-chi2
#fitc = fit_report_chi(out) # Chi2, Reduced-chi2
ZZ = Zbest # This is really important/does affect lnprob/residual.
print('\n\n')
print('#####################################')
print('Zbest, chi are;', Zbest, chidef)
print('Params are;', fit_report(out))
print('#####################################')
print('\n\n')
fwz.close()
Av_tmp = out.params['Av'].value
AA_tmp = np.zeros(len(age), dtype='float32')
ZZ_tmp = np.zeros(len(age), dtype='float32')
fm_tmp, xm_tmp = fnc.tmp04_val(ID0, PA0, out, zprev, lib, tau0=tau0)
########################
# Check redshift
########################
zrecom = zprev
# Observed data.
con_cz = (NR < 10000) #& (sn>snlim)
fy_cz = fy[con_cz]
ey_cz = ey[con_cz]
x_cz = x[con_cz] # Observed range
NR_cz = NR[con_cz]
xm_s = xm_tmp / (1 + zprev) * (1 + zrecom)
fm_s = np.interp(x_cz, xm_s, fm_tmp)
if fzvis == 1:
plt.plot(x_cz / (1 + zprev) * (1. + zrecom),
fm_s,
'gray',
linestyle='--',
linewidth=0.5,
label='Default ($z=%.5f$)' %
(zprev)) # Model based on input z.
plt.plot(x_cz,
fy_cz,
'b',
linestyle='-',
linewidth=0.5,
label='Obs.') # Observation
plt.errorbar(x_cz,
fy_cz,
yerr=ey_cz,
color='b',
capsize=0,
linewidth=0.5) # Observation
if flag_m == 0:
dez = 0.5
else:
dez = 0.2
#
# For Eazy
#
'''
dprob = np.loadtxt(eaz_path + 'photz_' + str(int(ID0)) + '.pz', comments='#')
zprob = dprob[:,0]
cprob = dprob[:,1]
zz_prob = np.arange(0,13,delzz)
cprob_s = np.interp(zz_prob, zprob, cprob)
prior_s = 1/cprob_s
prior_s /= np.sum(prior_s)
'''
zz_prob = np.arange(0, 13, delzz)
prior_s = zz_prob * 0 + 1.
prior_s /= np.sum(prior_s)
try:
print('############################')
print('Start MCMC for redshift fit')
print('############################')
res_cz, fitc_cz = check_redshift(fy_cz, ey_cz, x_cz, fm_tmp,
xm_tmp / (1 + zprev), zprev, dez,
prior_s, NR_cz, zliml, zlimu,
delzz, nmc_cz, nwalk_cz)
z_cz = np.percentile(res_cz.flatchain['z'], [16, 50, 84])
scl_cz0 = np.percentile(res_cz.flatchain['Cz0'], [16, 50, 84])
scl_cz1 = np.percentile(res_cz.flatchain['Cz1'], [16, 50, 84])
zrecom = z_cz[1]
Czrec0 = scl_cz0[1]
Czrec1 = scl_cz1[1]
# Switch to peak redshift:
from scipy import stats
from scipy.stats import norm
# find minimum and maximum of xticks, so we know
# where we should compute theoretical distribution
ser = res_cz.flatchain['z']
xmin, xmax = zprev - 0.2, zprev + 0.2
lnspc = np.linspace(xmin, xmax, len(ser))
print('\n\n')
print(
'Recommended redshift, Cz0 and Cz1, %.5f %.5f %.5f, with chi2/nu=%.3f'
% (zrecom, Cz0 * Czrec0, Cz1 * Czrec1, fitc_cz[1]))
print('\n\n')
except:
print('z fit failed. No spectral data set?')
try:
ezl = float(inputs['EZL'])
ezu = float(inputs['EZU'])
print('Redshift error is taken from input file.')
if ezl < ezmin:
ezl = ezmin #0.03
if ezu < ezmin:
ezu = ezmin #0.03
except:
ezl = ezmin
ezu = ezmin
print('Redshift error is assumed to %.1f.' % (ezl))
z_cz = [zprev - ezl, zprev, zprev + ezu]
zrecom = z_cz[1]
scl_cz0 = [1., 1., 1.]
scl_cz1 = [1., 1., 1.]
Czrec0 = scl_cz0[1]
Czrec1 = scl_cz1[1]
'''
try:
# lets try the normal distribution first
m, s = stats.norm.fit(ser) # get mean and standard deviation
pdf_g = stats.norm.pdf(lnspc, m, s) # now get theoretical values in our interval
z_cz[:] = [m-s, m, m+s]
zrecom = z_cz[1]
f_fitgauss = 1
except:
print('Guassian fitting to z distribution failed.')
f_fitgauss=0
'''
f_fitgauss = 0
xm_s = xm_tmp / (1 + zprev) * (1 + zrecom)
fm_s = np.interp(x_cz, xm_s, fm_tmp)
whtl = 1 / np.square(ey_cz)
try:
wht3, ypoly = check_line_cz_man(fy_cz, x_cz, whtl, fm_s, zrecom,
LW0)
except:
wht3, ypoly = whtl, fy_cz
con_line = (wht3 == 0)
if fzvis == 1:
plt.plot(x_cz,
fm_s,
'r',
linestyle='-',
linewidth=0.5,
label='Updated model ($z=%.5f$)' %
(zrecom)) # Model based on recomended z.
plt.plot(x_cz[con_line],
fm_s[con_line],
color='orange',
marker='o',
linestyle='',
linewidth=3.)
# Plot lines for reference
for ll in range(len(LW)):
try:
conpoly = (x_cz /
(1. + zrecom) > 3000) & (x_cz /
(1. + zrecom) < 8000)
yline = np.max(ypoly[conpoly])
yy = np.arange(yline / 1.02, yline * 1.1)
xxpre = yy * 0 + LW[ll] * (1. + zprev)
xx = yy * 0 + LW[ll] * (1. + zrecom)
plt.plot(xxpre,
yy / 1.02,
linewidth=0.5,
linestyle='--',
color='gray')
plt.text(LW[ll] * (1. + zprev),
yline / 1.05,
'%s' % (LN[ll]),
fontsize=8,
color='gray')
plt.plot(xx,
yy,
linewidth=0.5,
linestyle='-',
color='orangered')
plt.text(LW[ll] * (1. + zrecom),
yline,
'%s' % (LN[ll]),
fontsize=8,
color='orangered')
except:
pass
plt.plot(xbb,
fybb,
'.r',
linestyle='',
linewidth=0,
zorder=4,
label='Obs.(BB)')
plt.plot(xm_tmp,
fm_tmp,
color='gray',
marker='.',
ms=0.5,
linestyle='',
linewidth=0.5,
zorder=4,
label='Model')
try:
xmin, xmax = np.min(x_cz) / 1.1, np.max(x_cz) * 1.1
except:
xmin, xmax = 2000, 10000
plt.xlim(xmin, xmax)
try:
plt.ylim(0, yline * 1.1)
except:
pass
plt.xlabel('Wavelength ($\mathrm{\AA}$)')
plt.ylabel('$F_\\nu$ (arb.)')
plt.legend(loc=0)
zzsigma = ((z_cz[2] - z_cz[0]) / 2.) / zprev
zsigma = np.abs(zprev - zrecom) / (zprev)
C0sigma = np.abs(Czrec0 - Cz0) / Cz0
eC0sigma = ((scl_cz0[2] - scl_cz0[0]) / 2.) / Cz0
C1sigma = np.abs(Czrec1 - Cz1) / Cz1
eC1sigma = ((scl_cz1[2] - scl_cz1[0]) / 2.) / Cz1
print('Input redshift is %.3f per cent agreement.' %
((1. - zsigma) * 100))
print('Error is %.3f per cent.' % (zzsigma * 100))
print('Input Cz0 is %.3f per cent agreement.' %
((1. - C0sigma) * 100))
print('Error is %.3f per cent.' % (eC0sigma * 100))
print('Input Cz1 is %.3f per cent agreement.' %
((1. - C1sigma) * 100))
print('Error is %.3f per cent.' % (eC1sigma * 100))
plt.show()
#
# Ask interactively;
#
flag_z = raw_input(
'Do you want to continue with original redshift, Cz0 and Cz1, %.5f %.5f %.5f? ([y]/n/m) '
% (zprev, Cz0, Cz1))
else:
flag_z = 'y'
#################################################
# Gor for mcmc phase
#################################################
if flag_z == 'y' or flag_z == '':
zrecom = zprev
Czrec0 = Cz0
Czrec1 = Cz1
#######################
# Added
#######################
if fzmc == 1:
out_keep = out
#sigz = 1.0 #3.0
fit_params.add('zmc',
value=zrecom,
min=zrecom - (z_cz[1] - z_cz[0]) * sigz,
max=zrecom + (z_cz[2] - z_cz[1]) * sigz)
#print(zrecom,zrecom-(z_cz[1]-z_cz[0])*sigz,zrecom+(z_cz[2]-z_cz[1])*sigz)
#####################
# Error parameter
#####################
try:
ferr = int(inputs['F_ERR'])
if ferr == 1:
fit_params.add('f', value=1e-2, min=0, max=1e2)
ndim += 1
except:
ferr = 0
pass
#####################
# Dust;
#####################
if f_dust:
Tdust = np.arange(DT0, DT1, dDT)
fit_params.add('TDUST',
value=len(Tdust) / 2.,
min=0,
max=len(Tdust) - 1)
#fit_params.add('TDUST', value=1, min=0, max=len(Tdust)-1)
fit_params.add('MDUST', value=1e6, min=0, max=1e10)
ndim += 2
# Append data;
dat_d = np.loadtxt(DIR_TMP + 'spec_dust_obs_' + ID0 +
'_PA' + PA0 + '.cat',
comments='#')
x_d = dat_d[:, 1]
fy_d = dat_d[:, 2]
ey_d = dat_d[:, 3]
fy = np.append(fy, fy_d)
x = np.append(x, x_d)
wht = np.append(wht, 1. / np.square(ey_d))
wht2 = check_line_man(fy, x, wht, fy, zprev, LW0)
# Then, minimize again.
out = minimize(
residual,
fit_params,
args=(fy, wht2, f_dust),
method=fit_name
) # It needs to define out with redshift constrain.
print(fit_report(out))
# Fix params to what we had before.
out.params['zmc'].value = zrecom
out.params['Av'].value = out_keep.params['Av'].value
for aa in range(len(age)):
out.params['A' +
str(aa)].value = out_keep.params['A' +
str(aa)].value
try:
out.params['Z' + str(aa)].value = out_keep.params[
'Z' + str(aa)].value
except:
out.params['Z0'].value = out_keep.params['Z0'].value
##############################
# Save fig of z-distribution.
##############################
try: # if spectrum;
fig = plt.figure(figsize=(6.5, 2.5))
fig.subplots_adjust(top=0.96,
bottom=0.16,
left=0.09,
right=0.99,
hspace=0.15,
wspace=0.25)
ax1 = fig.add_subplot(111)
#n, nbins, patches = ax1.hist(res_cz.flatchain['z'], bins=200, normed=False, color='gray',label='')
n, nbins, patches = ax1.hist(res_cz.flatchain['z'],
bins=200,
normed=True,
color='gray',
label='')
if f_fitgauss == 1:
ax1.plot(lnspc,
pdf_g,
label='Gaussian fit',
color='g',
linestyle='-') # plot it
ax1.set_xlim(m - s * 3, m + s * 3)
yy = np.arange(0, np.max(n), 1)
xx = yy * 0 + z_cz[1]
ax1.plot(
xx,
yy,
linestyle='-',
linewidth=1,
color='orangered',
label='$z=%.5f_{-%.5f}^{+%.5f}$\n$C_z0=%.3f$\n$C_z1=%.3f$'
% (z_cz[1], z_cz[1] - z_cz[0], z_cz[2] - z_cz[1], Czrec0,
Czrec1))
xx = yy * 0 + z_cz[0]
ax1.plot(xx,
yy,
linestyle='--',
linewidth=1,
color='orangered')
xx = yy * 0 + z_cz[2]
ax1.plot(xx,
yy,
linestyle='--',
linewidth=1,
color='orangered')
xx = yy * 0 + zprev
ax1.plot(xx, yy, linestyle='-', linewidth=1, color='royalblue')
ax1.set_xlabel('Redshift')
ax1.set_ylabel('$dn/dz$')
ax1.legend(loc=0)
plt.savefig('zprob_' + ID0 + '_PA' + PA0 + '.pdf', dpi=300)
plt.close()
except:
print('z-distribution figure is not generated.')
pass
##############################
print('\n\n')
print('###############################')
print('Input redshift is adopted.')
print('Starting long journey in MCMC.')
print('###############################')
print('\n\n')
#################
# Initialize mm.
#################
mm = 0
# add a noise parameter
# out.params.add('f', value=1, min=0.001, max=20)
wht2 = wht
################################
print('\nMinimizer Defined\n')
mini = Minimizer(lnprob,
out.params,
fcn_args=[fy, wht2, f_dust],
f_disp=f_disp,
f_move=f_move)
print('######################')
print('### Starting emcee ###')
print('######################')
import multiprocess
ncpu0 = int(multiprocess.cpu_count() / 2)
try:
ncpu = int(inputs['NCPU'])
if ncpu > ncpu0:
print('!!! NCPU is larger than No. of CPU. !!!')
#print('Now set to %d'%(ncpu0))
#ncpu = ncpu0
except:
ncpu = ncpu0
pass
print('No. of CPU is set to %d' % (ncpu))
start_mc = timeit.default_timer()
res = mini.emcee(burn=int(nmc / 2),
steps=nmc,
thin=10,
nwalkers=nwalk,
params=out.params,
is_weighted=True,
ntemps=ntemp,
workers=ncpu)
stop_mc = timeit.default_timer()
tcalc_mc = stop_mc - start_mc
print('###############################')
print('### MCMC part took %.1f sec ###' % (tcalc_mc))
print('###############################')
#----------- Save pckl file
#-------- store chain into a cpkl file
start_mc = timeit.default_timer()
import corner
burnin = int(nmc / 2)
savepath = './'
cpklname = 'chain_' + ID0 + '_PA' + PA0 + '_corner.cpkl'
savecpkl(
{
'chain': res.flatchain,
'burnin': burnin,
'nwalkers': nwalk,
'niter': nmc,
'ndim': ndim
}, savepath + cpklname) # Already burn in
stop_mc = timeit.default_timer()
tcalc_mc = stop_mc - start_mc
print('#################################')
print('### Saving chain took %.1f sec' % (tcalc_mc))
print('#################################')
Avmc = np.percentile(res.flatchain['Av'], [16, 50, 84])
#Zmc = np.percentile(res.flatchain['Z'], [16,50,84])
Avpar = np.zeros((1, 3), dtype='float32')
Avpar[0, :] = Avmc
out = res
####################
# Best parameters
####################
Amc = np.zeros((len(age), 3), dtype='float32')
Ab = np.zeros(len(age), dtype='float32')
Zmc = np.zeros((len(age), 3), dtype='float32')
Zb = np.zeros(len(age), dtype='float32')
NZbest = np.zeros(len(age), dtype='int')
f0 = fits.open(DIR_TMP + 'ms_' + ID0 + '_PA' + PA0 + '.fits')
sedpar = f0[1]
ms = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
Ab[aa] = out.params['A' + str(aa)].value
Amc[aa, :] = np.percentile(res.flatchain['A' + str(aa)],
[16, 50, 84])
try:
Zb[aa] = out.params['Z' + str(aa)].value
Zmc[aa, :] = np.percentile(res.flatchain['Z' + str(aa)],
[16, 50, 84])
except:
Zb[aa] = out.params['Z0'].value
Zmc[aa, :] = np.percentile(res.flatchain['Z0'],
[16, 50, 84])
NZbest[aa] = bfnc.Z2NZ(Zb[aa])
ms[aa] = sedpar.data['ML_' + str(NZbest[aa])][aa]
Avb = out.params['Av'].value
if f_dust:
Mdust_mc = np.zeros(3, dtype='float32')
Tdust_mc = np.zeros(3, dtype='float32')
Mdust_mc[:] = np.percentile(res.flatchain['MDUST'],
[16, 50, 84])
Tdust_mc[:] = np.percentile(res.flatchain['TDUST'],
[16, 50, 84])
print(Mdust_mc)
print(Tdust_mc)
####################
# MCMC corner plot.
####################
if mcmcplot:
fig1 = corner.corner(res.flatchain, labels=res.var_names, \
label_kwargs={'fontsize':16}, quantiles=[0.16, 0.84], show_titles=False, \
title_kwargs={"fontsize": 14}, truths=list(res.params.valuesdict().values()), \
plot_datapoints=False, plot_contours=True, no_fill_contours=True, \
plot_density=False, levels=[0.68, 0.95, 0.997], truth_color='gray', color='#4682b4')
fig1.savefig('SPEC_' + ID0 + '_PA' + PA0 + '_corner.pdf')
plt.close()
#########################
msmc0 = 0
for aa in range(len(age)):
msmc0 += res.flatchain['A' + str(aa)] * ms[aa]
msmc = np.percentile(msmc0, [16, 50, 84])
# Do analysis on MCMC results.
# Write to file.
stop = timeit.default_timer()
tcalc = stop - start
# Load writing package;
from .writing import Analyze
wrt = Analyze(inputs) # Set up for input
start_mc = timeit.default_timer()
wrt.get_param(res,
lib_all,
zrecom,
Czrec0,
Czrec1,
z_cz[:],
scl_cz0[:],
scl_cz1[:],
fitc[:],
tau0=tau0,
tcalc=tcalc)
stop_mc = timeit.default_timer()
tcalc_mc = stop_mc - start_mc
print('##############################################')
print('### Writing params tp file took %.1f sec ###' % (tcalc_mc))
print('##############################################')
return 0, zrecom, Czrec0, Czrec1
###################################################################
elif flag_z == 'm':
zrecom = float(
raw_input('What is your manual input for redshift? '))
Czrec0 = float(raw_input('What is your manual input for Cz0? '))
Czrec1 = float(raw_input('What is your manual input for Cz1? '))
print('\n\n')
print('Generate model templates with input redshift and Scale.')
print('\n\n')
return 1, zrecom, Czrec0, Czrec1
else:
print('\n\n')
print('Terminated because of redshift estimate.')
print('Generate model templates with recommended redshift.')
print('\n\n')
flag_gen = raw_input(
'Do you want to make templates with recommended redshift, Cz0, and Cz1 , %.5f %.5f %.5f? ([y]/n) '
% (zrecom, Czrec0, Czrec1))
if flag_gen == 'y' or flag_gen == '':
#return 1, zrecom, Cz0 * Czrec0, Cz1 * Czrec1
return 1, zrecom, Czrec0, Czrec1
else:
print('\n\n')
print('There is nothing to do.')
print('\n\n')
return 0, zprev, Czrec0, Czrec1
def fit_gaussian(spec,
vels=None,
vcent=None,
err=None,
amp_const=None,
cent_const=None,
sigma_const=None,
verbose=True,
plot_fit=True,
use_emcee=False,
emcee_kwargs={}):
'''
'''
if vels is None:
spec = spec.with_spectral_unit(u.km / u.s)
vels = spec.spectral_axis
# Set parameter limits:
if amp_const is None:
amp_min = 0.
amp_max = 1.1 * np.nanmax(spec.value)
else:
amp_min = amp_const[0]
amp_max = amp_const[1]
if cent_const is None:
cent_min = vels.value.min() - 0.1 * np.ptp(vels.value)
cent_max = vels.value.max() + 0.1 * np.ptp(vels.value)
else:
cent_min = cent_const[0]
cent_max = cent_const[1]
if sigma_const is None:
sig_min = np.abs(np.diff(vels.value)[0])
sig_max = 0.3 * np.ptp(vels.value)
else:
sig_min = sigma_const[0]
sig_max = sigma_const[1]
if vcent is None:
vcent = np.mean(vels.value)
else:
vcent = vcent.to(u.km / u.s).value
pfit = Parameters()
pfit.add(name='amp', value=20.,
min=amp_min, max=amp_max)
pfit.add(name='cent', value=vcent,
min=cent_min, max=cent_max)
pfit.add(name='sigma', value=10.,
min=sig_min, max=sig_max)
# valid_data = np.isfinite(spec.filled_data[:])
# yfit = spec.filled_data[:].value[valid_data]
# xfit = spec.spectral_axis.value[valid_data]
yfit = spec.value
xfit = vels.value
mini = Minimizer(residual_single, pfit,
fcn_args=(xfit, yfit,
err if err is not None else 1.))
out = mini.leastsq()
if use_emcee:
mini = Minimizer(residual_single, out.params,
fcn_args=(xfit, yfit,
err if err is not None else 1.))
out = mini.emcee(**emcee_kwargs)
if plot_fit:
plt.plot(vels.value, spec.value, drawstyle='steps-mid')
model = gaussian(vels.value,
out.params["amp"],
out.params["cent"],
out.params["sigma"])
plt.plot(vels.value, model)
plt.plot(vels.value, spec.value - model, '--',
zorder=-10)
plt.draw()
return out
def check_redshift(fobs, eobs, xobs, fm_tmp, xm_tmp, zbest, zprior, prior, NR, zliml, zlimu, \
nmc_cz=100, nwalk_cz=10, nthin=5, f_line_check=False, f_vary=True):
'''
Purpose
-------
Fit observed flux with a template to get redshift probability.
Parameters
----------
zbest :
Initial value for redshift.
zprior :
Redshift grid for prior.
prior :
Prior for redshift determination. E.g., Eazy z-probability.
zliml :
Lowest redshift for fitting range.
zlimu :
Highest redshift for fitting range.
f_vary : bool
If want to fix redshift.
fm_tmp :
Template spectrum at RF.
xm_tmp :
Template spectrum at RF.
fobs :
Observed spectrum. (Already scaled with Cz0prev.)
eobs :
Observed spectrum. (Already scaled with Cz0prev.)
xobs :
Observed spectrum. (Already scaled with Cz0prev.)
Returns
-------
res_cz :
fitc_cz :
'''
from .function import check_line_cz_man
import numpy as np
from lmfit import Model, Parameters, minimize, fit_report, Minimizer
import scipy.interpolate as interpolate
fit_par_cz = Parameters()
fit_par_cz.add('z', value=zbest, min=zliml, max=zlimu, vary=f_vary)
fit_par_cz.add('Cz0', value=1, min=0.5, max=1.5)
fit_par_cz.add('Cz1', value=1, min=0.5, max=1.5)
##############################
def residual_z(pars):
vals = pars.valuesdict()
z = vals['z']
Cz0s = vals['Cz0']
Cz1s = vals['Cz1']
xm_s = xm_tmp * (1 + z)
fint = interpolate.interp1d(xm_s,
fm_tmp,
kind='nearest',
fill_value="extrapolate")
fm_s = fint(xobs)
con0 = (NR < 1000)
fy0 = fobs[con0] * Cz0s
ey0 = eobs[con0] * Cz0s
con1 = (NR >= 1000) & (NR < 10000)
fy1 = fobs[con1] * Cz1s
ey1 = eobs[con1] * Cz1s
con2 = (NR >= 10000) # BB
fy2 = fobs[con2]
ey2 = eobs[con2]
fy01 = np.append(fy0, fy1)
fcon = np.append(fy01, fy2)
ey01 = np.append(ey0, ey1)
eycon = np.append(ey01, ey2)
wht = 1. / np.square(eycon)
if f_line_check:
try:
wht2, ypoly = check_line_cz_man(fcon, xobs, wht, fm_s, z)
except:
wht2 = wht
else:
wht2 = wht
if fobs is None:
print('Data is none')
return fm_s
else:
return (fm_s - fcon) * np.sqrt(wht2) # i.e. residual/sigma
###############################
def lnprob_cz(pars):
resid = residual_z(pars) # i.e. (data - model) * wht
z = pars['z']
s_z = 1 #pars['f_cz']
resid *= 1 / s_z
resid *= resid
nzz = np.argmin(np.abs(zprior - z))
# For something unacceptable;
if nzz < 0 or zprior[nzz] < zliml or zprior[nzz] > zlimu or prior[
nzz] <= 0:
return -np.inf
else:
respr = np.log(prior[nzz])
resid += np.log(2 * np.pi * s_z**2)
return -0.5 * np.sum(resid) + respr
#################################
#
# Best fit
#
out_cz = minimize(residual_z, fit_par_cz, method='nelder')
keys = fit_report(out_cz).split('\n')
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc_cz = [csq, rcsq] # Chi2, Reduced-chi2
mini_cz = Minimizer(lnprob_cz, out_cz.params)
res_cz = mini_cz.emcee(burn=int(nmc_cz / 2),
steps=nmc_cz,
thin=nthin,
nwalkers=nwalk_cz,
params=out_cz.params,
is_weighted=True)
return res_cz, fitc_cz
def main(self,
ID0,
PA0,
zgal,
flag_m,
zprev,
Cz0,
Cz1,
mcmcplot=1,
fzvis=1,
specplot=1,
fneld=0,
ntemp=5
): # flag_m related to redshift error in redshift check func.
start = timeit.default_timer()
inputs = self.inputs
DIR_TMP = inputs['DIR_TEMP']
if os.path.exists(DIR_TMP) == False:
os.mkdir(DIR_TMP)
#
# Age
#
age = inputs['AGE']
age = [float(x.strip()) for x in age.split(',')]
nage = np.arange(0, len(age), 1)
#
# Metallicity
#
Zmax, Zmin = float(inputs['ZMAX']), float(inputs['ZMIN'])
delZ = float(inputs['DELZ'])
Zall = np.arange(Zmin, Zmax, delZ) # in logZsun
# For minimizer.
delZtmp = delZ
#delZtmp = 0.4 # to increase speed.
# For z prior.
delzz = 0.001
zlimu = 6.
snlim = 1
zliml = zgal - 0.5
# N of param:
try:
ndim = int(inputs['NDIM'])
except:
if int(inputs['ZEVOL']) == 1:
ndim = int(len(nage) * 2 + 1)
else:
ndim = int(len(nage) + 1 + 1)
pass
#
# Line
#
LW0 = inputs['LINE']
LW0 = [float(x.strip()) for x in LW0.split(',')]
#
# Params for MCMC
#
nmc = int(inputs['NMC'])
nwalk = int(inputs['NWALK'])
nmc_cz = int(inputs['NMCZ'])
nwalk_cz = int(inputs['NWALKZ'])
f_Zevol = int(inputs['ZEVOL'])
#f_zvis = int(inputs['ZVIS'])
#
# Tau for MCMC parameter; not as fitting parameters.
#
tau0 = inputs['TAU0']
tau0 = [float(x.strip()) for x in tau0.split(',')]
#tau0 = [0.1,0.2,0.3] # Gyr
from .function_class import Func
from .basic_func import Basic
fnc = Func(Zall, nage) # Set up the number of Age/ZZ
bfnc = Basic(Zall)
from .writing import Analyze
wrt = Analyze(inputs) # Set up for input
# Open ascii file and stock to array.
#lib = open_spec(ID0, PA0)
lib = fnc.open_spec_fits(ID0, PA0, fall=0, tau0=tau0)
lib_all = fnc.open_spec_fits(ID0, PA0, fall=1, tau0=tau0)
#####################
# Model templates.
#####################
chimax = 1.
#################
# Observed Data
#################
##############
# Spectrum
##############
dat = np.loadtxt(DIR_TMP + 'spec_obs_' + ID0 + '_PA' + PA0 + '.cat',
comments='#')
NR = dat[:, 0]
x = dat[:, 1]
fy00 = dat[:, 2]
ey00 = dat[:, 3]
con0 = (NR < 1000)
fy0 = fy00[con0] * Cz0
ey0 = ey00[con0] * Cz0
con1 = (NR >= 1000) & (NR < 10000)
fy1 = fy00[con1] * Cz1
ey1 = ey00[con1] * Cz1
con2 = (NR >= 10000) # BB
fy2 = fy00[con2]
ey2 = ey00[con2]
fy01 = np.append(fy0, fy1)
fy = np.append(fy01, fy2)
ey01 = np.append(ey0, ey1)
ey = np.append(ey01, ey2)
wht = 1. / np.square(ey)
sn = fy / ey
##############
# Broadband
##############
dat = np.loadtxt(DIR_TMP + 'bb_obs_' + ID0 + '_PA' + PA0 + '.cat',
comments='#')
NRbb = dat[:, 0]
xbb = dat[:, 1]
fybb = dat[:, 2]
eybb = dat[:, 3]
exbb = dat[:, 4]
wht2 = check_line_man(fy, x, wht, fy, zprev, LW0)
#####################
# Function fo MCMC
#####################
def residual(pars): # x, y, wht are taken from out of the definition.
vals = pars.valuesdict()
Av = vals['Av']
######################
'''
for aa in range(len(age)):
if aa == 0:
mod0, x1 = fnc.tmp03(ID0, PA0, vals['A'+str(aa)], Av, aa, vals['Z'+str(aa)], zprev, lib, tau0)
model = mod0
else:
mod0, xxbs = fnc.tmp03(ID0, PA0, vals['A'+str(aa)], Av, aa, vals['Z'+str(aa)], zprev, lib, tau0)
model += mod0
'''
model, x1 = fnc.tmp04(ID0, PA0, vals, zprev, lib, tau0=tau0)
######################
if fy is None:
print('Data is none')
return model
else:
return (model - fy) * np.sqrt(wht2) # i.e. residual/sigma
def lnprob(pars):
vals = pars.valuesdict()
Av = vals['Av']
resid = residual(pars)
s = 1. #pars['f']
resid *= 1 / s
resid *= resid
resid += np.log(2 * np.pi * s**2)
if Av < 0:
return -np.inf
else:
respr = np.log(1)
return -0.5 * np.sum(resid) + respr
###############################
# Add parameters
###############################
fit_params = Parameters()
for aa in range(len(age)):
if age[aa] == 99:
fit_params.add('A' + str(aa), value=0, min=0, max=0.01)
else:
fit_params.add('A' + str(aa), value=1, min=0, max=400)
#fit_params.add('Av', value=1., min=0, max=4.0)
fit_params.add('Av', value=0.2, min=0, max=4.0)
#fit_params.add('Z', value=0, min=np.min(Zall), max=np.max(Zall))
for aa in range(len(age)):
if age[aa] == 99:
fit_params.add('Z' + str(aa), value=1, min=0, max=0.01)
else:
fit_params.add('Z' + str(aa),
value=0,
min=np.min(Zall),
max=np.max(Zall))
##################################
# Metallicity determination
##################################
chidef = 1e5
Zbest = 0
fwz = open('Z_' + ID0 + '_PA' + PA0 + '.cat', 'w')
fwz.write('# ID Zini chi/nu AA Av Zbest\n')
nZtmp = int((Zmax - Zmin) / delZtmp)
if fneld == 1:
for zz in range(0, nZtmp, 1):
#for zz in range(2,7,2):
ZZ = zz * delZtmp + np.min(Zall)
for aa in range(len(age)):
fit_params['Z' + str(aa)].value = ZZ
out_tmp = minimize(
residual, fit_params,
method='nelder') # nelder is the most efficient.
#print(ZZ, bfnc.Z2NZ(ZZ))
#print(fit_report(out_tmp))
#print('\n')
keys = fit_report(out_tmp).split('\n')
csq = 99999
rcsq = 99999
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc = [csq, rcsq] # Chi2, Reduced-chi2
fwz.write('%s %.2f %.5f' % (ID0, ZZ, fitc[1]))
AA_tmp = np.zeros(len(age), dtype='float32')
ZZ_tmp = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
AA_tmp[aa] = out_tmp.params['A' + str(aa)].value
fwz.write(' %.5f' % (AA_tmp[aa]))
Av_tmp = out_tmp.params['Av'].value
fwz.write(' %.5f' % (Av_tmp))
for aa in range(len(age)):
ZZ_tmp[aa] = out_tmp.params['Z' + str(aa)].value
fwz.write(' %.5f' % (ZZ_tmp[aa]))
fwz.write('\n')
if fitc[1] < chidef:
chidef = fitc[1]
out = out_tmp
else:
for zz in range(0, nZtmp, 2):
ZZ = zz * delZtmp + np.min(Zall)
for aa in range(len(age)):
fit_params['Z' + str(aa)].value = ZZ
#
out_tmp = minimize(
residual, fit_params,
method='powell') # powel is the more accurate.
#print(ZZ, bfnc.Z2NZ(ZZ))
#print(fit_report(out_tmp))
#print('\n')
keys = fit_report(out_tmp).split('\n')
csq = 99999
rcsq = 99999
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc = [csq, rcsq] # Chi2, Reduced-chi2
fwz.write('%s %.2f %.5f' % (ID0, ZZ, fitc[1]))
AA_tmp = np.zeros(len(age), dtype='float32')
ZZ_tmp = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
AA_tmp[aa] = out_tmp.params['A' + str(aa)].value
fwz.write(' %.5f' % (AA_tmp[aa]))
Av_tmp = out_tmp.params['Av'].value
fwz.write(' %.5f' % (Av_tmp))
#Z_tmp = out_tmp.params['Z'].value
for aa in range(len(age)):
ZZ_tmp[aa] = out_tmp.params['Z' + str(aa)].value
fwz.write(' %.5f' % (ZZ_tmp[aa]))
fwz.write('\n')
if fitc[1] < chidef:
chidef = fitc[1]
out = out_tmp
#
# Best fit
#
keys = fit_report(out).split('\n')
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc = [csq, rcsq] # Chi2, Reduced-chi2
#fitc = fit_report_chi(out) # Chi2, Reduced-chi2
ZZ = Zbest # This is really important/does affect lnprob/residual.
print('\n\n')
print('#####################################')
print('Zbest, chi are;', Zbest, chidef)
print('Params are;', fit_report(out))
print('#####################################')
print('\n\n')
fwz.close()
Av_tmp = out.params['Av'].value
#Z_tmp = Zbest
AA_tmp = np.zeros(len(age), dtype='float32')
ZZ_tmp = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
AA_tmp[aa] = out.params['A' + str(aa)].value
ZZ_tmp[aa] = out.params['Z' + str(aa)].value
if aa == 0:
mod0_tmp, xm_tmp = fnc.tmp03(ID0, PA0, AA_tmp[aa], Av_tmp, aa,
ZZ_tmp[aa], zprev, lib, tau0)
fm_tmp = mod0_tmp
else:
mod0_tmp, xx_tmp = fnc.tmp03(ID0, PA0, AA_tmp[aa], Av_tmp, aa,
ZZ_tmp[aa], zprev, lib, tau0)
fm_tmp += mod0_tmp
########################
# Check redshift
########################
zrecom = zprev
# Observed data.
con_cz = (NR < 10000) #& (sn>snlim)
fy_cz = fy[con_cz]
ey_cz = ey[con_cz]
x_cz = x[con_cz] # Observed range
NR_cz = NR[con_cz]
xm_s = xm_tmp / (1 + zprev) * (1 + zrecom)
fm_s = np.interp(x_cz, xm_s, fm_tmp)
if fzvis == 1:
plt.plot(x_cz / (1 + zprev) * (1. + zrecom),
fm_s,
'gray',
linestyle='--',
linewidth=0.5,
label='Default ($z=%.5f$)' %
(zprev)) # Model based on input z.
plt.plot(x_cz,
fy_cz,
'b',
linestyle='-',
linewidth=0.5,
label='Obs.') # Observation
plt.errorbar(x_cz,
fy_cz,
yerr=ey_cz,
color='b',
capsize=0,
linewidth=0.5) # Observation
if flag_m == 0:
dez = 0.5
else:
dez = 0.2
#
# For Eazy
#
'''
dprob = np.loadtxt(eaz_path + 'photz_' + str(int(ID0)) + '.pz', comments='#')
zprob = dprob[:,0]
cprob = dprob[:,1]
zz_prob = np.arange(0,13,delzz)
cprob_s = np.interp(zz_prob, zprob, cprob)
prior_s = 1/cprob_s
prior_s /= np.sum(prior_s)
'''
zz_prob = np.arange(0, 13, delzz)
prior_s = zz_prob * 0 + 1.
prior_s /= np.sum(prior_s)
print(
'################\nStart MCMC for redshift fit\n################')
res_cz, fitc_cz = check_redshift(fy_cz, ey_cz, x_cz, fm_tmp,
xm_tmp / (1 + zprev), zprev, dez,
prior_s, NR_cz, zliml, zlimu, delzz,
nmc_cz, nwalk_cz)
z_cz = np.percentile(res_cz.flatchain['z'], [16, 50, 84])
scl_cz0 = np.percentile(res_cz.flatchain['Cz0'], [16, 50, 84])
scl_cz1 = np.percentile(res_cz.flatchain['Cz1'], [16, 50, 84])
zrecom = z_cz[1]
Czrec0 = scl_cz0[1]
Czrec1 = scl_cz1[1]
xm_s = xm_tmp / (1 + zprev) * (1 + zrecom)
fm_s = np.interp(x_cz, xm_s, fm_tmp)
whtl = 1 / np.square(ey_cz)
wht2, ypoly = check_line_cz_man(fy_cz, x_cz, whtl, fm_s, zrecom, LW0)
con_line = (wht2 == 0)
print('\n\n')
print(
'Recommended redshift, Cz0 and Cz1, %.5f %.5f %.5f, with chi2/nu=%.3f'
% (zrecom, Cz0 * Czrec0, Cz1 * Czrec1, fitc_cz[1]))
print('\n\n')
if fzvis == 1:
plt.plot(x_cz,
fm_s,
'r',
linestyle='-',
linewidth=0.5,
label='Updated model ($z=%.5f$)' %
(zrecom)) # Model based on recomended z.
plt.plot(x_cz[con_line],
fm_s[con_line],
color='orange',
marker='o',
linestyle='',
linewidth=3.)
# Plot lines for reference
for ll in range(len(LW)):
conpoly = (x_cz > 12000) & (x_cz < 16500)
yline = np.max(ypoly[conpoly])
yy = np.arange(yline / 1.02, yline * 1.1)
xxpre = yy * 0 + LW[ll] * (1. + zprev)
xx = yy * 0 + LW[ll] * (1. + zrecom)
plt.plot(xxpre,
yy / 1.02,
linewidth=0.5,
linestyle='--',
color='gray')
plt.text(LW[ll] * (1. + zprev),
yline / 1.05,
'%s' % (LN[ll]),
fontsize=8,
color='gray')
plt.plot(xx,
yy,
linewidth=0.5,
linestyle='-',
color='orangered')
plt.text(LW[ll] * (1. + zrecom),
yline,
'%s' % (LN[ll]),
fontsize=8,
color='orangered')
plt.plot(xbb,
fybb,
'.r',
linestyle='',
linewidth=0,
zorder=4,
label='Obs.(BB)')
plt.plot(xm_tmp,
fm_tmp,
color='gray',
marker='.',
ms=0.5,
linestyle='',
linewidth=0.5,
zorder=4,
label='Model')
xmin, xmax = np.min(x_cz) / 1.1, np.max(x_cz) * 1.1
plt.xlim(xmin, xmax)
plt.ylim(0, yline * 1.1)
plt.xlabel('Wavelength ($\mathrm{\AA}$)')
plt.ylabel('$F_\\nu$ (arb.)')
plt.legend(loc=0)
zzsigma = ((z_cz[2] - z_cz[0]) / 2.) / zprev
zsigma = np.abs(zprev - zrecom) / (zprev)
C0sigma = np.abs(Czrec0 - Cz0) / Cz0
eC0sigma = ((scl_cz0[2] - scl_cz0[0]) / 2.) / Cz0
C1sigma = np.abs(Czrec1 - Cz1) / Cz1
eC1sigma = ((scl_cz1[2] - scl_cz1[0]) / 2.) / Cz1
print('Input redshift is %.3f per cent agreement.' %
((1. - zsigma) * 100))
print('Error is %.3f per cent.' % (zzsigma * 100))
print('Input Cz0 is %.3f per cent agreement.' %
((1. - C0sigma) * 100))
print('Error is %.3f per cent.' % (eC0sigma * 100))
print('Input Cz1 is %.3f per cent agreement.' %
((1. - C1sigma) * 100))
print('Error is %.3f per cent.' % (eC1sigma * 100))
plt.show()
#
# Ask interactively;
#
flag_z = raw_input(
'Do you want to continue with original redshift, Cz0 and Cz1, %.5f %.5f %.5f? ([y]/n/m) '
% (zprev, Cz0, Cz1))
else:
flag_z = 'y'
#################################################
# Gor for mcmc phase
#################################################
if flag_z == 'y' or flag_z == '':
zrecom = zprev
Czrec0 = Cz0
Czrec1 = Cz1
##############################
# Save fig of z-distribution.
##############################
fig = plt.figure(figsize=(6.5, 2.5))
fig.subplots_adjust(top=0.96,
bottom=0.16,
left=0.09,
right=0.99,
hspace=0.15,
wspace=0.25)
ax1 = fig.add_subplot(111)
n, nbins, patches = ax1.hist(res_cz.flatchain['z'],
bins=200,
normed=False,
color='gray',
label='')
yy = np.arange(0, np.max(n), 1)
xx = yy * 0 + z_cz[1]
ax1.plot(
xx,
yy,
linestyle='-',
linewidth=1,
color='orangered',
label='$z=%.5f_{-%.5f}^{+%.5f}$\n$C_z0=%.3f$\n$C_z1=%.3f$' %
(z_cz[1], z_cz[1] - z_cz[0], z_cz[2] - z_cz[1], Czrec0,
Czrec1))
xx = yy * 0 + z_cz[0]
ax1.plot(xx, yy, linestyle='--', linewidth=1, color='orangered')
xx = yy * 0 + z_cz[2]
ax1.plot(xx, yy, linestyle='--', linewidth=1, color='orangered')
xx = yy * 0 + zprev
ax1.plot(xx, yy, linestyle='-', linewidth=1, color='royalblue')
ax1.set_xlabel('Redshift')
ax1.set_ylabel('$dn/dz$')
ax1.legend(loc=0)
plt.savefig('zprob_' + ID0 + '_PA' + PA0 + '.pdf', dpi=300)
plt.close()
##############################
print('\n\n')
print('Input redshift is adopted.')
print('Starting long journey in MCMC.')
print('\n\n')
#################
# Initialize mm.
#################
mm = 0
# add a noise parameter
# out.params.add('f', value=1, min=0.001, max=20)
wht2 = wht
################################
print('Defined Minimizer\n')
mini = Minimizer(lnprob, out.params)
print('################\nStarting emcee\n################\n')
import multiprocessing
ncpu0 = int(multiprocessing.cpu_count() / 2)
try:
ncpu = int(inputs['NCPU'])
if ncpu > ncpu0:
print('!!! NCPU is larger than No. of CPU. !!!')
#print('Now set to %d'%(ncpu0))
#ncpu = ncpu0
except:
ncpu = ncpu0
pass
print('No. of CPU is set to %d' % (ncpu))
start_mc = timeit.default_timer()
res = mini.emcee(burn=int(nmc / 2),
steps=nmc,
thin=10,
nwalkers=nwalk,
params=out.params,
is_weighted=True,
ntemps=ntemp,
workers=ncpu)
stop_mc = timeit.default_timer()
tcalc_mc = stop_mc - start_mc
print(
'#######################\nMCMC part took %.1f sec\n#######################'
% (tcalc_mc))
#----------- Save pckl file
#-------- store chain into a cpkl file
import corner
burnin = int(nmc / 2)
savepath = './'
cpklname = 'chain_' + ID0 + '_PA' + PA0 + '_corner.cpkl'
savecpkl(
{
'chain': res.flatchain,
'burnin': burnin,
'nwalkers': nwalk,
'niter': nmc,
'ndim': ndim
}, savepath + cpklname) # Already burn in
Avmc = np.percentile(res.flatchain['Av'], [16, 50, 84])
#Zmc = np.percentile(res.flatchain['Z'], [16,50,84])
Avpar = np.zeros((1, 3), dtype='float32')
Avpar[0, :] = Avmc
out = res
##############################
# Best parameters
Amc = np.zeros((len(age), 3), dtype='float32')
Ab = np.zeros(len(age), dtype='float32')
Zmc = np.zeros((len(age), 3), dtype='float32')
Zb = np.zeros(len(age), dtype='float32')
NZbest = np.zeros(len(age), dtype='int')
f0 = fits.open(DIR_TMP + 'ms_' + ID0 + '_PA' + PA0 + '.fits')
sedpar = f0[1]
ms = np.zeros(len(age), dtype='float32')
for aa in range(len(age)):
Ab[aa] = out.params['A' + str(aa)].value
Amc[aa, :] = np.percentile(res.flatchain['A' + str(aa)],
[16, 50, 84])
Zb[aa] = out.params['Z' + str(aa)].value
Zmc[aa, :] = np.percentile(res.flatchain['Z' + str(aa)],
[16, 50, 84])
NZbest[aa] = bfnc.Z2NZ(Zb[aa])
ms[aa] = sedpar.data['ML_' + str(NZbest[aa])][aa]
Avb = out.params['Av'].value
####################
# MCMC corner plot.
####################
if mcmcplot == 1:
fig1 = corner.corner(res.flatchain,
labels=res.var_names,
label_kwargs={'fontsize': 16},
quantiles=[0.16, 0.84],
show_titles=False,
title_kwargs={"fontsize": 14},
truths=list(
res.params.valuesdict().values()),
plot_datapoints=False,
plot_contours=True,
no_fill_contours=True,
plot_density=False,
levels=[0.68, 0.95, 0.997],
truth_color='gray',
color='#4682b4')
fig1.savefig('SPEC_' + ID0 + '_PA' + PA0 + '_corner.pdf')
plt.close()
#########################
msmc0 = 0
for aa in range(len(age)):
msmc0 += res.flatchain['A' + str(aa)] * ms[aa]
msmc = np.percentile(msmc0, [16, 50, 84])
# Do analysis on MCMC results.
# Write to file.
stop = timeit.default_timer()
tcalc = stop - start
wrt.get_param(res,
lib_all,
zrecom,
Czrec0,
Czrec1,
z_cz[:],
scl_cz0[:],
scl_cz1[:],
fitc[:],
tau0=tau0,
tcalc=tcalc)
##########
# Plot
##########
if specplot == 1:
plt.close()
try:
DIR_FILT = inputs['DIR_FILT']
except:
DIR_FILT = './'
plot_sed_Z(ID0, PA0, Zall, age, tau0=tau0, fil_path=DIR_FILT)
plot_sfh_pcl2(ID0,
PA0,
Zall,
age,
f_comp=ftaucomp,
fil_path=DIR_FILT)
return 0, zrecom, Czrec0, Czrec1
###################################################################
elif flag_z == 'm':
zrecom = float(
raw_input('What is your manual input for redshift? '))
Czrec0 = float(raw_input('What is your manual input for Cz0? '))
Czrec1 = float(raw_input('What is your manual input for Cz1? '))
print('\n\n')
print('Generate model templates with input redshift and Scale.')
print('\n\n')
return 1, zrecom, Czrec0, Czrec1
else:
print('\n\n')
print('Terminated because of redshift estimate.')
print('Generate model templates with recommended redshift.')
print('\n\n')
flag_gen = raw_input(
'Do you want to make templates with recommended redshift, Cz0, and Cz1 , %.5f %.5f %.5f? ([y]/n) '
% (zrecom, Czrec0, Czrec1))
if flag_gen == 'y' or flag_gen == '':
#return 1, zrecom, Cz0 * Czrec0, Cz1 * Czrec1
return 1, zrecom, Czrec0, Czrec1
else:
print('\n\n')
print('There is nothing to do.')
print('\n\n')
return 0, zprev, Czrec0, Czrec1
def fit_blended_line_emcee(self, x, y, zero_lev, err_continuum, Ncomps, fitting_parameters, add_wide_component, fitting_parameters_wide, bootstrap_iterations = 1000, MC_iterations = 200):
#---First we integrate the brute area with all the components
if self.fit_dict['MC_iterations'] == 1:
self.fit_dict['area_intg'] = simps(y, x) - simps(zero_lev, x)
self.fit_dict['area_intg_err'] = 0.0
else:
area_array = empty(bootstrap_iterations)
len_x_array = len(x)
for i in range(bootstrap_iterations):
y_new = y + np_normal_dist(0.0, err_continuum, len_x_array)
area_array[i] = simps(y_new, x) - simps(zero_lev, x)
self.fit_dict['area_intg'] = mean(area_array)
self.fit_dict['area_intg_err'] = std(area_array)
#---Second we proceed to analyze as gaussian components
idcs_components = map(str, range(Ncomps))
mini_posterior = Minimizer(lnprob_gaussMix, fitting_parameters, fcn_args = ([x, y, zero_lev, err_continuum, idcs_components]), method='powell')
fit_output = mini_posterior.emcee(steps=MC_iterations, params = fitting_parameters)
output_params = fit_output.params
if add_wide_component: #This currently only valid for Halpha
sigma_limit = output_params['sigma1'].value
limit_0, limit_1 = 6548.05 - self.fit_dict['x_scaler'] - sigma_limit * 1.5, 6548.05 - self.fit_dict['x_scaler'] + sigma_limit * 1.5
limit_2, limit_3 = 0 - sigma_limit * 4, 0 + sigma_limit * 4
limit_4, limit_5 = 6583.46 - self.fit_dict['x_scaler'] - sigma_limit * 3, 6583.46 - self.fit_dict['x_scaler'] + sigma_limit * 3
#Get the wide component area
indeces = ((x >= limit_0) & (x <= limit_1)) + ((x >= limit_2) & (x <= limit_3)) + ((x >= limit_4) & (x <= limit_5))
mask = invert(indeces)
x_wide, y_wide, zero_wide = x[mask], y[mask], zero_lev[mask]
Ncomps_wide = ['3']
#Fit the wide component without narrow component
mini_posterior_wide = Minimizer(lnprob_gaussMix, fitting_parameters_wide, fcn_args = ([x_wide, y_wide, zero_wide, err_continuum, Ncomps_wide]), method='powell')
fit_output_wide = mini_posterior_wide.emcee(steps=MC_iterations, params = fitting_parameters_wide)
output_params_wide = fit_output_wide.params
#Calculate wide component curve
y_wide_fit = gaussian_mixture(output_params_wide.valuesdict(), x, zero_lev, Ncomps_wide)
#Calculate emission line region again
y_pure_narrow = y - y_wide_fit + zero_lev
#Fit narrow components again
mini_posterior = Minimizer(lnprob_gaussMix, fitting_parameters, fcn_args = ([x, y_pure_narrow, zero_lev, err_continuum, idcs_components]), method='powell')
fit_output_narrow = mini_posterior.emcee(steps=MC_iterations, params=fitting_parameters)
output_params_narrow = fit_output_narrow.params
#Combine the results from both fits
output_params = output_params_narrow + output_params_wide
#Add the wide component to the fit we are performing
self.fit_dict.line_number = self.fit_dict.line_number + 1
for key in self.fit_dict['parameters_list']:
self.fit_dict[key + '_norm'] = output_params[key].value if output_params[key].value is not None else np_nan
self.fit_dict[key + '_norm_er'] = output_params[key].stderr if output_params[key].stderr is not None else np_nan
return
def fit_isoturbHI_model_simple(vels,
spec,
vcent,
delta_vcent=5 * u.km / u.s,
err=None,
verbose=True,
plot_fit=True,
return_model=False,
use_emcee=False,
emcee_kwargs={}):
vels = vels.copy().to(u.km / u.s)
vel_min = (vcent - delta_vcent).to(vels.unit).value
vel_max = (vcent + delta_vcent).to(vels.unit).value
# Create the parameter list.
pfit = Parameters()
# pfit.add(name='Ts', value=100., min=10**1.2, max=8000)
pfit.add(name='Ts', value=np.nanmax(spec.value), min=10**1.2, max=8000)
pfit.add(name='sigma', value=15., min=0.30, max=31.6) # min v at Ts=16 K
# pfit.add(name='log_NH', value=21., min=20., max=23.5)
pfit.add(name='Tpeak',
value=np.nanmax(spec.value),
min=0,
max=5. * np.nanmax(spec.value))
pfit.add(name='vcent',
value=vcent.to(vels.unit).value,
min=vel_min,
max=vel_max)
try:
finite_mask = np.isfinite(spec.filled_data[:])
except AttributeError:
finite_mask = np.isfinite(spec)
# Some cases are failing with a TypeError due to a lack
# of data. This really shouldn't happen, but I'll throw in this
# to catch those cases.
if finite_mask.sum() <= 4:
return None
if err is not None:
fcn_args = (vels[finite_mask].value, spec[finite_mask].value,
err.value)
else:
fcn_args = (vels[finite_mask].value, spec[finite_mask].value, 1.)
try:
mini = Minimizer(residual,
pfit,
fcn_args=fcn_args,
max_nfev=vels.size * 1000,
nan_policy='omit')
out = mini.leastsq()
except TypeError:
return None
if use_emcee:
mini = Minimizer(residual,
out.params,
fcn_args=fcn_args,
max_nfev=vels.size * 1000)
out = mini.emcee(**emcee_kwargs)
if verbose:
report_fit(out)
pars = out.params
model = isoturbHI_simple(vels.value, pars['Ts'].value, pars['sigma'].value,
pars['Tpeak'].value, pars['vcent'].value)
if plot_fit:
plt.plot(vels.value, spec.value, drawstyle='steps-mid')
plt.plot(vels.value, model)
if return_model:
return out, vels.value, model
return out
class CommonMinimizerTest(unittest.TestCase):
def setUp(self):
"""
test scale minimizers except newton-cg (needs jacobian) and
anneal (doesn't work out of the box).
"""
p_true = Parameters()
p_true.add('amp', value=14.0)
p_true.add('period', value=5.33)
p_true.add('shift', value=0.123)
p_true.add('decay', value=0.010)
self.p_true = p_true
n = 2500
xmin = 0.
xmax = 250.0
noise = np.random.normal(scale=0.7215, size=n)
self.x = np.linspace(xmin, xmax, n)
self.data = self.residual(p_true, self.x) + noise
fit_params = Parameters()
fit_params.add('amp', value=11.0, min=5, max=20)
fit_params.add('period', value=5., min=1., max=7)
fit_params.add('shift', value=.10, min=0.0, max=0.2)
fit_params.add('decay', value=6.e-3, min=0, max=0.1)
self.fit_params = fit_params
self.mini = Minimizer(self.residual, fit_params, [self.x, self.data])
def residual(self, pars, x, data=None):
amp = pars['amp']
per = pars['period']
shift = pars['shift']
decay = pars['decay']
if abs(shift) > pi/2:
shift = shift - np.sign(shift) * pi
model = amp*np.sin(shift + x/per) * np.exp(-x*x*decay*decay)
if data is None:
return model
return model - data
def test_diffev_bounds_check(self):
# You need finite (min, max) for each parameter if you're using
# differential_evolution.
self.fit_params['decay'].min = -np.inf
self.fit_params['decay'].vary = True
self.minimizer = 'differential_evolution'
pytest.raises(ValueError, self.scalar_minimizer)
# but only if a parameter is not fixed
self.fit_params['decay'].vary = False
self.mini.scalar_minimize(method='differential_evolution', maxiter=1)
def test_scalar_minimizers(self):
# test all the scalar minimizers
for method in SCALAR_METHODS:
if method in ['newton', 'dogleg', 'trust-ncg', 'cg', 'trust-exact',
'trust-krylov', 'trust-constr']:
continue
self.minimizer = SCALAR_METHODS[method]
if method == 'Nelder-Mead':
sig = 0.2
else:
sig = 0.15
self.scalar_minimizer(sig=sig)
def scalar_minimizer(self, sig=0.15):
out = self.mini.scalar_minimize(method=self.minimizer)
self.residual(out.params, self.x)
for para, true_para in zip(out.params.values(), self.p_true.values()):
check_wo_stderr(para, true_para.value, sig=sig)
def test_nan_policy(self):
# check that an error is raised if there are nan in
# the data returned by userfcn
self.data[0] = np.nan
major, minor, _micro = scipy_version.split('.', 2)
for method in SCALAR_METHODS:
if (method == 'differential_evolution' and int(major) > 0 and
int(minor) >= 2):
pytest.raises(RuntimeError, self.mini.scalar_minimize,
SCALAR_METHODS[method])
else:
pytest.raises(ValueError, self.mini.scalar_minimize,
SCALAR_METHODS[method])
pytest.raises(ValueError, self.mini.minimize)
# now check that the fit proceeds if nan_policy is 'omit'
self.mini.nan_policy = 'omit'
res = self.mini.minimize()
assert_equal(res.ndata, np.size(self.data, 0) - 1)
for para, true_para in zip(res.params.values(), self.p_true.values()):
check_wo_stderr(para, true_para.value, sig=0.15)
def test_nan_policy_function(self):
a = np.array([0, 1, 2, 3, np.nan])
pytest.raises(ValueError, _nan_policy, a)
assert_(np.isnan(_nan_policy(a, nan_policy='propagate')[-1]))
assert_equal(_nan_policy(a, nan_policy='omit'), [0, 1, 2, 3])
a[-1] = np.inf
pytest.raises(ValueError, _nan_policy, a)
assert_(np.isposinf(_nan_policy(a, nan_policy='propagate')[-1]))
assert_equal(_nan_policy(a, nan_policy='omit'), [0, 1, 2, 3])
assert_equal(_nan_policy(a, handle_inf=False), a)
@dec.slow
def test_emcee(self):
# test emcee
if not HAS_EMCEE:
return True
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200, burn=50, thin=10)
check_paras(out.params, self.p_true, sig=3)
@dec.slow
def test_emcee_method_kwarg(self):
# test with emcee as method keyword argument
if not HAS_EMCEE:
return True
np.random.seed(123456)
out = self.mini.minimize(method='emcee', nwalkers=100, steps=200,
burn=50, thin=10)
assert out.method == 'emcee'
assert out.nfev == 100*200
check_paras(out.params, self.p_true, sig=3)
@dec.slow
def test_emcee_PT(self):
# test emcee with parallel tempering
if not HAS_EMCEE:
return True
np.random.seed(123456)
self.mini.userfcn = residual_for_multiprocessing
out = self.mini.emcee(ntemps=4, nwalkers=50, steps=200,
burn=100, thin=10, workers=2)
check_paras(out.params, self.p_true, sig=3)
@dec.slow
def test_emcee_multiprocessing(self):
# test multiprocessing runs
if not HAS_EMCEE:
return True
np.random.seed(123456)
self.mini.userfcn = residual_for_multiprocessing
self.mini.emcee(steps=10, workers=4)
def test_emcee_bounds_length(self):
# the log-probability functions check if the parameters are
# inside the bounds. Check that the bounds and parameters
# are the right lengths for comparison. This can be done
# if nvarys != nparams
if not HAS_EMCEE:
return True
self.mini.params['amp'].vary = False
self.mini.params['period'].vary = False
self.mini.params['shift'].vary = False
self.mini.emcee(steps=10)
@dec.slow
def test_emcee_partial_bounds(self):
# mcmc with partial bounds
if not HAS_EMCEE:
return True
np.random.seed(123456)
# test mcmc output vs lm, some parameters not bounded
self.fit_params['amp'].max = np.inf
# self.fit_params['amp'].min = -np.inf
out = self.mini.emcee(nwalkers=100, steps=300, burn=100, thin=10)
check_paras(out.params, self.p_true, sig=3)
def test_emcee_init_with_chain(self):
# can you initialise with a previous chain
if not HAS_EMCEE:
return True
out = self.mini.emcee(nwalkers=100, steps=5)
# can initialise with a chain
self.mini.emcee(nwalkers=100, steps=1, pos=out.chain)
# can initialise with a correct subset of a chain
self.mini.emcee(nwalkers=100, steps=1, pos=out.chain[..., -1, :])
# but you can't initialise if the shape is wrong.
pytest.raises(ValueError,
self.mini.emcee,
nwalkers=100,
steps=1,
pos=out.chain[..., -1, :-1])
def test_emcee_reuse_sampler(self):
if not HAS_EMCEE:
return True
self.mini.emcee(nwalkers=100, steps=5)
# if you've run the sampler the Minimizer object should have a _lastpos
# attribute
assert_(hasattr(self.mini, '_lastpos'))
# now try and re-use sampler
out2 = self.mini.emcee(steps=10, reuse_sampler=True)
assert_(out2.chain.shape[1] == 15)
# you shouldn't be able to reuse the sampler if nvarys has changed.
self.mini.params['amp'].vary = False
pytest.raises(ValueError, self.mini.emcee, reuse_sampler=True)
def test_emcee_lnpost(self):
# check ln likelihood is calculated correctly. It should be
# -0.5 * chi**2.
result = self.mini.minimize()
# obtain the numeric values
# note - in this example all the parameters are varied
fvars = np.array([par.value for par in result.params.values()])
# calculate the cost function with scaled values (parameters all have
# lower and upper bounds.
scaled_fvars = []
for par, fvar in zip(result.params.values(), fvars):
par.value = fvar
scaled_fvars.append(par.setup_bounds())
val = self.mini.penalty(np.array(scaled_fvars))
# calculate the log-likelihood value
bounds = np.array([(par.min, par.max)
for par in result.params.values()])
val2 = _lnpost(fvars,
self.residual,
result.params,
result.var_names,
bounds,
userargs=(self.x, self.data))
assert_almost_equal(-0.5 * val, val2)
def test_emcee_output(self):
# test mcmc output
if not HAS_EMCEE:
return True
try:
from pandas import DataFrame
except ImportError:
return True
out = self.mini.emcee(nwalkers=10, steps=20, burn=5, thin=2)
assert_(isinstance(out, MinimizerResult))
assert_(isinstance(out.flatchain, DataFrame))
# check that we can access the chains via parameter name
assert_(out.flatchain['amp'].shape[0] == 80)
assert out.errorbars
assert_(np.isfinite(out.params['amp'].correl['period']))
# the lnprob array should be the same as the chain size
assert_(np.size(out.chain)//out.nvarys == np.size(out.lnprob))
# test chain output shapes
assert_(out.lnprob.shape == (10, (20-5+1)/2))
assert_(out.chain.shape == (10, (20-5+1)/2, out.nvarys))
assert_(out.flatchain.shape == (10*(20-5+1)/2, out.nvarys))
def test_emcee_PT_output(self):
# test mcmc output when using parallel tempering
if not HAS_EMCEE:
return True
try:
from pandas import DataFrame
except ImportError:
return True
out = self.mini.emcee(ntemps=6, nwalkers=10, steps=20, burn=5, thin=2)
assert_(isinstance(out, MinimizerResult))
assert_(isinstance(out.flatchain, DataFrame))
# check that we can access the chains via parameter name
assert_(out.flatchain['amp'].shape[0] == 80)
assert out.errorbars
assert_(np.isfinite(out.params['amp'].correl['period']))
# the lnprob array should be the same as the chain size
assert_(np.size(out.chain)//out.nvarys == np.size(out.lnprob))
# test chain output shapes
assert_(out.lnprob.shape == (6, 10, (20-5+1)/2))
assert_(out.chain.shape == (6, 10, (20-5+1)/2, out.nvarys))
# Only the 0th temperature is returned
assert_(out.flatchain.shape == (10*(20-5+1)/2, out.nvarys))
@dec.slow
def test_emcee_float(self):
# test that it works if the residuals returns a float, not a vector
if not HAS_EMCEE:
return True
def resid(pars, x, data=None):
return -0.5 * np.sum(self.residual(pars, x, data=data)**2)
# just return chi2
def resid2(pars, x, data=None):
return np.sum(self.residual(pars, x, data=data)**2)
self.mini.userfcn = resid
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200, burn=50, thin=10)
check_paras(out.params, self.p_true, sig=3)
self.mini.userfcn = resid2
np.random.seed(123456)
out = self.mini.emcee(nwalkers=100, steps=200,
burn=50, thin=10, float_behavior='chi2')
check_paras(out.params, self.p_true, sig=3)
@dec.slow
def test_emcee_seed(self):
# test emcee seeding can reproduce a sampling run
if not HAS_EMCEE:
return True
out = self.mini.emcee(params=self.fit_params,
nwalkers=100,
steps=1, seed=1)
out2 = self.mini.emcee(params=self.fit_params,
nwalkers=100,
steps=1, seed=1)
assert_almost_equal(out.chain, out2.chain)
def check_redshift(fobs,
eobs,
xobs,
fm_tmp,
xm_tmp,
zbest,
dez,
prior,
NR,
zliml,
zlimu,
delzz=0.01,
nmc_cz=100,
nwalk_cz=10):
fit_par_cz = Parameters()
fit_par_cz.add('z', value=zbest, min=zliml, max=zlimu)
fit_par_cz.add('Cz0', value=1, min=0.5, max=1.5)
fit_par_cz.add('Cz1', value=1, min=0.5, max=1.5)
##############################
def residual_z(pars):
vals = pars.valuesdict()
z = vals['z']
Cz0s = vals['Cz0']
Cz1s = vals['Cz1']
xm_s = xm_tmp * (1 + z)
fm_s = np.interp(xobs, xm_s, fm_tmp)
con0 = (NR < 1000)
fy0 = fobs[con0] * Cz0s
ey0 = eobs[con0] * Cz0s
con1 = (NR >= 1000) & (NR < 10000)
fy1 = fobs[con1] * Cz1s
ey1 = eobs[con1] * Cz1s
con2 = (NR >= 10000) # BB
fy2 = fobs[con2]
ey2 = eobs[con2]
fy01 = np.append(fy0, fy1)
fcon = np.append(fy01, fy2)
ey01 = np.append(ey0, ey1)
eycon = np.append(ey01, ey2)
wht = 1. / np.square(eycon)
wht2, ypoly = check_line_cz_man(fcon, xobs, wht, fm_s, z)
if fobs is None:
print('Data is none')
return fm_s
else:
return (fm_s - fcon) * np.sqrt(wht2) # i.e. residual/sigma
###############################
def lnprob_cz(pars):
resid = residual_z(pars) # i.e. (data - model) * wht
z = pars['z']
s_z = 1 #pars['f_cz']
resid *= 1 / s_z
resid *= resid
nzz = int(z / delzz)
if nzz < 0 or z < zliml:
return -np.inf
else:
respr = np.log(prior[nzz])
resid += np.log(2 * np.pi * s_z**2) + respr
return -0.5 * np.sum(resid)
#################################
out_cz = minimize(residual_z, fit_par_cz, method='nelder')
#
# Best fit
#
keys = fit_report(out_cz).split('\n')
for key in keys:
if key[4:7] == 'chi':
skey = key.split(' ')
csq = float(skey[14])
if key[4:7] == 'red':
skey = key.split(' ')
rcsq = float(skey[7])
fitc_cz = [csq, rcsq] # Chi2, Reduced-chi2
zrecom = out_cz.params['z'].value
Czrec0 = out_cz.params['Cz0'].value
Czrec1 = out_cz.params['Cz1'].value
mini_cz = Minimizer(lnprob_cz, out_cz.params)
res_cz = mini_cz.emcee(burn=int(nmc_cz / 2),
steps=nmc_cz,
thin=10,
nwalkers=nwalk_cz,
params=out_cz.params,
is_weighted=True)
return res_cz, fitc_cz
