def mcmc_main(email, datasources, extrasources, flg_model=0, flg_plot=0):
'''
flg_model = Flag controlling the f(N) model fitted
0: JXP spline
1: Inoue+14 functional form
'''
import argparse
# PARSE
#parser = argparse.ArgumentParser(description='MCMC for f(N)')
#parser.add_argument("model", type=int, help="Model flag (0=JXP, 1=Inoue+14)")
#parser.add_argument("--plot", help="Plot the final model", action="store_true")
#parser.add_argument("-id_dir", type=str,
# help="Directory for ID files (ID_LINES is default)")
#args = parser.parse_args()
#flg_model = args.model
#if args.plot:
#flg_plot = 1
# ##########################
# Set Data
fN_data = set_fn_data(datasources, extrasources)
# Set f(N) functional form
fN_model = set_fn_model(flg=flg_model)
# Set variables
parm = set_pymc_var(fN_model)
# Check plot
if flg_plot:
xifd.tst_fn_data(fN_model=fN_model)
# Run
MC = run(fN_data, fN_model, parm, email)
# Save files
save_figures(MC, email, fN_model)
def mcmc_main(email, datasources, extrasources, flg_model=0, flg_plot=0):
'''
flg_model = Flag controlling the f(N) model fitted
0: JXP spline
1: Inoue+14 functional form
'''
import argparse
# PARSE
#parser = argparse.ArgumentParser(description='MCMC for f(N)')
#parser.add_argument("model", type=int, help="Model flag (0=JXP, 1=Inoue+14)")
#parser.add_argument("--plot", help="Plot the final model", action="store_true")
#parser.add_argument("-id_dir", type=str,
# help="Directory for ID files (ID_LINES is default)")
#args = parser.parse_args()
#flg_model = args.model
#if args.plot:
#flg_plot = 1
# ##########################
# Set Data
fN_data = set_fn_data(datasources, extrasources)
# Set f(N) functional form
fN_model = set_fn_model(flg=flg_model)
# Set variables
parm = set_pymc_var(fN_model)
# Check plot
if flg_plot:
xifd.tst_fn_data(fN_model=fN_model)
# Run
MC = run(fN_data, fN_model, parm, email)
# Save files
save_figures(MC, email, fN_model)
plt.clf()
plt.scatter(p13_data['LGN'], p13_data['FN'])
#plt.plot(p13_data['LGN'],p13_data['FN'], '-')
xplt = np.linspace(12., 22, 10000)
yplt = fN_model.eval(xplt, 2.4)
plt.plot(xplt, yplt, '-', color='red')
#plt.plot(xplt, yplt, '-')
plt.show()
#xdb.set_trace()
# Reproduce the main figure from P14
# Plot with Data
if (flg_test % 2**3) >= 2**2:
fN_model = xifm.default_model()
fN_data.tst_fn_data(fN_model=fN_model)
# Check l(X)
if (flg_test % 16) >= 8:
fN_model = xifm.default_model()
lX = fN_model.calc_lox(2.4, 17.19 + np.log10(2.), 23.)
print('l(X) = %g' % lX)
# Check teff_LL
if (flg_test % 32) >= 16:
fN_model = xifm.default_model()
zval, teff_LL = fN_model.teff_ll(0.5, 2.45)
xdb.xplot(zval, teff_LL) #,xlabel='z', ylabel=r'$\tau_{\rm LL}$')
# Check MFP
if (flg_test % 64) >= 32:
plt.clf()
plt.scatter(p13_data['LGN'],p13_data['FN'])
#plt.plot(p13_data['LGN'],p13_data['FN'], '-')
xplt = np.linspace(12., 22, 10000)
yplt = fN_model.eval(xplt, 2.4)
plt.plot(xplt, yplt, '-', color='red')
#plt.plot(xplt, yplt, '-')
plt.show()
#xdb.set_trace()
# Reproduce the main figure from P14
# Plot with Data
if (flg_test % 2**3) >= 2**2:
fN_model = xifm.default_model()
fN_data.tst_fn_data(fN_model=fN_model)
# Check l(X)
if (flg_test % 16) >= 8:
fN_model = xifm.default_model()
lX = fN_model.calc_lox(2.4, 17.19+np.log10(2.), 23.)
print('l(X) = %g' % lX)
# Check teff_LL
if (flg_test % 32) >= 16:
fN_model = xifm.default_model()
zval,teff_LL = fN_model.teff_ll(0.5, 2.45)
xdb.xplot(zval,teff_LL)#,xlabel='z', ylabel=r'$\tau_{\rm LL}$')
# Check MFP
if (flg_test % 64) >= 32:
def run(fN_cs, fN_model, parm, email, debug=0):
#
pymc_list = [parm]
# Parse data and combine as warranted
all_NHI = []
all_fN = []
all_sigfN = []
all_z = []
flg_teff = 0
flg_LLS = 0
for fN_c in fN_cs:
# Standard f(N)
if fN_c.fN_dtype == 'fN':
ip = range(fN_c.data['NPT'])
val = np.where(
fN_c.data['FN'][ip] > -90)[0] # Deal with limits later
ipv = np.array(ip)[val]
# Append the NHI
NHI = np.median(fN_c.data['BINS'][:, ipv], 0)
all_NHI += list(NHI)
# Append the f(N)
all_fN += list(fN_c.data['FN'][ipv])
# Append the Error
fNerror = np.median(fN_c.data['SIG_FN'][:, ipv], 0)
all_sigfN += list(fNerror)
# Append zeval
for ii in range(len(ipv)):
all_z.append(fN_c.zeval)
elif fN_c.fN_dtype == 'teff': # teff_Lya
if flg_teff:
raise ValueError('Only one teff allowed for now!')
else:
flg_teff = 1
teff = float(fN_c.data['TEFF'])
D_A = 1. - np.exp(-1. * teff)
SIGDA_LIMIT = 0.1 # Allows for systemtics and b-value uncertainty
sig_teff = np.max([fN_c.data['SIG_TEFF'], (SIGDA_LIMIT * teff)])
teff_zeval = float(fN_c.data['Z_TEFF'])
# Save input for later usage
teff_input = (teff_zeval, fN_c.data['NHI_MNX'][0],
fN_c.data['NHI_MNX'][1])
elif fN_c.fN_dtype == 'l(X)': # teff_Lya
if flg_LLS:
raise ValueError('Only one teff allowed for now!')
else:
flg_LLS = 1
LLS_lx = fN_c.data['LX']
LLS_siglx = fN_c.data['SIG_LX']
LLS_input = (fN_c.zeval, fN_c.data['TAU_LIM'])
#
fN_input = (np.array(all_NHI), np.array(all_z))
#flg_teff = 0
#######################################
# Generate the Models
#######################################
# Define f(N) model for PyMC
@pymc.deterministic(plot=False)
def pymc_fn_model(parm=parm):
# Set parameters
fN_model.upd_param(parm)
#
log_fNX = fN_model.eval(fN_input, 0.)
#
return log_fNX
pymc_list.append(pymc_fn_model)
# Define teff model for PyMC
if flg_teff:
@pymc.deterministic(plot=False)
def pymc_teff_model(parm=parm):
# Set parameters
fN_model.upd_param(parm)
# Calculate teff
model_teff = tau_eff.ew_teff_lyman(1215.6701 * (1 + teff_input[0]),
teff_input[0] + 0.1,
fN_model,
NHI_MIN=teff_input[1],
NHI_MAX=teff_input[2])
return model_teff
pymc_list.append(pymc_teff_model)
# Define l(X)_LLS model for PyMC
if flg_LLS:
@pymc.deterministic(plot=False)
def pymc_lls_model(parm=parm):
# Set parameters
fN_model.upd_param(parm)
# Calculate l(X)
lX = fN_model.calc_lox(LLS_input[0],
17.19 + np.log10(LLS_input[1]), 22.)
return lX
pymc_list.append(pymc_lls_model)
#######################################
# Generate the Data
#######################################
# Define f(N) data for PyMC
fNvalue = np.array(all_fN)
#xdb.set_trace()
pymc_fN_data = pymc.Normal(str('fNdata'),
mu=pymc_fn_model,
tau=1.0 / np.array(all_sigfN)**2,
value=fNvalue,
observed=True)
pymc_list.append(pymc_fN_data)
# Define teff data for PyMC
if flg_teff:
pymc_teff_data = pymc.Normal(str('teffdata'),
mu=pymc_teff_model,
tau=1.0 / np.array(sig_teff)**2,
value=teff,
observed=True)
pymc_list.append(pymc_teff_data)
# Define l(X)_LLS model for PyMC
if flg_LLS:
pymc_lls_data = pymc.Normal(str('LLSdata'),
mu=pymc_lls_model,
tau=1.0 / np.array(LLS_siglx)**2,
value=LLS_lx,
observed=True)
pymc_list.append(pymc_lls_data)
#######################################
# RUN THE MCMC
#######################################
MC = pymc.MCMC(pymc_list) #,verbose=2)
# Force step method to be Metropolis!
for ss in MC.stochastics - MC.observed_stochastics:
MC.use_step_method(pymc.Metropolis,
ss,
proposal_sd=0.025,
proposal_distribution='Normal')
#xdb.set_trace()
# Run a total of 40000 samples, but ignore the first 10000.
# Verbose just prints some details to screen.
#xdb.set_trace()
#MC.sample(20000, 3000, verbose=2, tune_interval=500)
#MC.sample(5000, 500, verbose=2, tune_interval=200)
#MC.sample(20000, 5000, verbose=2, tune_interval=500)
MC.sample(2000, 400, verbose=2, tune_interval=200)
#MC.isample(10000, 1000, verbose=2)
if debug:
xifd.tst_fn_data(fN_model=fN_model)
xdb.xhist(MC.trace(str('p0'))[:])
xdb.set_trace()
# Draw a contour plot with 1 & 2 sigma errors
#MCMC_errors.draw_contours(MC, 'p0', 'p1')
# Save the individual distributions to a file to check convergence
#pymc.Matplot.plot(MC)
#xdb.set_trace()
return MC
def run(fN_cs, fN_model, parm, email, debug=0):
#
pymc_list = [parm]
# Parse data and combine as warranted
all_NHI = []
all_fN = []
all_sigfN = []
all_z = []
flg_teff = 0
flg_LLS = 0
for fN_c in fN_cs:
# Standard f(N)
if fN_c.fN_dtype == 'fN':
ip = range(fN_c.data['NPT'])
val = np.where(fN_c.data['FN'][ip] > -90)[0] # Deal with limits later
ipv = np.array(ip)[val]
# Append the NHI
NHI = np.median(fN_c.data['BINS'][:,ipv],0)
all_NHI += list(NHI)
# Append the f(N)
all_fN += list(fN_c.data['FN'][ipv])
# Append the Error
fNerror = np.median(fN_c.data['SIG_FN'][:,ipv],0)
all_sigfN += list(fNerror)
# Append zeval
for ii in range(len(ipv)):
all_z.append(fN_c.zeval)
elif fN_c.fN_dtype == 'teff': # teff_Lya
if flg_teff:
raise ValueError('Only one teff allowed for now!')
else:
flg_teff = 1
teff=float(fN_c.data['TEFF'])
D_A = 1. - np.exp(-1. * teff)
SIGDA_LIMIT = 0.1 # Allows for systemtics and b-value uncertainty
sig_teff = np.max([fN_c.data['SIG_TEFF'], (SIGDA_LIMIT*teff)])
teff_zeval = float(fN_c.data['Z_TEFF'])
# Save input for later usage
teff_input = (teff_zeval, fN_c.data['NHI_MNX'][0], fN_c.data['NHI_MNX'][1])
elif fN_c.fN_dtype == 'l(X)': # teff_Lya
if flg_LLS:
raise ValueError('Only one teff allowed for now!')
else:
flg_LLS = 1
LLS_lx = fN_c.data['LX']
LLS_siglx = fN_c.data['SIG_LX']
LLS_input = (fN_c.zeval, fN_c.data['TAU_LIM'])
#
fN_input = (np.array(all_NHI), np.array(all_z))
#flg_teff = 0
#######################################
# Generate the Models
#######################################
# Define f(N) model for PyMC
@pymc.deterministic(plot=False)
def pymc_fn_model(parm=parm):
# Set parameters
fN_model.param = parm
fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param)
#
log_fNX = fN_model.eval( fN_input, 0. )
#
return log_fNX
pymc_list.append(pymc_fn_model)
# Define teff model for PyMC
if flg_teff:
@pymc.deterministic(plot=False)
def pymc_teff_model(parm=parm):
# Set parameters
fN_model.param = parm
fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param)
# Calculate teff
model_teff = tau_eff.ew_teff_lyman(1215.6701*(1+teff_input[0]), teff_input[0]+0.1,
fN_model, NHI_MIN=teff_input[1], NHI_MAX=teff_input[2])
return model_teff
pymc_list.append(pymc_teff_model)
# Define l(X)_LLS model for PyMC
if flg_LLS:
@pymc.deterministic(plot=False)
def pymc_lls_model(parm=parm):
# Set parameters
fN_model.param = parm
fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param)
# Calculate l(X)
lX = fN_model.calc_lox(LLS_input[0],
17.19+np.log10(LLS_input[1]), 22.)
return lX
pymc_list.append(pymc_lls_model)
#######################################
# Generate the Data
#######################################
# Define f(N) data for PyMC
fNvalue=np.array(all_fN)
pymc_fN_data = pymc.Normal(str('fNdata'), mu=pymc_fn_model, tau=1.0/np.array(all_sigfN)**2,
value=fNvalue, observed=True)
pymc_list.append(pymc_fN_data)
# Define teff data for PyMC
if flg_teff:
pymc_teff_data = pymc.Normal(str('teffdata'), mu=pymc_teff_model, tau=1.0/np.array(sig_teff)**2,
value=teff, observed=True)
pymc_list.append(pymc_teff_data)
# Define l(X)_LLS model for PyMC
if flg_LLS:
pymc_lls_data = pymc.Normal(str('LLSdata'), mu=pymc_lls_model, tau=1.0/np.array(LLS_siglx)**2,
value=LLS_lx, observed=True)
pymc_list.append(pymc_lls_data)
#######################################
# RUN THE MCMC
#######################################
MC = pymc.MCMC(pymc_list)#,verbose=2)
# Force step method to be Metropolis!
for ss in MC.stochastics-MC.observed_stochastics:
MC.use_step_method(pymc.Metropolis, ss, proposal_sd=0.025, proposal_distribution='Normal')
#xdb.set_trace()
# Run a total of 40000 samples, but ignore the first 10000.
# Verbose just prints some details to screen.
#xdb.set_trace()
#MC.sample(20000, 3000, verbose=2, tune_interval=500)
MC.sample(2000, 300, verbose=2, tune_interval=200)
#MC.isample(10000, 1000, verbose=2)
#######################################
# PRINT THE RESULTS
#######################################
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
f = open(email + t + 'asciifile', 'w+')
# Print the best values and their errors
best_pval = print_errors(MC)
f.write(best_pval)
fN_model.param = best_pval
fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param)
if debug:
xifd.tst_fn_data(fN_model=fN_model)
xdb.xhist(MC.trace(str('p0'))[:])
xdb.set_trace()
teststr, pval, perr1[0], perr1[1], perr2[0], perr2[1]))
all_pval.append(pval)
#ival += 1
return all_pval
#####
if __name__ == '__main__':
# Set Data
fN_data = set_fn_data()
# Set f(N) functional form
fN_model = set_fn_model()
# Set variables
parm = set_pymc_var(fN_model)
fN_model.param = np.array([iparm.value for iparm in parm])
# Check plot
if False:
xifd.tst_fn_data(fN_model=fN_model)
# Run
#xdb.set_trace()
run(fN_data, fN_model, parm)
# Set model
#xdb.set_trace()
print('mcmc: All done')