def mcmc_main(email, datasources, extrasources, flg_model=0, flg_plot=0): ''' flg_model = Flag controlling the f(N) model fitted 0: JXP spline 1: Inoue+14 functional form ''' import argparse # PARSE #parser = argparse.ArgumentParser(description='MCMC for f(N)') #parser.add_argument("model", type=int, help="Model flag (0=JXP, 1=Inoue+14)") #parser.add_argument("--plot", help="Plot the final model", action="store_true") #parser.add_argument("-id_dir", type=str, # help="Directory for ID files (ID_LINES is default)") #args = parser.parse_args() #flg_model = args.model #if args.plot: #flg_plot = 1 # ########################## # Set Data fN_data = set_fn_data(datasources, extrasources) # Set f(N) functional form fN_model = set_fn_model(flg=flg_model) # Set variables parm = set_pymc_var(fN_model) # Check plot if flg_plot: xifd.tst_fn_data(fN_model=fN_model) # Run MC = run(fN_data, fN_model, parm, email) # Save files save_figures(MC, email, fN_model)
plt.clf() plt.scatter(p13_data['LGN'], p13_data['FN']) #plt.plot(p13_data['LGN'],p13_data['FN'], '-') xplt = np.linspace(12., 22, 10000) yplt = fN_model.eval(xplt, 2.4) plt.plot(xplt, yplt, '-', color='red') #plt.plot(xplt, yplt, '-') plt.show() #xdb.set_trace() # Reproduce the main figure from P14 # Plot with Data if (flg_test % 2**3) >= 2**2: fN_model = xifm.default_model() fN_data.tst_fn_data(fN_model=fN_model) # Check l(X) if (flg_test % 16) >= 8: fN_model = xifm.default_model() lX = fN_model.calc_lox(2.4, 17.19 + np.log10(2.), 23.) print('l(X) = %g' % lX) # Check teff_LL if (flg_test % 32) >= 16: fN_model = xifm.default_model() zval, teff_LL = fN_model.teff_ll(0.5, 2.45) xdb.xplot(zval, teff_LL) #,xlabel='z', ylabel=r'$\tau_{\rm LL}$') # Check MFP if (flg_test % 64) >= 32:
plt.clf() plt.scatter(p13_data['LGN'],p13_data['FN']) #plt.plot(p13_data['LGN'],p13_data['FN'], '-') xplt = np.linspace(12., 22, 10000) yplt = fN_model.eval(xplt, 2.4) plt.plot(xplt, yplt, '-', color='red') #plt.plot(xplt, yplt, '-') plt.show() #xdb.set_trace() # Reproduce the main figure from P14 # Plot with Data if (flg_test % 2**3) >= 2**2: fN_model = xifm.default_model() fN_data.tst_fn_data(fN_model=fN_model) # Check l(X) if (flg_test % 16) >= 8: fN_model = xifm.default_model() lX = fN_model.calc_lox(2.4, 17.19+np.log10(2.), 23.) print('l(X) = %g' % lX) # Check teff_LL if (flg_test % 32) >= 16: fN_model = xifm.default_model() zval,teff_LL = fN_model.teff_ll(0.5, 2.45) xdb.xplot(zval,teff_LL)#,xlabel='z', ylabel=r'$\tau_{\rm LL}$') # Check MFP if (flg_test % 64) >= 32:
def run(fN_cs, fN_model, parm, email, debug=0): # pymc_list = [parm] # Parse data and combine as warranted all_NHI = [] all_fN = [] all_sigfN = [] all_z = [] flg_teff = 0 flg_LLS = 0 for fN_c in fN_cs: # Standard f(N) if fN_c.fN_dtype == 'fN': ip = range(fN_c.data['NPT']) val = np.where( fN_c.data['FN'][ip] > -90)[0] # Deal with limits later ipv = np.array(ip)[val] # Append the NHI NHI = np.median(fN_c.data['BINS'][:, ipv], 0) all_NHI += list(NHI) # Append the f(N) all_fN += list(fN_c.data['FN'][ipv]) # Append the Error fNerror = np.median(fN_c.data['SIG_FN'][:, ipv], 0) all_sigfN += list(fNerror) # Append zeval for ii in range(len(ipv)): all_z.append(fN_c.zeval) elif fN_c.fN_dtype == 'teff': # teff_Lya if flg_teff: raise ValueError('Only one teff allowed for now!') else: flg_teff = 1 teff = float(fN_c.data['TEFF']) D_A = 1. - np.exp(-1. * teff) SIGDA_LIMIT = 0.1 # Allows for systemtics and b-value uncertainty sig_teff = np.max([fN_c.data['SIG_TEFF'], (SIGDA_LIMIT * teff)]) teff_zeval = float(fN_c.data['Z_TEFF']) # Save input for later usage teff_input = (teff_zeval, fN_c.data['NHI_MNX'][0], fN_c.data['NHI_MNX'][1]) elif fN_c.fN_dtype == 'l(X)': # teff_Lya if flg_LLS: raise ValueError('Only one teff allowed for now!') else: flg_LLS = 1 LLS_lx = fN_c.data['LX'] LLS_siglx = fN_c.data['SIG_LX'] LLS_input = (fN_c.zeval, fN_c.data['TAU_LIM']) # fN_input = (np.array(all_NHI), np.array(all_z)) #flg_teff = 0 ####################################### # Generate the Models ####################################### # Define f(N) model for PyMC @pymc.deterministic(plot=False) def pymc_fn_model(parm=parm): # Set parameters fN_model.upd_param(parm) # log_fNX = fN_model.eval(fN_input, 0.) # return log_fNX pymc_list.append(pymc_fn_model) # Define teff model for PyMC if flg_teff: @pymc.deterministic(plot=False) def pymc_teff_model(parm=parm): # Set parameters fN_model.upd_param(parm) # Calculate teff model_teff = tau_eff.ew_teff_lyman(1215.6701 * (1 + teff_input[0]), teff_input[0] + 0.1, fN_model, NHI_MIN=teff_input[1], NHI_MAX=teff_input[2]) return model_teff pymc_list.append(pymc_teff_model) # Define l(X)_LLS model for PyMC if flg_LLS: @pymc.deterministic(plot=False) def pymc_lls_model(parm=parm): # Set parameters fN_model.upd_param(parm) # Calculate l(X) lX = fN_model.calc_lox(LLS_input[0], 17.19 + np.log10(LLS_input[1]), 22.) return lX pymc_list.append(pymc_lls_model) ####################################### # Generate the Data ####################################### # Define f(N) data for PyMC fNvalue = np.array(all_fN) #xdb.set_trace() pymc_fN_data = pymc.Normal(str('fNdata'), mu=pymc_fn_model, tau=1.0 / np.array(all_sigfN)**2, value=fNvalue, observed=True) pymc_list.append(pymc_fN_data) # Define teff data for PyMC if flg_teff: pymc_teff_data = pymc.Normal(str('teffdata'), mu=pymc_teff_model, tau=1.0 / np.array(sig_teff)**2, value=teff, observed=True) pymc_list.append(pymc_teff_data) # Define l(X)_LLS model for PyMC if flg_LLS: pymc_lls_data = pymc.Normal(str('LLSdata'), mu=pymc_lls_model, tau=1.0 / np.array(LLS_siglx)**2, value=LLS_lx, observed=True) pymc_list.append(pymc_lls_data) ####################################### # RUN THE MCMC ####################################### MC = pymc.MCMC(pymc_list) #,verbose=2) # Force step method to be Metropolis! for ss in MC.stochastics - MC.observed_stochastics: MC.use_step_method(pymc.Metropolis, ss, proposal_sd=0.025, proposal_distribution='Normal') #xdb.set_trace() # Run a total of 40000 samples, but ignore the first 10000. # Verbose just prints some details to screen. #xdb.set_trace() #MC.sample(20000, 3000, verbose=2, tune_interval=500) #MC.sample(5000, 500, verbose=2, tune_interval=200) #MC.sample(20000, 5000, verbose=2, tune_interval=500) MC.sample(2000, 400, verbose=2, tune_interval=200) #MC.isample(10000, 1000, verbose=2) if debug: xifd.tst_fn_data(fN_model=fN_model) xdb.xhist(MC.trace(str('p0'))[:]) xdb.set_trace() # Draw a contour plot with 1 & 2 sigma errors #MCMC_errors.draw_contours(MC, 'p0', 'p1') # Save the individual distributions to a file to check convergence #pymc.Matplot.plot(MC) #xdb.set_trace() return MC
def run(fN_cs, fN_model, parm, email, debug=0): # pymc_list = [parm] # Parse data and combine as warranted all_NHI = [] all_fN = [] all_sigfN = [] all_z = [] flg_teff = 0 flg_LLS = 0 for fN_c in fN_cs: # Standard f(N) if fN_c.fN_dtype == 'fN': ip = range(fN_c.data['NPT']) val = np.where(fN_c.data['FN'][ip] > -90)[0] # Deal with limits later ipv = np.array(ip)[val] # Append the NHI NHI = np.median(fN_c.data['BINS'][:,ipv],0) all_NHI += list(NHI) # Append the f(N) all_fN += list(fN_c.data['FN'][ipv]) # Append the Error fNerror = np.median(fN_c.data['SIG_FN'][:,ipv],0) all_sigfN += list(fNerror) # Append zeval for ii in range(len(ipv)): all_z.append(fN_c.zeval) elif fN_c.fN_dtype == 'teff': # teff_Lya if flg_teff: raise ValueError('Only one teff allowed for now!') else: flg_teff = 1 teff=float(fN_c.data['TEFF']) D_A = 1. - np.exp(-1. * teff) SIGDA_LIMIT = 0.1 # Allows for systemtics and b-value uncertainty sig_teff = np.max([fN_c.data['SIG_TEFF'], (SIGDA_LIMIT*teff)]) teff_zeval = float(fN_c.data['Z_TEFF']) # Save input for later usage teff_input = (teff_zeval, fN_c.data['NHI_MNX'][0], fN_c.data['NHI_MNX'][1]) elif fN_c.fN_dtype == 'l(X)': # teff_Lya if flg_LLS: raise ValueError('Only one teff allowed for now!') else: flg_LLS = 1 LLS_lx = fN_c.data['LX'] LLS_siglx = fN_c.data['SIG_LX'] LLS_input = (fN_c.zeval, fN_c.data['TAU_LIM']) # fN_input = (np.array(all_NHI), np.array(all_z)) #flg_teff = 0 ####################################### # Generate the Models ####################################### # Define f(N) model for PyMC @pymc.deterministic(plot=False) def pymc_fn_model(parm=parm): # Set parameters fN_model.param = parm fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param) # log_fNX = fN_model.eval( fN_input, 0. ) # return log_fNX pymc_list.append(pymc_fn_model) # Define teff model for PyMC if flg_teff: @pymc.deterministic(plot=False) def pymc_teff_model(parm=parm): # Set parameters fN_model.param = parm fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param) # Calculate teff model_teff = tau_eff.ew_teff_lyman(1215.6701*(1+teff_input[0]), teff_input[0]+0.1, fN_model, NHI_MIN=teff_input[1], NHI_MAX=teff_input[2]) return model_teff pymc_list.append(pymc_teff_model) # Define l(X)_LLS model for PyMC if flg_LLS: @pymc.deterministic(plot=False) def pymc_lls_model(parm=parm): # Set parameters fN_model.param = parm fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param) # Calculate l(X) lX = fN_model.calc_lox(LLS_input[0], 17.19+np.log10(LLS_input[1]), 22.) return lX pymc_list.append(pymc_lls_model) ####################################### # Generate the Data ####################################### # Define f(N) data for PyMC fNvalue=np.array(all_fN) pymc_fN_data = pymc.Normal(str('fNdata'), mu=pymc_fn_model, tau=1.0/np.array(all_sigfN)**2, value=fNvalue, observed=True) pymc_list.append(pymc_fN_data) # Define teff data for PyMC if flg_teff: pymc_teff_data = pymc.Normal(str('teffdata'), mu=pymc_teff_model, tau=1.0/np.array(sig_teff)**2, value=teff, observed=True) pymc_list.append(pymc_teff_data) # Define l(X)_LLS model for PyMC if flg_LLS: pymc_lls_data = pymc.Normal(str('LLSdata'), mu=pymc_lls_model, tau=1.0/np.array(LLS_siglx)**2, value=LLS_lx, observed=True) pymc_list.append(pymc_lls_data) ####################################### # RUN THE MCMC ####################################### MC = pymc.MCMC(pymc_list)#,verbose=2) # Force step method to be Metropolis! for ss in MC.stochastics-MC.observed_stochastics: MC.use_step_method(pymc.Metropolis, ss, proposal_sd=0.025, proposal_distribution='Normal') #xdb.set_trace() # Run a total of 40000 samples, but ignore the first 10000. # Verbose just prints some details to screen. #xdb.set_trace() #MC.sample(20000, 3000, verbose=2, tune_interval=500) MC.sample(2000, 300, verbose=2, tune_interval=200) #MC.isample(10000, 1000, verbose=2) ####################################### # PRINT THE RESULTS ####################################### t = strftime("%Y-%m-%d %H:%M:%S", gmtime()) f = open(email + t + 'asciifile', 'w+') # Print the best values and their errors best_pval = print_errors(MC) f.write(best_pval) fN_model.param = best_pval fN_model.model = scii.PchipInterpolator(fN_model.pivots, fN_model.param) if debug: xifd.tst_fn_data(fN_model=fN_model) xdb.xhist(MC.trace(str('p0'))[:]) xdb.set_trace()
teststr, pval, perr1[0], perr1[1], perr2[0], perr2[1])) all_pval.append(pval) #ival += 1 return all_pval ##### if __name__ == '__main__': # Set Data fN_data = set_fn_data() # Set f(N) functional form fN_model = set_fn_model() # Set variables parm = set_pymc_var(fN_model) fN_model.param = np.array([iparm.value for iparm in parm]) # Check plot if False: xifd.tst_fn_data(fN_model=fN_model) # Run #xdb.set_trace() run(fN_data, fN_model, parm) # Set model #xdb.set_trace() print('mcmc: All done')