def write_h5(self, savefile, warning=True):
"""
Save the fitting results and every fitter inputs (apart from the 'sps') in a given file.
Parameters
----------
savefile : [string]
File name in which to save the results.
Must be an hdf5 file (thus ending with ".h5").
If the file already exists, add the day and time before ".h5" in file name.
Options
-------
warnings : [bool]
If True, allow warnings to be printed.
Default is True.
Returns
-------
Void
"""
import h5py
import pickle
if os.path.isfile(savefile):
_savefile, savefile = savefile, savefile.replace(".h5", f"_{io.get_now()}.h5")
if warning:
warn(f"The file {_savefile} is already existing. Writing results in: {savefile}")
from prospect.io import write_results as writer
writer.write_hdf5(hfile=savefile, run_params=self.run_params, model=self.model, obs=self.obs,
sampler=self.fit_output["sampling"][0], optimize_result_list=self.fit_output["optimization"][0],
tsample=self.fit_output["sampling"][1], toptimize=self.fit_output["optimization"][1])
with h5py.File(savefile, "a") as _h5f:
_h5f.create_dataset("model", data=np.void(pickle.dumps(self.model)))
_h5f.create_dataset("sps", data=np.void(pickle.dumps(self.sps)))
_h5f.flush()
model,
pool=pool,
hdf5=hfile,
**run_params)
esampler, burn_loc0, burn_prob0 = out
edur = time.time() - tstart
sys.stdout = fout
print('done emcee in {0}s'.format(edur))
write_results.write_hdf5(hfile,
run_params,
model,
obs,
esampler,
guesses,
toptimize=pdur,
tsample=edur,
sampling_initial_center=initial_center)
print('Finished. Saved to ' + str(hfilename))
time.sleep(3)
#Formatting Data for Pippi
pippi_format(hfilename)
print('Reformated to ' + str(hfilename[:-3]) + str('_pippi.hdf5'))
imax = np.argmax(esampler.lnprobability)
i, j = np.unravel_index(imax, esampler.lnprobability.shape)
theta_max = esampler.chain[i, j, :].copy()
args = parser.parse_args()
run_params = vars(args)
obs, model, sps, noise = build_all(**run_params)
run_params["sps_libraries"] = sps.ssp.libraries
run_params["param_file"] = __file__
print(model)
if args.debug:
sys.exit()
#hfile = setup_h5(model=model, obs=obs, **run_params)
hfile = "{0}_{1}_mcmc.h5".format(args.outfile, int(time.time()))
output = fit_model(obs, model, sps, noise, **run_params)
writer.write_hdf5(hfile,
run_params,
model,
obs,
output["sampling"][0],
output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1],
sps=sps)
try:
hfile.close()
except (AttributeError):
pass
def main(field, galaxy_seq):
#vers = (np.__version__, scipy.__version__, h5py.__version__, fsps.__version__, prospect.__version__)
#print("Numpy: {}\nScipy: {}\nH5PY: {}\nFSPS: {}\nProspect: {}".format(*vers))
# -------------- Decide field and filters
# Read in catalog from Lou
if 'North' in field:
df = pandas.read_pickle(adap_dir + 'GOODS_North_SNeIa_host_phot.pkl')
all_filters = [
'LBC_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX', 'ACS_F775W_FLUX',
'ACS_F814W_FLUX', 'ACS_F850LP_FLUX', 'WFC3_F105W_FLUX',
'WFC3_F125W_FLUX', 'WFC3_F140W_FLUX', 'WFC3_F160W_FLUX',
'MOIRCS_K_FLUX', 'CFHT_Ks_FLUX', 'IRAC_CH1_SCANDELS_FLUX',
'IRAC_CH2_SCANDELS_FLUX', 'IRAC_CH3_FLUX', 'IRAC_CH4_FLUX'
]
#all_filters = ['LBC_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX',
#'ACS_F775W_FLUX', 'ACS_F850LP_FLUX']
seq = np.array(df['ID'])
i = int(np.where(seq == galaxy_seq)[0])
elif 'South' in field:
df = pandas.read_pickle(adap_dir + 'GOODS_South_SNeIa_host_phot.pkl')
all_filters = [
'CTIO_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX',
'ACS_F775W_FLUX', 'ACS_F814W_FLUX', 'ACS_F850LP_FLUX',
'WFC3_F098M_FLUX', 'WFC3_F105W_FLUX', 'WFC3_F125W_FLUX',
'WFC3_F160W_FLUX', 'HAWKI_KS_FLUX', 'IRAC_CH1_FLUX',
'IRAC_CH2_FLUX', 'IRAC_CH3_FLUX', 'IRAC_CH4_FLUX'
]
#all_filters = ['CTIO_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX',
#'ACS_F775W_FLUX', 'ACS_F850LP_FLUX']
seq = np.array(df['Seq'])
i = int(np.where(seq == galaxy_seq)[0])
#print('Read in pickle with the following columns:')
#print(df.columns)
#print('Rows in DataFrame:', len(df)
print("Match index:", i, "for Seq:", galaxy_seq)
# -------------- Preliminary stuff
# Set up for emcee
nwalkers = 1000
niter = 500
ndim = 12
# Other set up
obj_ra = df['RA'][i]
obj_dec = df['DEC'][i]
obj_z = df['zbest'][i]
print("Object redshift:", obj_z)
age_at_z = astropy_cosmo.age(obj_z).value
print("Age of Universe at object redshift [Gyr]:", age_at_z)
# ------------- Get obs data
fluxes = []
fluxes_unc = []
useable_filters = []
for ft in range(len(all_filters)):
filter_name = all_filters[ft]
flux = df[filter_name][i]
fluxerr = df[filter_name + 'ERR'][i]
if np.isnan(flux):
continue
if flux <= 0.0:
continue
if (fluxerr < 0) or np.isnan(fluxerr):
fluxerr = 0.1 * flux
fluxes.append(flux)
fluxes_unc.append(fluxerr)
useable_filters.append(filter_name)
#print("\n")
#print(df.loc[i])
#print(fluxes, len(fluxes))
#print(useable_filters, len(useable_filters))
fluxes = np.array(fluxes)
fluxes_unc = np.array(fluxes_unc)
# Now build the prospector observation
obs = build_obs(fluxes, fluxes_unc, useable_filters)
# Set params for run
run_params = {}
run_params["object_redshift"] = obj_z
run_params["fixed_metallicity"] = None
run_params["add_duste"] = True
#run_params["dust_type"] = 4
run_params["zcontinuous"] = 1
# Generate the model SED at the initial value of theta
#theta = model.theta.copy()
#initial_spec, initial_phot, initial_mfrac = model.sed(theta, obs=obs, sps=sps)
verbose = True
run_params["verbose"] = verbose
model = build_model(**run_params)
print("\nInitial free parameter vector theta:\n {}\n".format(model.theta))
#print("Initial parameter dictionary:\n{}".format(model.params))
print("\n----------------------- Model details: -----------------------")
print(model)
print(
"----------------------- End model details. -----------------------\n")
# Here we will run all our building functions
obs = build_obs(fluxes, fluxes_unc, useable_filters)
sps = build_sps(**run_params)
#plot_data(obs)
#sys.exit(0)
# --- start fitting ----
# Set this to False if you don't want to do another optimization
# before emcee sampling (but note that the "optimization" entry
# in the output dictionary will be (None, 0.) in this case)
# If set to true then another round of optmization will be performed
# before sampling begins and the "optmization" entry of the output
# will be populated.
"""
run_params["optimize"] = False
run_params["min_method"] = 'lm'
# We'll start minimization from "nmin" separate places,
# the first based on the current values of each parameter and the
# rest drawn from the prior. Starting from these extra draws
# can guard against local minima, or problems caused by
# starting at the edge of a prior (e.g. dust2=0.0)
run_params["nmin"] = 5
run_params["emcee"] = True
run_params["dynesty"] = False
# Number of emcee walkers
run_params["nwalkers"] = nwalkers
# Number of iterations of the MCMC sampling
run_params["niter"] = niter
# Number of iterations in each round of burn-in
# After each round, the walkers are reinitialized based on the
# locations of the highest probablity half of the walkers.
run_params["nburn"] = [8, 16, 32, 64]
run_params["progress"] = True
hfile = adap_dir + "emcee_" + field + "_" + str(galaxy_seq) + ".h5"
if not os.path.isfile(hfile):
print("Now running with Emcee.")
output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params)
print('done emcee in {0}s'.format(output["sampling"][1]))
writer.write_hdf5(hfile, run_params, model, obs,
output["sampling"][0], output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
print('Finished with Seq: ' + str(galaxy_seq))
"""
hfile = adap_dir + "dynesty_" + field + "_" + str(galaxy_seq) + ".h5"
if not os.path.isfile(hfile):
print("Now running with Dynesty.")
run_params["emcee"] = False
run_params["dynesty"] = True
run_params["nested_method"] = "rwalk"
run_params["nlive_init"] = 400
run_params["nlive_batch"] = 200
run_params["nested_dlogz_init"] = 0.05
run_params["nested_posterior_thresh"] = 0.05
run_params["nested_maxcall"] = int(1e6)
#from multiprocessing import Pool
#with Pool(6) as pool:
# run_params["pool"] = pool
output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params)
print('done dynesty in {0}s'.format(output["sampling"][1]))
writer.write_hdf5(hfile,
run_params,
model,
obs,
output["sampling"][0],
output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
print('Finished with Seq: ' + str(galaxy_seq))
# -------------------------
# Visualizing results
results_type = "dynesty" # "emcee" | "dynesty"
# grab results (dictionary), the obs dictionary, and our corresponding models
# When using parameter files set `dangerous=True`
#result, obs, _ = reader.results_from("{}_" + str(galaxy_seq) + \
# ".h5".format(results_type), dangerous=False)
result, obs, _ = reader.results_from(adap_dir + results_type + "_" + \
field + "_" + str(galaxy_seq) + ".h5", dangerous=False)
#The following commented lines reconstruct the model and sps object,
# if a parameter file continaing the `build_*` methods was saved along with the results
#model = reader.get_model(result)
#sps = reader.get_sps(result)
# let's look at what's stored in the `result` dictionary
print(result.keys())
parnames = np.array(result['theta_labels'])
print('Parameters in this model:', parnames)
"""
if results_type == "emcee":
chosen = np.random.choice(result["run_params"]["nwalkers"], size=150, replace=False)
tracefig = reader.traceplot(result, figsize=(10,6), chains=chosen)
tracefig.savefig(adap_dir + 'trace_' + field + '_' + str(galaxy_seq) + '.pdf',
dpi=200, bbox_inches='tight')
else:
tracefig = reader.traceplot(result, figsize=(10,6))
tracefig.savefig(adap_dir + 'trace_' + field + '_' + str(galaxy_seq) + '.pdf',
dpi=200, bbox_inches='tight')
"""
# Get chain for corner plot
if results_type == 'emcee':
trace = result['chain']
thin = 5
trace = trace[:, ::thin, :]
samples = trace.reshape(trace.shape[0] * trace.shape[1],
trace.shape[2])
else:
samples = result['chain']
# Plot SFH
#print(model) # to copy paste agebins from print out
nagebins = 6
agebins = np.array([[0., 8.], [8., 8.47712125], [8.47712125, 9.],
[9., 9.47712125], [9.47712125, 9.77815125],
[9.77815125, 10.13353891]])
# Get the zfractions from corner quantiles
zf1 = corner.quantile(samples[:, 2], q=[0.16, 0.5, 0.84])
zf2 = corner.quantile(samples[:, 3], q=[0.16, 0.5, 0.84])
zf3 = corner.quantile(samples[:, 4], q=[0.16, 0.5, 0.84])
zf4 = corner.quantile(samples[:, 5], q=[0.16, 0.5, 0.84])
zf5 = corner.quantile(samples[:, 6], q=[0.16, 0.5, 0.84])
zf_arr = np.array([zf1[1], zf2[1], zf3[1], zf4[1], zf5[1]])
zf_arr_l = np.array([zf1[0], zf2[0], zf3[0], zf4[0], zf5[0]])
zf_arr_u = np.array([zf1[2], zf2[2], zf3[2], zf4[2], zf5[2]])
cq_mass = corner.quantile(samples[:, 7], q=[0.16, 0.5, 0.84])
print("Total mass:", "{:.3e}".format(cq_mass[1]), "+",
"{:.3e}".format(cq_mass[2] - cq_mass[1]), "-",
"{:.3e}".format(cq_mass[1] - cq_mass[0]))
# -----------
new_agebins = pt.zred_to_agebins(zred=obj_z, agebins=agebins)
print("New agebins:", new_agebins)
# -----------------------
# now convert to sfh and its errors
sfr = pt.zfrac_to_sfr(total_mass=cq_mass[1],
z_fraction=zf_arr,
agebins=new_agebins)
sfr_l = pt.zfrac_to_sfr(total_mass=cq_mass[1],
z_fraction=zf_arr_l,
agebins=new_agebins)
sfr_u = pt.zfrac_to_sfr(total_mass=cq_mass[1],
z_fraction=zf_arr_u,
agebins=new_agebins)
print("----------")
print("z fractions: ", zf_arr)
print("Lower z fractions:", zf_arr_l)
print("Upper z fractions:", zf_arr_u)
print("----------")
print("Inferred SFR: ", sfr)
print("Lower sfr vals:", sfr_l)
print("Upper sfr vals:", sfr_u)
#############
x_agebins = 10**new_agebins / 1e9
fig = plt.figure(figsize=(8, 4))
ax = fig.add_subplot(111)
ax.set_xlabel(r'$\mathrm{Time\, [Gyr];\, since\ galaxy\ formation}$',
fontsize=20)
ax.set_ylabel(r'$\mathrm{SFR\, [M_\odot/yr]}$', fontsize=20)
for a in range(len(agebins)):
ax.plot(x_agebins[a],
np.ones(len(x_agebins[a])) * sfr[a],
color='mediumblue',
lw=3.0)
# put in some poisson errors
sfr_err = np.ones(len(x_agebins[a])) * np.sqrt(sfr[a])
sfr_plt = np.ones(len(x_agebins[a])) * sfr[a]
sfr_low_fill = sfr_plt - sfr_err
sfr_up_fill = sfr_plt + sfr_err
ax.fill_between(x_agebins[a],
sfr_low_fill,
sfr_up_fill,
color='gray',
alpha=0.6)
#ax.set_ylim(np.min(sfr_low_fill) * 0.3, np.max(sfr_up_fill) * 1.1)
fig.savefig(adap_dir + 'sfh_' + field + '_' + str(galaxy_seq) + '.pdf',
dpi=200,
bbox_inches='tight')
sys.exit(0)
# Keep code block for future use if needed
"""
# combination of linear and log axis from
# https://stackoverflow.com/questions/21746491/combining-a-log-and-linear-scale-in-matplotlib
# linear part i.e., first age bin
ax.plot(x_agebins[0], np.ones(len(x_agebins[0])) * sfr[0], color='mediumblue', lw=3.5)
#ax.fill_between(x_agebins[0], np.ones(len(x_agebins[0])) * sfr_l[0],
# np.ones(len(x_agebins[0])) * sfr_u[0], color='gray', alpha=0.5)
ax.set_xlim(0.0, 8.0)
ax.spines['right'].set_visible(False)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_label_coords(x=1.2,y=-0.06)
# now log axis
divider = make_axes_locatable(ax)
axlog = divider.append_axes("right", size=3.0, pad=0, sharey=ax)
axlog.set_xscale('log')
for a in range(1, nagebins):
axlog.plot(x_agebins[a], np.ones(len(x_agebins[a])) * sfr[a], color='mediumblue', lw=3.5)
#axlog.fill_between(x_agebins[a], np.ones(len(x_agebins[a])) * sfr_l[a],
# np.ones(len(x_agebins[a])) * sfr_u[a], color='gray', alpha=0.5)
axlog.set_xlim(8.0, x_agebins[-1, -1] + 0.1)
axlog.spines['left'].set_visible(False)
axlog.yaxis.set_ticks_position('right')
axlog.tick_params(labelright=False)
axlog.xaxis.set_ticks(ticks=[8.0, 9.0])
axlog.xaxis.set_ticks(ticks=[8.2, 8.4, 8.6, 8.8, 9.2, 9.4, 9.6], minor=True)
axlog.set_xticklabels(['8', '9'])
axlog.set_xticklabels(labels=[], minor=True)
fig.savefig(adap_dir + 'sfh_' + field + '_' + str(galaxy_seq) + '.pdf',
dpi=200, bbox_inches='tight')
"""
# ---------- corner plot
# set up corner ranges and labels
math_parnames = [
r'$\mathrm{log(Z_\odot)}$', r'$\mathrm{dust2}$', r'$zf_1$', r'$zf_2$',
r'$zf_3$', r'$zf_4$', r'$zf_5$', r'$\mathrm{M_s}$',
r'$\mathrm{f_{agn}}$', r'$\mathrm{agn_\tau}$',
r'$\mathrm{dust_{ratio}}$', r'$\mathrm{dust_{index}}$'
]
#math_parnames = [r'$\mathrm{M_s}$', r'$\mathrm{log(Z_\odot)}$',
#r'$\mathrm{dust2}$', r'$\mathrm{t_{age}}$', r'$\mathrm{log(\tau)}$']
# Fix labels for corner plot and
# Figure out ranges for corner plot
corner_range = []
for d in range(ndim):
# Get corner estimate and errors
cq = corner.quantile(x=samples[:, d], q=[0.16, 0.5, 0.84])
low_err = cq[1] - cq[0]
up_err = cq[2] - cq[1]
# Decide the padding for the plot range
# depending on how large the error is relative
# to the central estimate.
if low_err * 2.5 >= cq[1]:
sigma_padding_low = 1.2
else:
sigma_padding_low = 3.0
if up_err * 2.5 >= cq[1]:
sigma_padding_up = 1.2
else:
sigma_padding_up = 3.0
low_lim = cq[1] - sigma_padding_low * low_err
up_lim = cq[1] + sigma_padding_up * up_err
corner_range.append((low_lim, up_lim))
# Print estimate to screen
if 'mass' in parnames[d]:
pn = '{:.3e}'.format(cq[1])
pnu = '{:.3e}'.format(up_err)
pnl = '{:.3e}'.format(low_err)
else:
pn = '{:.3f}'.format(cq[1])
pnu = '{:.3f}'.format(up_err)
pnl = '{:.3f}'.format(low_err)
print(parnames[d], ": ", pn, "+", pnu, "-", pnl)
# Corner plot
cornerfig = corner.corner(samples,
quantiles=[0.16, 0.5, 0.84],
labels=math_parnames,
label_kwargs={"fontsize": 14},
range=corner_range,
smooth=0.5,
smooth1d=0.5)
# loop over all axes *again* and set title
# because it won't let me set the
# format for soem titles separately
# Looping has to be done twice because corner
# plotting has to be done to get the figure.
corner_axes = np.array(cornerfig.axes).reshape((ndim, ndim))
for d in range(ndim):
# Get corner estimate and errors
cq = corner.quantile(x=samples[:, d], q=[0.16, 0.5, 0.84])
low_err = cq[1] - cq[0]
up_err = cq[2] - cq[1]
ax_c = corner_axes[d, d]
if 'mass' in parnames[d]:
ax_c.set_title(math_parnames[d] + r"$ \, =\,$" + csn(cq[1], sigfigs=3) + \
r"$\substack{+$" + csn(up_err, sigfigs=3) + r"$\\ -$" + \
csn(low_err, sigfigs=3) + r"$}$", fontsize=11, pad=15)
else:
ax_c.set_title(math_parnames[d] + r"$ \, =\,$" + r"${:.3f}$".format(cq[1]) + \
r"$\substack{+$" + r"${:.3f}$".format(up_err) + r"$\\ -$" + \
r"${:.3f}$".format(low_err) + r"$}$", fontsize=11, pad=10)
cornerfig.savefig(adap_dir + 'corner_' + field + '_' + \
str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight')
sys.exit(0)
# maximum a posteriori (of the locations visited by the MCMC sampler)
pmax = np.argmax(result['lnprobability'])
if results_type == "emcee":
p, q = np.unravel_index(pmax, result['lnprobability'].shape)
theta_max = result['chain'][p, q, :].copy()
else:
theta_max = result["chain"][pmax, :]
#print('Optimization value: {}'.format(theta_best))
#print('MAP value: {}'.format(theta_max))
# make SED plot for MAP model and some random model
# randomly chosen parameters from chain
randint = np.random.randint
if results_type == "emcee":
theta = result['chain'][randint(nwalkers), randint(niter)]
else:
theta = result["chain"][randint(len(result["chain"]))]
# generate models
a = 1.0 + model.params.get('zred', 0.0) # cosmological redshifting
# photometric effective wavelengths
wphot = obs["phot_wave"]
# spectroscopic wavelengths
if obs["wavelength"] is None:
# *restframe* spectral wavelengths, since obs["wavelength"] is None
wspec = sps.wavelengths
wspec *= a #redshift them
else:
wspec = obs["wavelength"]
# sps = reader.get_sps(result) # this works if using parameter files
mspec, mphot, mextra = model.mean_model(theta, obs, sps=sps)
mspec_map, mphot_map, _ = model.mean_model(theta_max, obs, sps=sps)
# establish bounds
xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8
ymin, ymax = obs["maggies"].min() * 0.8, obs["maggies"].max() / 0.4
# Make plot of data and model
fig3 = plt.figure(figsize=(9, 4))
ax3 = fig3.add_subplot(111)
ax3.set_xlabel(r'$\mathrm{\lambda\ [\AA]}$', fontsize=15)
#ax3.set_ylabel(r'$\mathrm{f_\lambda\ [erg\, s^{-1}\, cm^{-2}\, \AA^{-1}]}$', fontsize=15)
ax3.set_ylabel(r'$\mathrm{Flux\ Density\ [maggies]}$', fontsize=15)
ax3.loglog(wspec,
mspec,
label='Model spectrum (random draw)',
lw=0.7,
color='navy',
alpha=0.7)
ax3.loglog(wspec,
mspec_map,
label='Model spectrum (MAP)',
lw=0.7,
color='green',
alpha=0.7)
ax3.errorbar(wphot,
mphot,
label='Model photometry (random draw)',
marker='s',
markersize=10,
alpha=0.8,
ls='',
lw=3,
markerfacecolor='none',
markeredgecolor='blue',
markeredgewidth=3)
ax3.errorbar(wphot,
mphot_map,
label='Model photometry (MAP)',
marker='s',
markersize=10,
alpha=0.8,
ls='',
lw=3,
markerfacecolor='none',
markeredgecolor='green',
markeredgewidth=3)
ax3.errorbar(wphot,
obs['maggies'],
yerr=obs['maggies_unc'],
label='Observed photometry',
ecolor='red',
marker='o',
markersize=10,
ls='',
lw=3,
alpha=0.8,
markerfacecolor='none',
markeredgecolor='red',
markeredgewidth=3)
# plot transmission curves
for f in obs['filters']:
w, t = f.wavelength.copy(), f.transmission.copy()
t = t / t.max()
t = 10**(0.2 * (np.log10(ymax / ymin))) * t * ymin
ax3.loglog(w, t, lw=3, color='gray', alpha=0.7)
ax3.set_xlim([xmin, xmax])
ax3.set_ylim([ymin, ymax])
ax3.legend(loc='best', fontsize=11)
fig3.savefig(adap_dir + 'sedplot_' + field + '_' + str(galaxy_seq) +
'.pdf',
dpi=200,
bbox_inches='tight')
plt.clf()
plt.cla()
plt.close()
return None
# -------------------------
# Output HDF5 (and pickles if asked for)
# -------------------------
print("Writing to {}".format(outroot))
if rp.get("output_pickles", False):
write_results.write_pickles(rp,
model,
obsdat,
esampler,
guesses,
outroot=outroot,
toptimize=0,
tsample=edur,
sampling_initial_center=initial_center)
if hfile is None:
hfile = hfilename
write_results.write_hdf5(hfile,
rp,
model,
obsdat,
esampler,
guesses,
toptimize=0,
tsample=edur,
sampling_initial_center=initial_center)
try:
hfile.close()
except:
pass
halt('Finished')
tstart = time.time()
out = fitting.run_emcee_sampler(lnprobfn, initial_center, model,
postkwargs=postkwargs, initial_prob=initial_prob,
pool=pool, hdf5=hfile, **rp)
esampler, burn_p0, burn_prob0 = out
edur = time.time() - tstart
if rp['verbose']:
print('done emcee in {0}s'.format(edur))
# -------------------------
# Output pickles (and HDF5)
# -------------------------
print("Writing to {}".format(outroot))
write_results.write_pickles(rp, model, obsdat, esampler, guesses,
outroot=outroot, toptimize=pdur, tsample=edur,
sampling_initial_center=initial_center,
post_burnin_center=burn_p0,
post_burnin_prob=burn_prob0)
if hfile is None:
hfile = hfilename
write_results.write_hdf5(hfile, rp, model, obsdat, esampler, guesses,
toptimize=pdur, tsample=edur,
sampling_initial_center=initial_center,
post_burnin_center=burn_p0,
post_burnin_prob=burn_prob0)
try:
hfile.close()
except:
pass
halt('Finished')
def run_prospector(tde_name, path, z, withmpi, n_cores, gal_ebv, show_figs=True, init_theta=None, n_walkers=None,
n_inter=None, n_burn=None, read_only=False):
os.chdir(os.path.join(path, tde_name, 'host'))
if init_theta is None:
init_theta = [1e10, -1.0, 6, 0.5]
if n_walkers is None:
n_walkers = 100
if n_inter is None:
n_inter = 1000
if n_burn is None:
n_burn = [500]
obs, sps, model, run_params = configure(tde_name, path, z, init_theta, n_walkers, n_inter, n_burn, gal_ebv)
if not withmpi:
n_cores = 1
if not read_only:
print("Initial guess: {}".format(model.initial_theta))
print('Sampling the the SPS grid..')
if withmpi & ('logzsol' in model.free_params):
dummy_obs = dict(filters=None, wavelength=None)
logzsol_prior = model.config_dict["logzsol"]['prior']
lo, hi = logzsol_prior.range
logzsol_grid = np.around(np.arange(lo, hi, step=0.1), decimals=2)
sps.update(**model.params) # make sure we are caching the correct IMF / SFH / etc
for logzsol in logzsol_grid:
model.params["logzsol"] = np.array([logzsol])
_ = model.predict(model.theta, obs=dummy_obs, sps=sps)
print('Done')
from functools import partial
lnprobfn_fixed = partial(lnprobfn, sps=sps)
print('Starting posterior emcee sampling..')
if withmpi:
from multiprocessing import Pool
from multiprocessing import cpu_count
with Pool(int(n_cores)) as pool:
nprocs = n_cores
output = fit_model(obs, model, sps, pool=pool, queue_size=nprocs, lnprobfn=lnprobfn_fixed,
**run_params)
else:
output = fit_model(obs, model, sps, lnprobfn=lnprobfn_fixed, **run_params)
# output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params)
print('done emcee in {0}s'.format(output["sampling"][1]))
if os.path.exists("prospector_result.h5"):
os.system('rm prospector_result.h5')
hfile = "prospector_result.h5"
writer.write_hdf5(hfile, run_params, model, obs,
output["sampling"][0], output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
print('Finished')
# Loading results file
result, _, _ = reader.results_from("prospector_result.h5", dangerous=False)
# Finding the Maximum A Posteriori (MAP) model
imax = np.argmax(result['lnprobability'])
i, j = np.unravel_index(imax, result['lnprobability'].shape)
theta_max = result['chain'][i, j, :].copy()
# saving results
save_results(result, model, obs, sps, theta_max, tde_name, path, n_walkers, n_inter, n_burn, n_cores)
print('MAP value: {}'.format(theta_max))
fit_plot = plot_resulting_fit(tde_name, path)
try:
os.mkdir(os.path.join(path, tde_name, 'plots'))
except:
pass
fit_plot.savefig(os.path.join(path, tde_name, 'plots', tde_name + '_host_fit.png'), bbox_inches='tight', dpi=300)
if show_figs:
plt.show()
corner_fig = corner_plot(result)
corner_fig.savefig(os.path.join(path, tde_name, 'plots', tde_name + '_cornerplot.png'), bbox_inches='tight',
dpi=300)
if show_figs:
plt.show()
os.chdir(path)
if rp['verbose']:
print('done nestle in {0}s'.format(dur))
# -------------------------
# Output pickles (and HDF5)
# -------------------------
print("Writing to {}".format(outroot))
# Write the nestle Result object as a pickle
import pickle
with open(outroot + '_nmc.pkl', 'w') as f:
pickle.dump(nestleout, f)
partext = write_results.paramfile_string(**rp)
# Write the model as a pickle
write_results.write_model_pickle(outroot + '_model',
model,
powell=None,
paramfile_text=partext)
# Write HDF5
if hfile is None:
hfile = hfilename
write_results.write_hdf5(hfile,
rp,
model,
obsdat,
nestleout,
None,
tsample=dur)
print('Finished')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--prefix',
type=str,
default='redmapper_sdssphot',
help='String to prepend to I/O files.')
parser.add_argument('--nthreads',
type=int,
default=16,
help='Number of cores to use concurrently.')
parser.add_argument('--first',
type=int,
default=0,
help='Index of first object to fit.')
parser.add_argument('--last',
type=int,
default=None,
help='Index of last object to fit.')
parser.add_argument('--seed',
type=int,
default=1,
help='Random number seed.')
parser.add_argument('--build-sample',
action='store_true',
help='Build the sample.')
parser.add_argument('--dofit', action='store_true', help='Run prospector!')
parser.add_argument(
'--refit',
action='store_true',
help='Refit even if the prospector output files exist.')
parser.add_argument('--qaplots',
action='store_true',
help='Make some neat plots.')
parser.add_argument('--verbose',
action='store_true',
help='Be loquacious.')
args = parser.parse_args()
# Specify the run parameters and initialize the SPS object.
run_params = {
'prefix': args.prefix,
'verbose': args.verbose,
'seed': args.seed,
# initial optimization choices (nmin is only for L-M optimization)
'do_levenburg': True,
'do_powell': False,
'do_nelder_mead': False,
'nmin': np.max((10, args.nthreads)),
# emcee fitting parameters monte carlo markov chain samples parameters ft. certain technique
'nwalkers': 128,
'nburn': [32, 32, 64],
'niter': 256, # 512,
'interval': 0.1, # save 10% of the chains at a time
# Nestle fitting parameters
'nestle_method': 'single',
'nestle_npoints': 200,
'nestle_maxcall': int(1e6),
# Multiprocessing
'nthreads': args.nthreads,
# SPS initialization parameters
'compute_vega_mags': False,
'vactoair_flag': False, # use wavelengths in air
'zcontinuous': 1, # interpolate in metallicity
}
rand = np.random.RandomState(args.seed)
if not args.build_sample:
t0 = time()
print('Initializing CSPSpecBasis...')
sps = CSPSpecBasis(zcontinuous=run_params['zcontinuous'],
compute_vega_mags=run_params['compute_vega_mags'],
vactoair_flag=run_params['vactoair_flag'])
print('...took {:.1f} seconds.'.format(time() - t0))
if args.build_sample:
# READ OUR SHIT INSTEAD.
# Read the parent redmapper catalog, choose a subset of objects and
# write out.
cat = read_redmapper()
# Choose objects with masses from iSEDfit, Kravtsov, and pymorph, but
# for now just pick three random galaxies.
#these = np.arange(2) # [300, 301, 302]
these = np.arange(100) # [300, 301, 302]
print('Selecting {} galaxies.'.format(len(these)))
out = cat[these]
#out = cat[:200]
outfile = os.path.join(datadir(),
'{}_sample.fits'.format(run_params['prefix']))
print('Writing {}'.format(outfile))
fitsio.write(outfile, out, clobber=True)
if args.dofit:
import h5py
import emcee
from prospect import fitting
from prospect.io import write_results
# MAKE THIS SHIT WORK WITH OUR SHIT TOO ft getobs
# Read the parent sample and loop on each object.
cat = read_parent(
prefix=run_params['prefix']) #, first=args.first, last=args.last)
for ii, obj in enumerate(cat):
objprefix = '{0:05}'.format(obj['ISEDFIT_ID'])
print('Working on object {}/{} with prefix {}.'.format(
ii + 1, len(cat), objprefix))
# Check for the HDF5 output file / fitting results --
outroot = os.path.join(
datadir(), '{}_{}'.format(run_params['prefix'], objprefix))
hfilename = os.path.join(
datadir(), '{}_{}_mcmc.h5'.format(run_params['prefix'],
objprefix))
if os.path.isfile(hfilename):
if args.refit:
os.remove(hfilename)
else:
print('Prospector fitting results {} exist; skipping.'.
format(hfilename))
continue
# Grab the photometry for this object and then initialize the priors
# and the SED model.
obs = getobs(obj)
model = load_model(zred=obs['zred'], seed=args.seed)
# Get close to the right answer doing a simple minimization.
if run_params['verbose']:
print('Free parameters: {}'.format(model.free_params))
print('Initial parameter values: {}'.format(
model.initial_theta))
initial_theta = model.rectify_theta(
model.initial_theta) # make zeros tiny numbers
if bool(run_params.get('do_powell', True)):
tstart = time()
# optimization options
powell_opt = {
'ftol': run_params.get('ftol', 0.5e-5),
'xtol': 1e-6,
'maxfev': run_params.get('maxfev', 5000)
}
chi2args = (model, obs, sps, run_params['verbose']
) # extra arguments for chisqfn
guesses, pinit = fitting.pminimize(
chisqfn,
initial_theta,
args=chi2args,
model=model,
method='powell',
opts=powell_opt,
pool=pool,
nthreads=run_params['nthreads'])
best = np.argmin([p.fun for p in guesses])
# Hack to recenter values outside the parameter bounds!
initial_center = fitting.reinitialize(
guesses[best].x,
model,
edge_trunc=run_params.get('edge_trunc', 0.1))
initial_prob = -1 * guesses[best]['fun']
pdur = time() - tstart
if run_params['verbose']:
print('Powell initialization took {:.1f} seconds.'.format(
pdur))
print('Best Powell guesses: {}'.format(initial_center))
print('Initial probability: {}'.format(initial_prob))
elif bool(run_params.get('do_nelder_mead', True)):
from scipy.optimize import minimize
tstart = time()
chi2args = (model, obs, sps, run_params['verbose']
) # extra arguments for chisqfn
guesses = minimize(chisqfn,
initial_theta,
args=chi2args,
method='nelder-mead')
pdur = time() - tstart
# Hack to recenter values outside the parameter bounds!
initial_center = fitting.reinitialize(
guesses.x,
model,
edge_trunc=run_params.get('edge_trunc', 0.1))
initial_prob = -1 * guesses['fun']
if run_params['verbose']:
print('Nelder-Mead initialization took {:.1f} seconds.'.
format(pdur))
print('Best guesses: {}'.format(initial_center))
print('Initial probability: {}'.format(initial_prob))
elif bool(run_params.get('do_levenburg', True)):
tstart = time()
nmin = run_params['nmin']
chi2args = (model, obs, sps)
pinitial = fitting.minimizer_ball(initial_theta,
nmin,
model,
seed=run_params['seed'])
lsargs = list()
for pinit in pinitial:
lsargs.append((chivecfn, pinit, chi2args))
if run_params['nthreads'] > 1:
p = multiprocessing.Pool(run_params['nthreads'])
guesses = p.map(_doleast_squares, lsargs)
p.close()
else:
guesses = list()
for lsargs1 in lsargs:
guesses.append(_doleast_squares(lsargs1))
chisq = [np.sum(r.fun**2) for r in guesses]
best = np.argmin(chisq)
initial_prob = -np.log(chisq[best] / 2)
#initial_center = guesses[best].x
initial_center = fitting.reinitialize(
guesses[best].x,
model,
edge_trunc=run_params.get('edge_trunc', 0.1))
pdur = time() - tstart
if run_params['verbose']:
print(
'Levenburg-Marquardt initialization took {:.1f} seconds.'
.format(pdur))
print('Best guesses: {}'.format(initial_center))
print('Initial probability: {}'.format(initial_prob))
from prospector_plot_utilities import bestfit_sed
fig = bestfit_sed(obs,
theta=initial_center,
sps=sps,
model=model)
fig.savefig('test.png')
else:
if run_params['verbose']:
print('Skipping initial minimization.')
guesses = None
pdur = 0.0
initial_center = initial_theta.copy()
initial_prob = None
# Write some basic info to the HDF5 file--
hfile = h5py.File(hfilename, 'a')
write_results.write_h5_header(hfile, run_params, model)
write_results.write_obs_to_h5(hfile, obs)
if run_params['verbose']:
print('Started emcee sampling on {}'.format(asctime()))
tstart = time()
out = fitting.run_emcee_sampler(lnprobfn,
initial_center,
model,
verbose=run_params['verbose'],
nthreads=run_params['nthreads'],
nwalkers=run_params['nwalkers'],
nburn=run_params['nburn'],
niter=run_params['niter'],
prob0=initial_prob,
hdf5=hfile,
postargs=(model, obs, sps),
pool=pool)
esampler, burn_p0, burn_prob0 = out
del out
edur = time() - tstart
if run_params['verbose']:
print('Finished emcee sampling in {:.2f} minutes.'.format(
edur / 60.0))
# Update the HDF5 file with the results.
write_results.write_pickles(run_params,
model,
obs,
esampler,
guesses,
outroot=outroot,
toptimize=pdur,
tsample=edur,
sampling_initial_center=initial_center,
post_burnin_center=burn_p0,
post_burnin_prob=burn_prob0)
write_results.write_hdf5(hfilename,
run_params,
model,
obs,
esampler,
guesses,
toptimize=pdur,
tsample=edur,
sampling_initial_center=initial_center,
post_burnin_center=burn_p0,
post_burnin_prob=burn_prob0)
hfile.close()
if args.qaplots:
import h5py
from prospect.io import read_results
from prospector_plot_utilities import param_evol, subtriangle, bestfit_sed
# Read the parent sample and loop on each object.
cat = read_parent(
prefix=run_params['prefix']) #, first=args.first, last=args.last)
for obj in cat:
objprefix = '{0:05}'.format(obj['ISEDFIT_ID'])
# Grab the emcee / prospector outputs.
h5file = os.path.join(
datadir(), '{}_{}_mcmc.h5'.format(run_params['prefix'],
objprefix))
if not os.path.isfile(h5file):
print('HDF5 file {} not found; skipping.'.format(h5file))
continue
print('Reading {}'.format(h5file))
results, guesses, model = read_results.results_from(h5file)
nwalkers, niter, nparams = results['chain'][:, :, :].shape
# --------------------------------------------------
# Figure: Generate the best-fitting SED.
qafile = os.path.join(
datadir(), '{}_{}_sed.png'.format(args.prefix, objprefix))
print('Writing {}'.format(qafile))
fig = bestfit_sed(results['obs'],
chain=results['chain'],
lnprobability=results['lnprobability'],
sps=sps,
model=model,
seed=results['run_params']['seed'])
fig.savefig(qafile)
# --------------------------------------------------
# Figure: Visualize a random sampling of the MCMC chains.
qafile = os.path.join(
datadir(), '{}_{}_chains.png'.format(args.prefix, objprefix))
print('Writing {}'.format(qafile))
thesechains = rand.choice(nwalkers,
size=int(0.3 * nwalkers),
replace=False)
fig = param_evol(results, chains=thesechains)
fig.savefig(qafile)
# --------------------------------------------------
# Figure: Generate a corner/triangle plot of the free parameters.
qafile = os.path.join(
datadir(), '{}_{}_corner.png'.format(args.prefix, objprefix))
print('Writing {}'.format(qafile))
fig = subtriangle(results, thin=2)
fig.savefig(qafile)
# -----------------------
if rp['verbose']:
print('emcee sampling...')
tstart = time.time()
out = fitting.restart_emcee_sampler(lnprobfn, initial_positions,
postkwargs=postkwargs,
pool=pool, hdf5=hfile, **rp)
esampler = out
edur = time.time() - tstart
if rp['verbose']:
print('done emcee in {0}s'.format(edur))
# -------------------------
# Output HDF5 (and pickles if asked for)
# -------------------------
print("Writing to {}".format(outroot))
if rp.get("output_pickles", False):
write_results.write_pickles(rp, model, obsdat, esampler, guesses,
outroot=outroot, toptimize=0, tsample=edur,
sampling_initial_center=initial_center)
if hfile is None:
hfile = hfilename
write_results.write_hdf5(hfile, rp, model, obsdat, esampler, guesses,
toptimize=0, tsample=edur,
sampling_initial_center=initial_center)
try:
hfile.close()
except:
pass
halt('Finished')
hfile = None
# -------
# Sample
# -------
if rp['verbose']:
print('dynesty sampling...')
tstart = time.time() # time it
dynestyout = fitting.run_dynesty_sampler(lnprobfn, prior_transform, model.ndim,
pool=pool, queue_size=nprocs,
stop_function=stopping_function,
wt_function=weight_function,
**rp)
ndur = time.time() - tstart
print('done dynesty in {0}s'.format(ndur))
# -------------------------
# Output pickles (and HDF5)
# -------------------------
partext = write_results.paramfile_string(**rp)
# Write the model as a pickle
write_results.write_model_pickle(outroot + '_model', model, powell=None,
paramfile_text=partext)
# Write HDF5
if hfile is None:
hfile = hfilename
write_results.write_hdf5(hfile, rp, model, obs, dynestyout,
None, tsample=ndur)
def main():
vers = (np.__version__, scipy.__version__, h5py.__version__,
fsps.__version__, prospect.__version__)
print(
"Numpy: {}\nScipy: {}\nH5PY: {}\nFSPS: {}\nProspect: {}".format(*vers))
# And we will store some meta-parameters that control the input arguments to this method:
run_params = {}
run_params["snr"] = 10.0
run_params["ldist"] = 10.0
# Build the obs dictionary using the meta-parameters
obs = build_obs(**run_params)
# Look at the contents of the obs dictionary
print("Obs Dictionary Keys:\n\n{}\n".format(obs.keys()))
print("--------\nFilter objects:\n")
print(obs["filters"])
# --- Plot the Data ----
"""
# This is why we stored these...
wphot = obs["phot_wave"]
# establish bounds
xmin, xmax = np.min(wphot)*0.8, np.max(wphot)/0.8
ymin, ymax = obs["maggies"].min()*0.8, obs["maggies"].max()/0.4
fig = plt.figure(figsize=(10,5))
ax = fig.add_subplot(111)
# plot all the data
ax.plot(wphot, obs['maggies'],
label='All observed photometry',
marker='o', markersize=12, alpha=0.8, ls='', lw=3,
color='slateblue')
# overplot only the data we intend to fit
mask = obs["phot_mask"]
ax.errorbar(wphot[mask], obs['maggies'][mask],
yerr=obs['maggies_unc'][mask],
label='Photometry to fit',
marker='o', markersize=8, alpha=0.8, ls='', lw=3,
ecolor='tomato', markerfacecolor='none', markeredgecolor='tomato',
markeredgewidth=3)
# plot Filters
for f in obs['filters']:
w, t = f.wavelength.copy(), f.transmission.copy()
t = t / t.max()
t = 10**(0.2*(np.log10(ymax/ymin)))*t * ymin
ax.loglog(w, t, lw=3, color='gray', alpha=0.7)
# prettify
ax.set_xlabel('Wavelength [A]')
ax.set_ylabel('Flux Density [maggies]')
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
ax.set_xscale("log")
ax.set_yscale("log")
ax.legend(loc='best', fontsize=20)
fig.tight_layout()
plt.show()
"""
# ----------------------------
from prospect.models import priors
mass_param = {
"name": "mass",
# The mass parameter here is a scalar, so it has N=1
"N": 1,
# We will be fitting for the mass, so it is a free parameter
"isfree": True,
# This is the initial value. For fixed parameters this is the
# value that will always be used.
"init": 1e8,
# This sets the prior probability for the parameter
"prior": priors.LogUniform(mini=1e6, maxi=1e12),
# this sets the initial dispersion to use when generating
# clouds of emcee "walkers". It is not required, but can be very helpful.
"init_disp": 1e6,
# this sets the minimum dispersion to use when generating
#clouds of emcee "walkers". It is not required, but can be useful if
# burn-in rounds leave the walker distribution too narrow for some reason.
"disp_floor": 1e6,
# This is not required, but can be helpful
"units": "solar masses formed",
}
from prospect.models.templates import TemplateLibrary
# Look at all the prepackaged parameter sets
TemplateLibrary.show_contents()
# Check a specific model.
# This shows the defaults and params for the model.
# parametric_sfh in this case (delayed-tau)
# This is the one we will use
TemplateLibrary.describe("parametric_sfh")
run_params["object_redshift"] = 0.0
run_params["fixed_metallicity"] = None
run_params["add_duste"] = True
model = build_model(**run_params)
print(model)
print("\nInitial free parameter vector theta:\n {}\n".format(model.theta))
print("Initial parameter dictionary:\n{}".format(model.params))
run_params["zcontinuous"] = 1
sps = build_sps(**run_params)
# Generate the model SED at the initial value of theta
theta = model.theta.copy()
initial_spec, initial_phot, initial_mfrac = model.sed(theta,
obs=obs,
sps=sps)
#title_text = ','.join(["{}={}".format(p, model.params[p][0])
# for p in model.free_params])
a = 1.0 + model.params.get('zred', 0.0) # cosmological redshifting
# photometric effective wavelengths
wphot = obs["phot_wave"]
# spectroscopic wavelengths
if obs["wavelength"] is None:
# *restframe* spectral wavelengths, since obs["wavelength"] is None
wspec = sps.wavelengths
wspec *= a #redshift them
else:
wspec = obs["wavelength"]
# establish bounds
xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8
temp = np.interp(np.linspace(xmin, xmax, 10000), wspec, initial_spec)
ymin, ymax = temp.min() * 0.8, temp.max() / 0.4
"""
fig = plt.figure(figsize=(10,5))
ax = fig.add_subplot(111)
# plot model + data
ax.loglog(wspec, initial_spec, label='Model spectrum',
lw=0.7, color='navy', alpha=0.7)
ax.errorbar(wphot, initial_phot, label='Model photometry',
marker='s',markersize=10, alpha=0.8, ls='', lw=3,
markerfacecolor='none', markeredgecolor='blue',
markeredgewidth=3)
ax.errorbar(wphot, obs['maggies'], yerr=obs['maggies_unc'],
label='Observed photometry',
marker='o', markersize=10, alpha=0.8, ls='', lw=3,
ecolor='red', markerfacecolor='none', markeredgecolor='red',
markeredgewidth=3)
ax.set_title(title_text)
# plot Filters
for f in obs['filters']:
w, t = f.wavelength.copy(), f.transmission.copy()
t = t / t.max()
t = 10**(0.2*(np.log10(ymax/ymin)))*t * ymin
ax.loglog(w, t, lw=3, color='gray', alpha=0.7)
# prettify
ax.set_xlabel('Wavelength [A]')
ax.set_ylabel('Flux Density [maggies]')
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
ax.legend(loc='best', fontsize=20)
fig.tight_layout()
plt.show()
"""
verbose = True
run_params["verbose"] = verbose
# Here we will run all our building functions
obs = build_obs(**run_params)
sps = build_sps(**run_params)
model = build_model(**run_params)
# For fsps based sources it is useful to
# know which stellar isochrone and spectral library
# we are using
print(sps.ssp.libraries)
# --- start minimization ----
run_params["dynesty"] = False
run_params["emcee"] = False
run_params["optimize"] = True
run_params["min_method"] = 'lm'
# We'll start minimization from "nmin" separate places,
# the first based on the current values of each parameter and the
# rest drawn from the prior. Starting from these extra draws
# can guard against local minima, or problems caused by
# starting at the edge of a prior (e.g. dust2=0.0)
run_params["nmin"] = 2
output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params)
print("Done optmization in {}s".format(output["optimization"][1]))
print(model.theta)
(results, topt) = output["optimization"]
# Find which of the minimizations gave the best result,
# and use the parameter vector for that minimization
ind_best = np.argmin([r.cost for r in results])
print(ind_best)
theta_best = results[ind_best].x.copy()
print(theta_best)
# generate model
prediction = model.mean_model(theta_best, obs=obs, sps=sps)
pspec, pphot, pfrac = prediction
"""
fig = plt.figure(figsize=(10,5))
ax = fig.add_subplot(111)
# plot Data, best fit model, and old models
ax.loglog(wspec, initial_spec, label='Old model spectrum',
lw=0.7, color='gray', alpha=0.5)
ax.errorbar(wphot, initial_phot, label='Old model Photometry',
marker='s', markersize=10, alpha=0.6, ls='', lw=3,
markerfacecolor='none', markeredgecolor='gray',
markeredgewidth=3)
ax.loglog(wspec, pspec, label='Model spectrum',
lw=0.7, color='slateblue', alpha=0.7)
ax.errorbar(wphot, pphot, label='Model photometry',
marker='s', markersize=10, alpha=0.8, ls='', lw=3,
markerfacecolor='none', markeredgecolor='slateblue',
markeredgewidth=3)
ax.errorbar(wphot, obs['maggies'], yerr=obs['maggies_unc'],
label='Observed photometry',
marker='o', markersize=10, alpha=0.8, ls='', lw=3,
ecolor='tomato', markerfacecolor='none', markeredgecolor='tomato',
markeredgewidth=3)
# plot filter transmission curves
for f in obs['filters']:
w, t = f.wavelength.copy(), f.transmission.copy()
t = t / t.max()
t = 10**(0.2*(np.log10(ymax/ymin)))*t * ymin
ax.loglog(w, t, lw=3, color='gray', alpha=0.7)
# Prettify
ax.set_xlabel('Wavelength [A]')
ax.set_ylabel('Flux Density [maggies]')
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
ax.legend(loc='best', fontsize=20)
fig.tight_layout()
plt.show()
"""
# Set this to False if you don't want to do another optimization
# before emcee sampling (but note that the "optimization" entry
# in the output dictionary will be (None, 0.) in this case)
# If set to true then another round of optmization will be performed
# before sampling begins and the "optmization" entry of the output
# will be populated.
run_params["optimize"] = False
run_params["emcee"] = True
run_params["dynesty"] = False
# Number of emcee walkers
run_params["nwalkers"] = 128
# Number of iterations of the MCMC sampling
run_params["niter"] = 512
# Number of iterations in each round of burn-in
# After each round, the walkers are reinitialized based on the
# locations of the highest probablity half of the walkers.
run_params["nburn"] = [16, 32, 64]
print("Now running with Emcee.")
output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params)
print('done emcee in {0}s'.format(output["sampling"][1]))
hfile = "demo_emcee_mcmc.h5"
writer.write_hdf5(hfile,
run_params,
model,
obs,
output["sampling"][0],
output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
print('Finished')
# -------------------
print("Now running with Dynesty.")
run_params["dynesty"] = True
run_params["optmization"] = False
run_params["emcee"] = False
run_params["nested_method"] = "rwalk"
run_params["nlive_init"] = 400
run_params["nlive_batch"] = 200
run_params["nested_dlogz_init"] = 0.05
run_params["nested_posterior_thresh"] = 0.05
run_params["nested_maxcall"] = int(1e7)
output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params)
print('done dynesty in {0}s'.format(output["sampling"][1]))
hfile = "demo_dynesty_mcmc.h5"
writer.write_hdf5(hfile,
run_params,
model,
obs,
output["sampling"][0],
output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
print('Finished')
# -------------------------
# Visualizing results
results_type = "dynesty" # "emcee" | "dynesty"
# grab results (dictionary), the obs dictionary, and our corresponding models
# When using parameter files set `dangerous=True`
result, obs, _ = reader.results_from(
"demo_{}_mcmc.h5".format(results_type), dangerous=False)
#The following commented lines reconstruct the model and sps object,
# if a parameter file continaing the `build_*` methods was saved along with the results
#model = reader.get_model(result)
#sps = reader.get_sps(result)
# let's look at what's stored in the `result` dictionary
print(result.keys())
if results_type == "emcee":
chosen = np.random.choice(result["run_params"]["nwalkers"],
size=10,
replace=False)
tracefig = reader.traceplot(result, figsize=(20, 10), chains=chosen)
else:
tracefig = reader.traceplot(result, figsize=(20, 10))
# maximum a posteriori (of the locations visited by the MCMC sampler)
imax = np.argmax(result['lnprobability'])
if results_type == "emcee":
i, j = np.unravel_index(imax, result['lnprobability'].shape)
theta_max = result['chain'][i, j, :].copy()
thin = 5
else:
theta_max = result["chain"][imax, :]
thin = 1
print('Optimization value: {}'.format(theta_best))
print('MAP value: {}'.format(theta_max))
cornerfig = reader.subcorner(result,
start=0,
thin=thin,
truths=theta_best,
fig=plt.subplots(5, 5, figsize=(8, 8))[0])
plt.show()
return None
# - Parser with default arguments -
parser = prospect_args.get_parser()
# - Add custom arguments -
parser.add_argument('--zred', type=float, default=0.1,
help="redshift for the model")
parser.add_argument('--add_neb', action="store_true",
help="If set, add nebular emission in the model")
parser.add_argument('--add_noise', action="store_true",
help="If set, noise up the mock")
parser.add_argument('--snr', type=float, default=20,
help="S/N ratio for the mock photometry")
args = parser.parse_args()
run_params = vars(args)
obs, model, sps, noise = build_all(**run_params)
run_params["param_file"] = __file__
hfile = setup_h5(model=model, obs=obs, **run_params)
output = fit_model(obs, model, sps, noise, **run_params)
writer.write_hdf5(hfile, run_params, model, obs,
output["sampling"][0], output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
try:
hfile.close()
except(AttributeError):
pass
# -------
# Sample
# -------
if rp['verbose']:
print('emcee sampling...')
tstart = time.time()
out = fitting.run_emcee_sampler(lnprobfn, initial_center, model,
postkwargs=postkwargs, initial_prob=initial_prob,
pool=pool, hdf5=hfile, **rp)
esampler, burn_p0, burn_prob0 = out
edur = time.time() - tstart
if rp['verbose']:
print('done emcee in {0}s'.format(edur))
# -------------------------
# Output pickles (and HDF5)
# -------------------------
write_results.write_pickles(rp, model, obsdat, esampler, powell_guesses,
outroot=outroot, toptimize=pdur, tsample=edur,
sampling_initial_center=initial_center,
post_burnin_center=burn_p0,
post_burnin_prob=burn_prob0)
if hfile is None:
hfile = hfilename
write_results.write_hdf5(hfile, rp, model, obsdat, esampler, powell_guesses,
toptimize=pdur, tsample=edur,
sampling_initial_center=initial_center,
post_burnin_center=burn_p0,
post_burnin_prob=burn_prob0)
halt('Finished')
def run_prospector(name_z):
import time, sys, os
import h5py
import numpy as np
import scipy
from matplotlib.pyplot import *
#%matplotlib inline
import fsps
import sedpy
import prospect
import emcee
import astropy
import math
from prospect.fitting import fit_model
from prospect.fitting import lnprobfn
from func import *
import matplotlib.pyplot as plt
# --- Output files ---
name = name_z[0]
z = name_z[1]
path_input = '/home/kk/work/data/red_magic/' + name
path = '/home/kk/work/result/red_magic/' + name[:-4]
'''try:
os.mkdir(path)
except:
pass
fe = open(path+'/para.txt','w')'''
# --- Set up the essential parameter ---
data_list = [
'sdss_u', 'sdss_g', 'sdss_r', 'sdss_i', 'galex_fuv', 'galex_nuv',
'pan-starrs_ps1_i', 'pan-starrs_ps1_r', 'pan-starrs_ps1_g',
'pan-starrs_ps1_y', 'pan-starrs_ps1_z', 'sdss_z', 'ukidss_j',
'ukidss_h', 'ukidss_k', 'wise_w1', 'spitzer_irac_3.6', '_=3.6um',
'spitzer_irac_4.5', '_=4.5um', 'wise_w2', 'spitzer_irac_5.8',
'spitzer_irac_8.0', 'wise_w3', 'wise_w4', 'spitzer_mips_24', '2mass_h',
'2mass_ks', '2mass_j', 'gaia_gaia2_gbp', 'gaia_gaia2_g',
'gaia_gaia2_grp', 'gaia_g', 'ukidss_y', 'vista_j', 'vista_h',
'vista_ks'
]
run_params = {}
run_params["snr"] = 10.0
run_params["object_redshift"] = z
run_params["fixed_metallicity"] = None
run_params["add_duste"] = True
# --- Start build up the model ---
fit_data = load_data(path_input, data_list)
obs = build_obs(fit_data, **run_params)
sps = build_sps(**run_params)
model = build_model(**run_params)
#fe.write(str(obs))
#print(fit_data)
# --- Draw initial data plot ---
wphot = obs["phot_wave"]
# establish bounds
xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8
ymin, ymax = obs["maggies"].min() * 0.8, obs["maggies"].max() / 0.4
figure(figsize=(16, 8))
# plot all the data
plot(wphot,
obs['maggies'],
label='All observed photometry',
marker='o',
markersize=12,
alpha=0.8,
ls='',
lw=3,
color='slateblue')
# overplot only the data we intend to fit
mask = obs["phot_mask"]
errorbar(wphot[mask],
obs['maggies'][mask],
yerr=obs['maggies_unc'][mask],
label='Photometry to fit',
marker='o',
markersize=8,
alpha=0.8,
ls='',
lw=3,
ecolor='tomato',
markerfacecolor='none',
markeredgecolor='tomato',
markeredgewidth=3)
# prettify
xlabel('Wavelength [A]')
ylabel('Flux Density [maggies]')
xlim([xmin, xmax])
ylim([ymin, ymax])
xscale("log")
yscale("log")
legend(loc='best', fontsize=20)
#tight_layout()
#plt.savefig(path+'/input.png')
# run minimization and emcee with error handling
condition = False
while condition is False:
try:
# --- start minimization ----
run_params["dynesty"] = False
run_params["emcee"] = False
run_params["optimize"] = True
run_params["min_method"] = 'lm'
run_params["nmin"] = 5
output = fit_model(obs,
model,
sps,
lnprobfn=lnprobfn,
**run_params)
print("Done optmization in {}s".format(output["optimization"][1]))
# save theta_best for later use
(results, topt) = output["optimization"]
ind_best = np.argmin([r.cost for r in results])
theta_best = results[ind_best].x.copy()
# --- start emcee ---
run_params["optimize"] = False
run_params["emcee"] = True
run_params["dynesty"] = False
run_params["nwalkers"] = 30
run_params["niter"] = 8000
run_params["nburn"] = [300, 800, 1600]
output = fit_model(obs,
model,
sps,
lnprobfn=lnprobfn,
**run_params)
print('done emcee in {0}s'.format(output["sampling"][1]))
condition = True
# sort out different errors and apply method to proceed
except AssertionError as e:
# error: sfr cannot be negative
if str(e) == 'sfr cannot be negative.':
obs = build_obs(fit_data, **run_params)
sps = build_sps(**run_params)
model = build_model(**run_params)
# error: number of residules are less than variables
except ValueError as e:
if str(
e
) == "Method 'lm' doesn't work when the number of residuals is less than the number of variables.":
print(name)
return
# error: residule are not finite in the initial point
elif str(e) == "Residuals are not finite in the initial point.":
print(name)
return
# write final results to file
from prospect.io import write_results as writer
hfile = path + '_emcee.h5'
print(hfile)
writer.write_hdf5(hfile,
run_params,
model,
obs,
output["sampling"][0],
output["optimization"][0],
tsample=output["sampling"][1],
toptimize=output["optimization"][1])
#fe.write(str(obs))
print('Finished')
##############################################################################
import prospect.io.read_results as reader
results_type = "emcee" # | "dynesty"
# grab results (dictionary), the obs dictionary, and our corresponding models
# When using parameter files set `dangerous=True`
#tem_name = hfile[:-8]+'{}.h5'
#result, obs, _ = reader.results_from(tem_name.format(results_type), dangerous=False)
result, obs, _ = reader.results_from(hfile, dangerous=False)
#The following commented lines reconstruct the model and sps object, # if a parameter file continaing the `build_*` methods was saved along with the results
#model = reader.get_model(result)
#sps = reader.get_sps(result)
# let's look at what's stored in the `result` dictionary
#print(result.keys())
# Make plot of data and model
if results_type == "emcee":
chosen = np.random.choice(result["run_params"]["nwalkers"],
size=10,
replace=False)
tracefig = reader.traceplot(result, figsize=(20, 10), chains=chosen)
else:
tracefig = reader.traceplot(result, figsize=(20, 10))
#plt.savefig(path+'/trace.png')
# maximum a posteriori (of the locations visited by the MCMC sampler)
imax = np.argmax(result['lnprobability'])
if results_type == "emcee":
i, j = np.unravel_index(imax, result['lnprobability'].shape)
theta_max = result['chain'][i, j, :].copy()
thin = 5
else:
theta_max = result["chain"][imax, :]
thin = 1
#f.Write('Optimization value: {}'.format(theta_best))
#fe.write(str('MAP value: {}'.format(theta_max)))
#fe.close()
cornerfig = reader.subcorner(result,
start=0,
thin=thin,
truths=theta_best,
fig=subplots(15, 15, figsize=(27, 27))[0])
#plt.savefig(path+'/corner.png')
# --- Draw final fitting curve ---
a = 1.0 + model.params.get('zred', 0.0) # cosmological redshifting
# photometric effective wavelengths
wphot = obs["phot_wave"]
# spectroscopic wavelengths
if obs["wavelength"] is None:
# *restframe* spectral wavelengths, since obs["wavelength"] is None
wspec = sps.wavelengths
wspec *= a #redshift them
else:
wspec = obs["wavelength"]
# Get enssential data to calculate sfr
z_fraction = theta_max[5:10]
total_mass = theta_max[0]
agebins = model.params['agebins']
x = np.zeros(6)
for i in range(6):
if i == 5:
x[i] = (agebins[i][0] + agebins[i][1]) / 2
else:
x[i] = (agebins[i][0] + agebins[i + 1][0]) / 2
sfr = prospect.models.transforms.zfrac_to_sfr(total_mass, z_fraction,
agebins)
# Calculate the 16% and 84% value from emcee results
z_fraction1 = []
z_fraction2 = []
z_fraction3 = []
z_fraction4 = []
z_fraction5 = []
for i in range(30):
for j in range(3000):
z_fraction1.append(result['chain'][i][j][5])
z_fraction2.append(result['chain'][i][j][6])
z_fraction3.append(result['chain'][i][j][7])
z_fraction4.append(result['chain'][i][j][8])
z_fraction5.append(result['chain'][i][j][9])
zerr_1 = [np.percentile(z_fraction1, 16), np.percentile(z_fraction1, 84)]
zerr_2 = [np.percentile(z_fraction2, 16), np.percentile(z_fraction2, 84)]
zerr_3 = [np.percentile(z_fraction3, 16), np.percentile(z_fraction3, 84)]
zerr_4 = [np.percentile(z_fraction4, 16), np.percentile(z_fraction4, 84)]
zerr_5 = [np.percentile(z_fraction5, 16), np.percentile(z_fraction5, 84)]
# Build upper and lower z_fraction array
z_max = np.zeros(5)
z_min = np.zeros(5)
for i in range(5):
z_min[i] = eval('zerr_{}[0]'.format(i + 1))
z_max[i] = eval('zerr_{}[1]'.format(i + 1))
#print(z_min)
#print(z_max)
# Build new sfr
sfr_max = prospect.models.transforms.zfrac_to_sfr(total_mass, z_max,
agebins)
sfr_min = prospect.models.transforms.zfrac_to_sfr(total_mass, z_min,
agebins)
# randomly chosen parameters from chain
randint = np.random.randint
if results_type == "emcee":
nwalkers, niter = run_params['nwalkers'], run_params['niter']
# Draw 100 random plots from mcmc
figure(figsize=(16, 16))
ax = plt.subplot(211)
for i in range(100):
# Get data from any random chain
theta = result['chain'][randint(nwalkers), randint(niter)]
mspec_t, mphot_t, mextra = model.mean_model(theta, obs, sps=sps)
# unit conversion to erg/cm^2/s
mspec = (mspec_t * 1e-23) * 3e18 / wspec
mphot = (mphot_t * 1e-23) * 3e18 / wphot
# plot them out
loglog(wspec, mspec, lw=0.7, color='grey', alpha=0.3)
else:
theta = result["chain"][randint(len(result["chain"]))]
# now do it again for best fit model
# sps = reader.get_sps(result) # this works if using parameter files
mspec_map_t, mphot_map_t, _ = model.mean_model(theta_max, obs, sps=sps)
# units conversion to erg/cm^2/s
mspec_map = (mspec_map_t * 1e-23) * 3e18 / wspec
mphot_map = (mphot_map_t * 1e-23) * 3e18 / wphot
ob = (obs['maggies'] * 1e-23) * 3e18 / wphot
ob_unc = (obs['maggies_unc'] * 1e-23) * 3e18 / wphot
# calculate reduced chi^2
chi = 0
for i in range(len(wphot)):
var = ob_unc[i]
chi += (mphot_map[i] - ob[i])**2 / var**2
# Make plot of best fit model
loglog(wspec,
mspec_map,
label='Model spectrum (MAP)',
lw=0.7,
color='green',
alpha=0.7)
errorbar(wphot,
mphot_map,
label='Model photometry (MAP)',
marker='s',
markersize=10,
alpha=0.8,
ls='',
lw=3,
markerfacecolor='none',
markeredgecolor='green',
markeredgewidth=3)
errorbar(wphot,
ob,
yerr=ob_unc,
label='Observed photometry',
ecolor='red',
marker='o',
markersize=10,
ls='',
lw=3,
alpha=0.8,
markerfacecolor='none',
markeredgecolor='red',
markeredgewidth=3)
text(0,
1,
'Chi-squared/Nphot =' + str(chi / len(wphot)),
horizontalalignment='center',
verticalalignment='center',
transform=ax.transAxes)
# Setup bound
xmin, xmax = np.min(wphot) * 0.6, np.max(wphot) / 0.8
temp = np.interp(np.linspace(xmin, xmax, 10000), wphot, mphot_map)
ymin, ymax = temp.min() * 0.8, temp.max() * 10
xlabel('Wavelength [A]', fontsize=20)
ylabel('Flux Density [erg/cm^2/s]', fontsize=20)
xlim([xmin, xmax])
ylim([ymin, ymax])
legend(loc='lower right', fontsize=15)
tight_layout()
# plt.show()
# plot sfr on the same canvas
from mpl_toolkits.axes_grid.inset_locator import inset_axes
in_axes = inset_axes(
ax,
width="25%", # width = 30% of parent_bbox
height=2, # height : 1 inch
loc='upper center')
in_axes.scatter(10**(x - 9), sfr)
#plt.fill_between(10**(x-9), sfr_min, sfr_max)
xlabel('Age [Gyr]', fontsize=10)
ylabel('SFR [M_sun/yr]', fontsize=10)
#plt.xscale('log')
plt.yscale('log')
plt.savefig(path + '_fitting.png')
# plot residue as subplot
figure(figsize=(16, 8))
plt.subplot(212, sharex=ax)
import pylab
residue = abs(mphot_map - ob)
scatter(wphot, abs(mphot_map - ob), label='Residue', lw=2, alpha=0.8)
plt.xscale('log')
plt.yscale('log')
xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8
temp = np.interp(np.linspace(xmin, xmax, 10000), wphot, residue)
ymin, ymax = temp.min() * 0.1, temp.max() / 0.8
xlabel('Wavelength [A]', fontsize=20)
ylabel('Flux Change [erg/cm^2/s]', fontsize=20)
xlim([xmin, xmax])
ylim([ymin, ymax])
legend(loc='best', fontsize=15)
tight_layout()
# -------
if rp['verbose']:
print('dynesty sampling...')
tstart = time.time() # time it
dynestyout = fitting.run_dynesty_sampler(lnprobfn, prior_transform, model.ndim,
pool=pool, queue_size=nprocs,
stop_function=stopping_function,
wt_function=weight_function,
**rp)
ndur = time.time() - tstart
print('done dynesty in {0}s'.format(ndur))
# -------------------------
# Output HDF5 (and pickles if asked for)
# -------------------------
if rp.get("output_pickles", False):
# Write the dynesty result object as a pickle
import pickle
with open(outroot + '_dns.pkl', 'w') as f:
pickle.dump(dynestyout, f)
# Write the model as a pickle
partext = write_results.paramfile_string(**rp)
write_results.write_model_pickle(outroot + '_model', model, powell=None,
paramfile_text=partext)
# Write HDF5
hfile = outroot + '_mcmc.h5'
write_results.write_hdf5(hfile, rp, model, obs, dynestyout,
None, tsample=ndur)
