def write_h5(self, savefile, warning=True): """ Save the fitting results and every fitter inputs (apart from the 'sps') in a given file. Parameters ---------- savefile : [string] File name in which to save the results. Must be an hdf5 file (thus ending with ".h5"). If the file already exists, add the day and time before ".h5" in file name. Options ------- warnings : [bool] If True, allow warnings to be printed. Default is True. Returns ------- Void """ import h5py import pickle if os.path.isfile(savefile): _savefile, savefile = savefile, savefile.replace(".h5", f"_{io.get_now()}.h5") if warning: warn(f"The file {_savefile} is already existing. Writing results in: {savefile}") from prospect.io import write_results as writer writer.write_hdf5(hfile=savefile, run_params=self.run_params, model=self.model, obs=self.obs, sampler=self.fit_output["sampling"][0], optimize_result_list=self.fit_output["optimization"][0], tsample=self.fit_output["sampling"][1], toptimize=self.fit_output["optimization"][1]) with h5py.File(savefile, "a") as _h5f: _h5f.create_dataset("model", data=np.void(pickle.dumps(self.model))) _h5f.create_dataset("sps", data=np.void(pickle.dumps(self.sps))) _h5f.flush()
model, pool=pool, hdf5=hfile, **run_params) esampler, burn_loc0, burn_prob0 = out edur = time.time() - tstart sys.stdout = fout print('done emcee in {0}s'.format(edur)) write_results.write_hdf5(hfile, run_params, model, obs, esampler, guesses, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center) print('Finished. Saved to ' + str(hfilename)) time.sleep(3) #Formatting Data for Pippi pippi_format(hfilename) print('Reformated to ' + str(hfilename[:-3]) + str('_pippi.hdf5')) imax = np.argmax(esampler.lnprobability) i, j = np.unravel_index(imax, esampler.lnprobability.shape) theta_max = esampler.chain[i, j, :].copy()
args = parser.parse_args() run_params = vars(args) obs, model, sps, noise = build_all(**run_params) run_params["sps_libraries"] = sps.ssp.libraries run_params["param_file"] = __file__ print(model) if args.debug: sys.exit() #hfile = setup_h5(model=model, obs=obs, **run_params) hfile = "{0}_{1}_mcmc.h5".format(args.outfile, int(time.time())) output = fit_model(obs, model, sps, noise, **run_params) writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1], sps=sps) try: hfile.close() except (AttributeError): pass
def main(field, galaxy_seq): #vers = (np.__version__, scipy.__version__, h5py.__version__, fsps.__version__, prospect.__version__) #print("Numpy: {}\nScipy: {}\nH5PY: {}\nFSPS: {}\nProspect: {}".format(*vers)) # -------------- Decide field and filters # Read in catalog from Lou if 'North' in field: df = pandas.read_pickle(adap_dir + 'GOODS_North_SNeIa_host_phot.pkl') all_filters = [ 'LBC_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX', 'ACS_F775W_FLUX', 'ACS_F814W_FLUX', 'ACS_F850LP_FLUX', 'WFC3_F105W_FLUX', 'WFC3_F125W_FLUX', 'WFC3_F140W_FLUX', 'WFC3_F160W_FLUX', 'MOIRCS_K_FLUX', 'CFHT_Ks_FLUX', 'IRAC_CH1_SCANDELS_FLUX', 'IRAC_CH2_SCANDELS_FLUX', 'IRAC_CH3_FLUX', 'IRAC_CH4_FLUX' ] #all_filters = ['LBC_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX', #'ACS_F775W_FLUX', 'ACS_F850LP_FLUX'] seq = np.array(df['ID']) i = int(np.where(seq == galaxy_seq)[0]) elif 'South' in field: df = pandas.read_pickle(adap_dir + 'GOODS_South_SNeIa_host_phot.pkl') all_filters = [ 'CTIO_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX', 'ACS_F775W_FLUX', 'ACS_F814W_FLUX', 'ACS_F850LP_FLUX', 'WFC3_F098M_FLUX', 'WFC3_F105W_FLUX', 'WFC3_F125W_FLUX', 'WFC3_F160W_FLUX', 'HAWKI_KS_FLUX', 'IRAC_CH1_FLUX', 'IRAC_CH2_FLUX', 'IRAC_CH3_FLUX', 'IRAC_CH4_FLUX' ] #all_filters = ['CTIO_U_FLUX', 'ACS_F435W_FLUX', 'ACS_F606W_FLUX', #'ACS_F775W_FLUX', 'ACS_F850LP_FLUX'] seq = np.array(df['Seq']) i = int(np.where(seq == galaxy_seq)[0]) #print('Read in pickle with the following columns:') #print(df.columns) #print('Rows in DataFrame:', len(df) print("Match index:", i, "for Seq:", galaxy_seq) # -------------- Preliminary stuff # Set up for emcee nwalkers = 1000 niter = 500 ndim = 12 # Other set up obj_ra = df['RA'][i] obj_dec = df['DEC'][i] obj_z = df['zbest'][i] print("Object redshift:", obj_z) age_at_z = astropy_cosmo.age(obj_z).value print("Age of Universe at object redshift [Gyr]:", age_at_z) # ------------- Get obs data fluxes = [] fluxes_unc = [] useable_filters = [] for ft in range(len(all_filters)): filter_name = all_filters[ft] flux = df[filter_name][i] fluxerr = df[filter_name + 'ERR'][i] if np.isnan(flux): continue if flux <= 0.0: continue if (fluxerr < 0) or np.isnan(fluxerr): fluxerr = 0.1 * flux fluxes.append(flux) fluxes_unc.append(fluxerr) useable_filters.append(filter_name) #print("\n") #print(df.loc[i]) #print(fluxes, len(fluxes)) #print(useable_filters, len(useable_filters)) fluxes = np.array(fluxes) fluxes_unc = np.array(fluxes_unc) # Now build the prospector observation obs = build_obs(fluxes, fluxes_unc, useable_filters) # Set params for run run_params = {} run_params["object_redshift"] = obj_z run_params["fixed_metallicity"] = None run_params["add_duste"] = True #run_params["dust_type"] = 4 run_params["zcontinuous"] = 1 # Generate the model SED at the initial value of theta #theta = model.theta.copy() #initial_spec, initial_phot, initial_mfrac = model.sed(theta, obs=obs, sps=sps) verbose = True run_params["verbose"] = verbose model = build_model(**run_params) print("\nInitial free parameter vector theta:\n {}\n".format(model.theta)) #print("Initial parameter dictionary:\n{}".format(model.params)) print("\n----------------------- Model details: -----------------------") print(model) print( "----------------------- End model details. -----------------------\n") # Here we will run all our building functions obs = build_obs(fluxes, fluxes_unc, useable_filters) sps = build_sps(**run_params) #plot_data(obs) #sys.exit(0) # --- start fitting ---- # Set this to False if you don't want to do another optimization # before emcee sampling (but note that the "optimization" entry # in the output dictionary will be (None, 0.) in this case) # If set to true then another round of optmization will be performed # before sampling begins and the "optmization" entry of the output # will be populated. """ run_params["optimize"] = False run_params["min_method"] = 'lm' # We'll start minimization from "nmin" separate places, # the first based on the current values of each parameter and the # rest drawn from the prior. Starting from these extra draws # can guard against local minima, or problems caused by # starting at the edge of a prior (e.g. dust2=0.0) run_params["nmin"] = 5 run_params["emcee"] = True run_params["dynesty"] = False # Number of emcee walkers run_params["nwalkers"] = nwalkers # Number of iterations of the MCMC sampling run_params["niter"] = niter # Number of iterations in each round of burn-in # After each round, the walkers are reinitialized based on the # locations of the highest probablity half of the walkers. run_params["nburn"] = [8, 16, 32, 64] run_params["progress"] = True hfile = adap_dir + "emcee_" + field + "_" + str(galaxy_seq) + ".h5" if not os.path.isfile(hfile): print("Now running with Emcee.") output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print('done emcee in {0}s'.format(output["sampling"][1])) writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) print('Finished with Seq: ' + str(galaxy_seq)) """ hfile = adap_dir + "dynesty_" + field + "_" + str(galaxy_seq) + ".h5" if not os.path.isfile(hfile): print("Now running with Dynesty.") run_params["emcee"] = False run_params["dynesty"] = True run_params["nested_method"] = "rwalk" run_params["nlive_init"] = 400 run_params["nlive_batch"] = 200 run_params["nested_dlogz_init"] = 0.05 run_params["nested_posterior_thresh"] = 0.05 run_params["nested_maxcall"] = int(1e6) #from multiprocessing import Pool #with Pool(6) as pool: # run_params["pool"] = pool output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print('done dynesty in {0}s'.format(output["sampling"][1])) writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) print('Finished with Seq: ' + str(galaxy_seq)) # ------------------------- # Visualizing results results_type = "dynesty" # "emcee" | "dynesty" # grab results (dictionary), the obs dictionary, and our corresponding models # When using parameter files set `dangerous=True` #result, obs, _ = reader.results_from("{}_" + str(galaxy_seq) + \ # ".h5".format(results_type), dangerous=False) result, obs, _ = reader.results_from(adap_dir + results_type + "_" + \ field + "_" + str(galaxy_seq) + ".h5", dangerous=False) #The following commented lines reconstruct the model and sps object, # if a parameter file continaing the `build_*` methods was saved along with the results #model = reader.get_model(result) #sps = reader.get_sps(result) # let's look at what's stored in the `result` dictionary print(result.keys()) parnames = np.array(result['theta_labels']) print('Parameters in this model:', parnames) """ if results_type == "emcee": chosen = np.random.choice(result["run_params"]["nwalkers"], size=150, replace=False) tracefig = reader.traceplot(result, figsize=(10,6), chains=chosen) tracefig.savefig(adap_dir + 'trace_' + field + '_' + str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight') else: tracefig = reader.traceplot(result, figsize=(10,6)) tracefig.savefig(adap_dir + 'trace_' + field + '_' + str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight') """ # Get chain for corner plot if results_type == 'emcee': trace = result['chain'] thin = 5 trace = trace[:, ::thin, :] samples = trace.reshape(trace.shape[0] * trace.shape[1], trace.shape[2]) else: samples = result['chain'] # Plot SFH #print(model) # to copy paste agebins from print out nagebins = 6 agebins = np.array([[0., 8.], [8., 8.47712125], [8.47712125, 9.], [9., 9.47712125], [9.47712125, 9.77815125], [9.77815125, 10.13353891]]) # Get the zfractions from corner quantiles zf1 = corner.quantile(samples[:, 2], q=[0.16, 0.5, 0.84]) zf2 = corner.quantile(samples[:, 3], q=[0.16, 0.5, 0.84]) zf3 = corner.quantile(samples[:, 4], q=[0.16, 0.5, 0.84]) zf4 = corner.quantile(samples[:, 5], q=[0.16, 0.5, 0.84]) zf5 = corner.quantile(samples[:, 6], q=[0.16, 0.5, 0.84]) zf_arr = np.array([zf1[1], zf2[1], zf3[1], zf4[1], zf5[1]]) zf_arr_l = np.array([zf1[0], zf2[0], zf3[0], zf4[0], zf5[0]]) zf_arr_u = np.array([zf1[2], zf2[2], zf3[2], zf4[2], zf5[2]]) cq_mass = corner.quantile(samples[:, 7], q=[0.16, 0.5, 0.84]) print("Total mass:", "{:.3e}".format(cq_mass[1]), "+", "{:.3e}".format(cq_mass[2] - cq_mass[1]), "-", "{:.3e}".format(cq_mass[1] - cq_mass[0])) # ----------- new_agebins = pt.zred_to_agebins(zred=obj_z, agebins=agebins) print("New agebins:", new_agebins) # ----------------------- # now convert to sfh and its errors sfr = pt.zfrac_to_sfr(total_mass=cq_mass[1], z_fraction=zf_arr, agebins=new_agebins) sfr_l = pt.zfrac_to_sfr(total_mass=cq_mass[1], z_fraction=zf_arr_l, agebins=new_agebins) sfr_u = pt.zfrac_to_sfr(total_mass=cq_mass[1], z_fraction=zf_arr_u, agebins=new_agebins) print("----------") print("z fractions: ", zf_arr) print("Lower z fractions:", zf_arr_l) print("Upper z fractions:", zf_arr_u) print("----------") print("Inferred SFR: ", sfr) print("Lower sfr vals:", sfr_l) print("Upper sfr vals:", sfr_u) ############# x_agebins = 10**new_agebins / 1e9 fig = plt.figure(figsize=(8, 4)) ax = fig.add_subplot(111) ax.set_xlabel(r'$\mathrm{Time\, [Gyr];\, since\ galaxy\ formation}$', fontsize=20) ax.set_ylabel(r'$\mathrm{SFR\, [M_\odot/yr]}$', fontsize=20) for a in range(len(agebins)): ax.plot(x_agebins[a], np.ones(len(x_agebins[a])) * sfr[a], color='mediumblue', lw=3.0) # put in some poisson errors sfr_err = np.ones(len(x_agebins[a])) * np.sqrt(sfr[a]) sfr_plt = np.ones(len(x_agebins[a])) * sfr[a] sfr_low_fill = sfr_plt - sfr_err sfr_up_fill = sfr_plt + sfr_err ax.fill_between(x_agebins[a], sfr_low_fill, sfr_up_fill, color='gray', alpha=0.6) #ax.set_ylim(np.min(sfr_low_fill) * 0.3, np.max(sfr_up_fill) * 1.1) fig.savefig(adap_dir + 'sfh_' + field + '_' + str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight') sys.exit(0) # Keep code block for future use if needed """ # combination of linear and log axis from # https://stackoverflow.com/questions/21746491/combining-a-log-and-linear-scale-in-matplotlib # linear part i.e., first age bin ax.plot(x_agebins[0], np.ones(len(x_agebins[0])) * sfr[0], color='mediumblue', lw=3.5) #ax.fill_between(x_agebins[0], np.ones(len(x_agebins[0])) * sfr_l[0], # np.ones(len(x_agebins[0])) * sfr_u[0], color='gray', alpha=0.5) ax.set_xlim(0.0, 8.0) ax.spines['right'].set_visible(False) ax.yaxis.set_ticks_position('left') ax.xaxis.set_label_coords(x=1.2,y=-0.06) # now log axis divider = make_axes_locatable(ax) axlog = divider.append_axes("right", size=3.0, pad=0, sharey=ax) axlog.set_xscale('log') for a in range(1, nagebins): axlog.plot(x_agebins[a], np.ones(len(x_agebins[a])) * sfr[a], color='mediumblue', lw=3.5) #axlog.fill_between(x_agebins[a], np.ones(len(x_agebins[a])) * sfr_l[a], # np.ones(len(x_agebins[a])) * sfr_u[a], color='gray', alpha=0.5) axlog.set_xlim(8.0, x_agebins[-1, -1] + 0.1) axlog.spines['left'].set_visible(False) axlog.yaxis.set_ticks_position('right') axlog.tick_params(labelright=False) axlog.xaxis.set_ticks(ticks=[8.0, 9.0]) axlog.xaxis.set_ticks(ticks=[8.2, 8.4, 8.6, 8.8, 9.2, 9.4, 9.6], minor=True) axlog.set_xticklabels(['8', '9']) axlog.set_xticklabels(labels=[], minor=True) fig.savefig(adap_dir + 'sfh_' + field + '_' + str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight') """ # ---------- corner plot # set up corner ranges and labels math_parnames = [ r'$\mathrm{log(Z_\odot)}$', r'$\mathrm{dust2}$', r'$zf_1$', r'$zf_2$', r'$zf_3$', r'$zf_4$', r'$zf_5$', r'$\mathrm{M_s}$', r'$\mathrm{f_{agn}}$', r'$\mathrm{agn_\tau}$', r'$\mathrm{dust_{ratio}}$', r'$\mathrm{dust_{index}}$' ] #math_parnames = [r'$\mathrm{M_s}$', r'$\mathrm{log(Z_\odot)}$', #r'$\mathrm{dust2}$', r'$\mathrm{t_{age}}$', r'$\mathrm{log(\tau)}$'] # Fix labels for corner plot and # Figure out ranges for corner plot corner_range = [] for d in range(ndim): # Get corner estimate and errors cq = corner.quantile(x=samples[:, d], q=[0.16, 0.5, 0.84]) low_err = cq[1] - cq[0] up_err = cq[2] - cq[1] # Decide the padding for the plot range # depending on how large the error is relative # to the central estimate. if low_err * 2.5 >= cq[1]: sigma_padding_low = 1.2 else: sigma_padding_low = 3.0 if up_err * 2.5 >= cq[1]: sigma_padding_up = 1.2 else: sigma_padding_up = 3.0 low_lim = cq[1] - sigma_padding_low * low_err up_lim = cq[1] + sigma_padding_up * up_err corner_range.append((low_lim, up_lim)) # Print estimate to screen if 'mass' in parnames[d]: pn = '{:.3e}'.format(cq[1]) pnu = '{:.3e}'.format(up_err) pnl = '{:.3e}'.format(low_err) else: pn = '{:.3f}'.format(cq[1]) pnu = '{:.3f}'.format(up_err) pnl = '{:.3f}'.format(low_err) print(parnames[d], ": ", pn, "+", pnu, "-", pnl) # Corner plot cornerfig = corner.corner(samples, quantiles=[0.16, 0.5, 0.84], labels=math_parnames, label_kwargs={"fontsize": 14}, range=corner_range, smooth=0.5, smooth1d=0.5) # loop over all axes *again* and set title # because it won't let me set the # format for soem titles separately # Looping has to be done twice because corner # plotting has to be done to get the figure. corner_axes = np.array(cornerfig.axes).reshape((ndim, ndim)) for d in range(ndim): # Get corner estimate and errors cq = corner.quantile(x=samples[:, d], q=[0.16, 0.5, 0.84]) low_err = cq[1] - cq[0] up_err = cq[2] - cq[1] ax_c = corner_axes[d, d] if 'mass' in parnames[d]: ax_c.set_title(math_parnames[d] + r"$ \, =\,$" + csn(cq[1], sigfigs=3) + \ r"$\substack{+$" + csn(up_err, sigfigs=3) + r"$\\ -$" + \ csn(low_err, sigfigs=3) + r"$}$", fontsize=11, pad=15) else: ax_c.set_title(math_parnames[d] + r"$ \, =\,$" + r"${:.3f}$".format(cq[1]) + \ r"$\substack{+$" + r"${:.3f}$".format(up_err) + r"$\\ -$" + \ r"${:.3f}$".format(low_err) + r"$}$", fontsize=11, pad=10) cornerfig.savefig(adap_dir + 'corner_' + field + '_' + \ str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight') sys.exit(0) # maximum a posteriori (of the locations visited by the MCMC sampler) pmax = np.argmax(result['lnprobability']) if results_type == "emcee": p, q = np.unravel_index(pmax, result['lnprobability'].shape) theta_max = result['chain'][p, q, :].copy() else: theta_max = result["chain"][pmax, :] #print('Optimization value: {}'.format(theta_best)) #print('MAP value: {}'.format(theta_max)) # make SED plot for MAP model and some random model # randomly chosen parameters from chain randint = np.random.randint if results_type == "emcee": theta = result['chain'][randint(nwalkers), randint(niter)] else: theta = result["chain"][randint(len(result["chain"]))] # generate models a = 1.0 + model.params.get('zred', 0.0) # cosmological redshifting # photometric effective wavelengths wphot = obs["phot_wave"] # spectroscopic wavelengths if obs["wavelength"] is None: # *restframe* spectral wavelengths, since obs["wavelength"] is None wspec = sps.wavelengths wspec *= a #redshift them else: wspec = obs["wavelength"] # sps = reader.get_sps(result) # this works if using parameter files mspec, mphot, mextra = model.mean_model(theta, obs, sps=sps) mspec_map, mphot_map, _ = model.mean_model(theta_max, obs, sps=sps) # establish bounds xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8 ymin, ymax = obs["maggies"].min() * 0.8, obs["maggies"].max() / 0.4 # Make plot of data and model fig3 = plt.figure(figsize=(9, 4)) ax3 = fig3.add_subplot(111) ax3.set_xlabel(r'$\mathrm{\lambda\ [\AA]}$', fontsize=15) #ax3.set_ylabel(r'$\mathrm{f_\lambda\ [erg\, s^{-1}\, cm^{-2}\, \AA^{-1}]}$', fontsize=15) ax3.set_ylabel(r'$\mathrm{Flux\ Density\ [maggies]}$', fontsize=15) ax3.loglog(wspec, mspec, label='Model spectrum (random draw)', lw=0.7, color='navy', alpha=0.7) ax3.loglog(wspec, mspec_map, label='Model spectrum (MAP)', lw=0.7, color='green', alpha=0.7) ax3.errorbar(wphot, mphot, label='Model photometry (random draw)', marker='s', markersize=10, alpha=0.8, ls='', lw=3, markerfacecolor='none', markeredgecolor='blue', markeredgewidth=3) ax3.errorbar(wphot, mphot_map, label='Model photometry (MAP)', marker='s', markersize=10, alpha=0.8, ls='', lw=3, markerfacecolor='none', markeredgecolor='green', markeredgewidth=3) ax3.errorbar(wphot, obs['maggies'], yerr=obs['maggies_unc'], label='Observed photometry', ecolor='red', marker='o', markersize=10, ls='', lw=3, alpha=0.8, markerfacecolor='none', markeredgecolor='red', markeredgewidth=3) # plot transmission curves for f in obs['filters']: w, t = f.wavelength.copy(), f.transmission.copy() t = t / t.max() t = 10**(0.2 * (np.log10(ymax / ymin))) * t * ymin ax3.loglog(w, t, lw=3, color='gray', alpha=0.7) ax3.set_xlim([xmin, xmax]) ax3.set_ylim([ymin, ymax]) ax3.legend(loc='best', fontsize=11) fig3.savefig(adap_dir + 'sedplot_' + field + '_' + str(galaxy_seq) + '.pdf', dpi=200, bbox_inches='tight') plt.clf() plt.cla() plt.close() return None
# ------------------------- # Output HDF5 (and pickles if asked for) # ------------------------- print("Writing to {}".format(outroot)) if rp.get("output_pickles", False): write_results.write_pickles(rp, model, obsdat, esampler, guesses, outroot=outroot, toptimize=0, tsample=edur, sampling_initial_center=initial_center) if hfile is None: hfile = hfilename write_results.write_hdf5(hfile, rp, model, obsdat, esampler, guesses, toptimize=0, tsample=edur, sampling_initial_center=initial_center) try: hfile.close() except: pass halt('Finished')
tstart = time.time() out = fitting.run_emcee_sampler(lnprobfn, initial_center, model, postkwargs=postkwargs, initial_prob=initial_prob, pool=pool, hdf5=hfile, **rp) esampler, burn_p0, burn_prob0 = out edur = time.time() - tstart if rp['verbose']: print('done emcee in {0}s'.format(edur)) # ------------------------- # Output pickles (and HDF5) # ------------------------- print("Writing to {}".format(outroot)) write_results.write_pickles(rp, model, obsdat, esampler, guesses, outroot=outroot, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center, post_burnin_center=burn_p0, post_burnin_prob=burn_prob0) if hfile is None: hfile = hfilename write_results.write_hdf5(hfile, rp, model, obsdat, esampler, guesses, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center, post_burnin_center=burn_p0, post_burnin_prob=burn_prob0) try: hfile.close() except: pass halt('Finished')
def run_prospector(tde_name, path, z, withmpi, n_cores, gal_ebv, show_figs=True, init_theta=None, n_walkers=None, n_inter=None, n_burn=None, read_only=False): os.chdir(os.path.join(path, tde_name, 'host')) if init_theta is None: init_theta = [1e10, -1.0, 6, 0.5] if n_walkers is None: n_walkers = 100 if n_inter is None: n_inter = 1000 if n_burn is None: n_burn = [500] obs, sps, model, run_params = configure(tde_name, path, z, init_theta, n_walkers, n_inter, n_burn, gal_ebv) if not withmpi: n_cores = 1 if not read_only: print("Initial guess: {}".format(model.initial_theta)) print('Sampling the the SPS grid..') if withmpi & ('logzsol' in model.free_params): dummy_obs = dict(filters=None, wavelength=None) logzsol_prior = model.config_dict["logzsol"]['prior'] lo, hi = logzsol_prior.range logzsol_grid = np.around(np.arange(lo, hi, step=0.1), decimals=2) sps.update(**model.params) # make sure we are caching the correct IMF / SFH / etc for logzsol in logzsol_grid: model.params["logzsol"] = np.array([logzsol]) _ = model.predict(model.theta, obs=dummy_obs, sps=sps) print('Done') from functools import partial lnprobfn_fixed = partial(lnprobfn, sps=sps) print('Starting posterior emcee sampling..') if withmpi: from multiprocessing import Pool from multiprocessing import cpu_count with Pool(int(n_cores)) as pool: nprocs = n_cores output = fit_model(obs, model, sps, pool=pool, queue_size=nprocs, lnprobfn=lnprobfn_fixed, **run_params) else: output = fit_model(obs, model, sps, lnprobfn=lnprobfn_fixed, **run_params) # output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print('done emcee in {0}s'.format(output["sampling"][1])) if os.path.exists("prospector_result.h5"): os.system('rm prospector_result.h5') hfile = "prospector_result.h5" writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) print('Finished') # Loading results file result, _, _ = reader.results_from("prospector_result.h5", dangerous=False) # Finding the Maximum A Posteriori (MAP) model imax = np.argmax(result['lnprobability']) i, j = np.unravel_index(imax, result['lnprobability'].shape) theta_max = result['chain'][i, j, :].copy() # saving results save_results(result, model, obs, sps, theta_max, tde_name, path, n_walkers, n_inter, n_burn, n_cores) print('MAP value: {}'.format(theta_max)) fit_plot = plot_resulting_fit(tde_name, path) try: os.mkdir(os.path.join(path, tde_name, 'plots')) except: pass fit_plot.savefig(os.path.join(path, tde_name, 'plots', tde_name + '_host_fit.png'), bbox_inches='tight', dpi=300) if show_figs: plt.show() corner_fig = corner_plot(result) corner_fig.savefig(os.path.join(path, tde_name, 'plots', tde_name + '_cornerplot.png'), bbox_inches='tight', dpi=300) if show_figs: plt.show() os.chdir(path)
if rp['verbose']: print('done nestle in {0}s'.format(dur)) # ------------------------- # Output pickles (and HDF5) # ------------------------- print("Writing to {}".format(outroot)) # Write the nestle Result object as a pickle import pickle with open(outroot + '_nmc.pkl', 'w') as f: pickle.dump(nestleout, f) partext = write_results.paramfile_string(**rp) # Write the model as a pickle write_results.write_model_pickle(outroot + '_model', model, powell=None, paramfile_text=partext) # Write HDF5 if hfile is None: hfile = hfilename write_results.write_hdf5(hfile, rp, model, obsdat, nestleout, None, tsample=dur) print('Finished')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--prefix', type=str, default='redmapper_sdssphot', help='String to prepend to I/O files.') parser.add_argument('--nthreads', type=int, default=16, help='Number of cores to use concurrently.') parser.add_argument('--first', type=int, default=0, help='Index of first object to fit.') parser.add_argument('--last', type=int, default=None, help='Index of last object to fit.') parser.add_argument('--seed', type=int, default=1, help='Random number seed.') parser.add_argument('--build-sample', action='store_true', help='Build the sample.') parser.add_argument('--dofit', action='store_true', help='Run prospector!') parser.add_argument( '--refit', action='store_true', help='Refit even if the prospector output files exist.') parser.add_argument('--qaplots', action='store_true', help='Make some neat plots.') parser.add_argument('--verbose', action='store_true', help='Be loquacious.') args = parser.parse_args() # Specify the run parameters and initialize the SPS object. run_params = { 'prefix': args.prefix, 'verbose': args.verbose, 'seed': args.seed, # initial optimization choices (nmin is only for L-M optimization) 'do_levenburg': True, 'do_powell': False, 'do_nelder_mead': False, 'nmin': np.max((10, args.nthreads)), # emcee fitting parameters monte carlo markov chain samples parameters ft. certain technique 'nwalkers': 128, 'nburn': [32, 32, 64], 'niter': 256, # 512, 'interval': 0.1, # save 10% of the chains at a time # Nestle fitting parameters 'nestle_method': 'single', 'nestle_npoints': 200, 'nestle_maxcall': int(1e6), # Multiprocessing 'nthreads': args.nthreads, # SPS initialization parameters 'compute_vega_mags': False, 'vactoair_flag': False, # use wavelengths in air 'zcontinuous': 1, # interpolate in metallicity } rand = np.random.RandomState(args.seed) if not args.build_sample: t0 = time() print('Initializing CSPSpecBasis...') sps = CSPSpecBasis(zcontinuous=run_params['zcontinuous'], compute_vega_mags=run_params['compute_vega_mags'], vactoair_flag=run_params['vactoair_flag']) print('...took {:.1f} seconds.'.format(time() - t0)) if args.build_sample: # READ OUR SHIT INSTEAD. # Read the parent redmapper catalog, choose a subset of objects and # write out. cat = read_redmapper() # Choose objects with masses from iSEDfit, Kravtsov, and pymorph, but # for now just pick three random galaxies. #these = np.arange(2) # [300, 301, 302] these = np.arange(100) # [300, 301, 302] print('Selecting {} galaxies.'.format(len(these))) out = cat[these] #out = cat[:200] outfile = os.path.join(datadir(), '{}_sample.fits'.format(run_params['prefix'])) print('Writing {}'.format(outfile)) fitsio.write(outfile, out, clobber=True) if args.dofit: import h5py import emcee from prospect import fitting from prospect.io import write_results # MAKE THIS SHIT WORK WITH OUR SHIT TOO ft getobs # Read the parent sample and loop on each object. cat = read_parent( prefix=run_params['prefix']) #, first=args.first, last=args.last) for ii, obj in enumerate(cat): objprefix = '{0:05}'.format(obj['ISEDFIT_ID']) print('Working on object {}/{} with prefix {}.'.format( ii + 1, len(cat), objprefix)) # Check for the HDF5 output file / fitting results -- outroot = os.path.join( datadir(), '{}_{}'.format(run_params['prefix'], objprefix)) hfilename = os.path.join( datadir(), '{}_{}_mcmc.h5'.format(run_params['prefix'], objprefix)) if os.path.isfile(hfilename): if args.refit: os.remove(hfilename) else: print('Prospector fitting results {} exist; skipping.'. format(hfilename)) continue # Grab the photometry for this object and then initialize the priors # and the SED model. obs = getobs(obj) model = load_model(zred=obs['zred'], seed=args.seed) # Get close to the right answer doing a simple minimization. if run_params['verbose']: print('Free parameters: {}'.format(model.free_params)) print('Initial parameter values: {}'.format( model.initial_theta)) initial_theta = model.rectify_theta( model.initial_theta) # make zeros tiny numbers if bool(run_params.get('do_powell', True)): tstart = time() # optimization options powell_opt = { 'ftol': run_params.get('ftol', 0.5e-5), 'xtol': 1e-6, 'maxfev': run_params.get('maxfev', 5000) } chi2args = (model, obs, sps, run_params['verbose'] ) # extra arguments for chisqfn guesses, pinit = fitting.pminimize( chisqfn, initial_theta, args=chi2args, model=model, method='powell', opts=powell_opt, pool=pool, nthreads=run_params['nthreads']) best = np.argmin([p.fun for p in guesses]) # Hack to recenter values outside the parameter bounds! initial_center = fitting.reinitialize( guesses[best].x, model, edge_trunc=run_params.get('edge_trunc', 0.1)) initial_prob = -1 * guesses[best]['fun'] pdur = time() - tstart if run_params['verbose']: print('Powell initialization took {:.1f} seconds.'.format( pdur)) print('Best Powell guesses: {}'.format(initial_center)) print('Initial probability: {}'.format(initial_prob)) elif bool(run_params.get('do_nelder_mead', True)): from scipy.optimize import minimize tstart = time() chi2args = (model, obs, sps, run_params['verbose'] ) # extra arguments for chisqfn guesses = minimize(chisqfn, initial_theta, args=chi2args, method='nelder-mead') pdur = time() - tstart # Hack to recenter values outside the parameter bounds! initial_center = fitting.reinitialize( guesses.x, model, edge_trunc=run_params.get('edge_trunc', 0.1)) initial_prob = -1 * guesses['fun'] if run_params['verbose']: print('Nelder-Mead initialization took {:.1f} seconds.'. format(pdur)) print('Best guesses: {}'.format(initial_center)) print('Initial probability: {}'.format(initial_prob)) elif bool(run_params.get('do_levenburg', True)): tstart = time() nmin = run_params['nmin'] chi2args = (model, obs, sps) pinitial = fitting.minimizer_ball(initial_theta, nmin, model, seed=run_params['seed']) lsargs = list() for pinit in pinitial: lsargs.append((chivecfn, pinit, chi2args)) if run_params['nthreads'] > 1: p = multiprocessing.Pool(run_params['nthreads']) guesses = p.map(_doleast_squares, lsargs) p.close() else: guesses = list() for lsargs1 in lsargs: guesses.append(_doleast_squares(lsargs1)) chisq = [np.sum(r.fun**2) for r in guesses] best = np.argmin(chisq) initial_prob = -np.log(chisq[best] / 2) #initial_center = guesses[best].x initial_center = fitting.reinitialize( guesses[best].x, model, edge_trunc=run_params.get('edge_trunc', 0.1)) pdur = time() - tstart if run_params['verbose']: print( 'Levenburg-Marquardt initialization took {:.1f} seconds.' .format(pdur)) print('Best guesses: {}'.format(initial_center)) print('Initial probability: {}'.format(initial_prob)) from prospector_plot_utilities import bestfit_sed fig = bestfit_sed(obs, theta=initial_center, sps=sps, model=model) fig.savefig('test.png') else: if run_params['verbose']: print('Skipping initial minimization.') guesses = None pdur = 0.0 initial_center = initial_theta.copy() initial_prob = None # Write some basic info to the HDF5 file-- hfile = h5py.File(hfilename, 'a') write_results.write_h5_header(hfile, run_params, model) write_results.write_obs_to_h5(hfile, obs) if run_params['verbose']: print('Started emcee sampling on {}'.format(asctime())) tstart = time() out = fitting.run_emcee_sampler(lnprobfn, initial_center, model, verbose=run_params['verbose'], nthreads=run_params['nthreads'], nwalkers=run_params['nwalkers'], nburn=run_params['nburn'], niter=run_params['niter'], prob0=initial_prob, hdf5=hfile, postargs=(model, obs, sps), pool=pool) esampler, burn_p0, burn_prob0 = out del out edur = time() - tstart if run_params['verbose']: print('Finished emcee sampling in {:.2f} minutes.'.format( edur / 60.0)) # Update the HDF5 file with the results. write_results.write_pickles(run_params, model, obs, esampler, guesses, outroot=outroot, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center, post_burnin_center=burn_p0, post_burnin_prob=burn_prob0) write_results.write_hdf5(hfilename, run_params, model, obs, esampler, guesses, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center, post_burnin_center=burn_p0, post_burnin_prob=burn_prob0) hfile.close() if args.qaplots: import h5py from prospect.io import read_results from prospector_plot_utilities import param_evol, subtriangle, bestfit_sed # Read the parent sample and loop on each object. cat = read_parent( prefix=run_params['prefix']) #, first=args.first, last=args.last) for obj in cat: objprefix = '{0:05}'.format(obj['ISEDFIT_ID']) # Grab the emcee / prospector outputs. h5file = os.path.join( datadir(), '{}_{}_mcmc.h5'.format(run_params['prefix'], objprefix)) if not os.path.isfile(h5file): print('HDF5 file {} not found; skipping.'.format(h5file)) continue print('Reading {}'.format(h5file)) results, guesses, model = read_results.results_from(h5file) nwalkers, niter, nparams = results['chain'][:, :, :].shape # -------------------------------------------------- # Figure: Generate the best-fitting SED. qafile = os.path.join( datadir(), '{}_{}_sed.png'.format(args.prefix, objprefix)) print('Writing {}'.format(qafile)) fig = bestfit_sed(results['obs'], chain=results['chain'], lnprobability=results['lnprobability'], sps=sps, model=model, seed=results['run_params']['seed']) fig.savefig(qafile) # -------------------------------------------------- # Figure: Visualize a random sampling of the MCMC chains. qafile = os.path.join( datadir(), '{}_{}_chains.png'.format(args.prefix, objprefix)) print('Writing {}'.format(qafile)) thesechains = rand.choice(nwalkers, size=int(0.3 * nwalkers), replace=False) fig = param_evol(results, chains=thesechains) fig.savefig(qafile) # -------------------------------------------------- # Figure: Generate a corner/triangle plot of the free parameters. qafile = os.path.join( datadir(), '{}_{}_corner.png'.format(args.prefix, objprefix)) print('Writing {}'.format(qafile)) fig = subtriangle(results, thin=2) fig.savefig(qafile)
# ----------------------- if rp['verbose']: print('emcee sampling...') tstart = time.time() out = fitting.restart_emcee_sampler(lnprobfn, initial_positions, postkwargs=postkwargs, pool=pool, hdf5=hfile, **rp) esampler = out edur = time.time() - tstart if rp['verbose']: print('done emcee in {0}s'.format(edur)) # ------------------------- # Output HDF5 (and pickles if asked for) # ------------------------- print("Writing to {}".format(outroot)) if rp.get("output_pickles", False): write_results.write_pickles(rp, model, obsdat, esampler, guesses, outroot=outroot, toptimize=0, tsample=edur, sampling_initial_center=initial_center) if hfile is None: hfile = hfilename write_results.write_hdf5(hfile, rp, model, obsdat, esampler, guesses, toptimize=0, tsample=edur, sampling_initial_center=initial_center) try: hfile.close() except: pass halt('Finished')
hfile = None # ------- # Sample # ------- if rp['verbose']: print('dynesty sampling...') tstart = time.time() # time it dynestyout = fitting.run_dynesty_sampler(lnprobfn, prior_transform, model.ndim, pool=pool, queue_size=nprocs, stop_function=stopping_function, wt_function=weight_function, **rp) ndur = time.time() - tstart print('done dynesty in {0}s'.format(ndur)) # ------------------------- # Output pickles (and HDF5) # ------------------------- partext = write_results.paramfile_string(**rp) # Write the model as a pickle write_results.write_model_pickle(outroot + '_model', model, powell=None, paramfile_text=partext) # Write HDF5 if hfile is None: hfile = hfilename write_results.write_hdf5(hfile, rp, model, obs, dynestyout, None, tsample=ndur)
def main(): vers = (np.__version__, scipy.__version__, h5py.__version__, fsps.__version__, prospect.__version__) print( "Numpy: {}\nScipy: {}\nH5PY: {}\nFSPS: {}\nProspect: {}".format(*vers)) # And we will store some meta-parameters that control the input arguments to this method: run_params = {} run_params["snr"] = 10.0 run_params["ldist"] = 10.0 # Build the obs dictionary using the meta-parameters obs = build_obs(**run_params) # Look at the contents of the obs dictionary print("Obs Dictionary Keys:\n\n{}\n".format(obs.keys())) print("--------\nFilter objects:\n") print(obs["filters"]) # --- Plot the Data ---- """ # This is why we stored these... wphot = obs["phot_wave"] # establish bounds xmin, xmax = np.min(wphot)*0.8, np.max(wphot)/0.8 ymin, ymax = obs["maggies"].min()*0.8, obs["maggies"].max()/0.4 fig = plt.figure(figsize=(10,5)) ax = fig.add_subplot(111) # plot all the data ax.plot(wphot, obs['maggies'], label='All observed photometry', marker='o', markersize=12, alpha=0.8, ls='', lw=3, color='slateblue') # overplot only the data we intend to fit mask = obs["phot_mask"] ax.errorbar(wphot[mask], obs['maggies'][mask], yerr=obs['maggies_unc'][mask], label='Photometry to fit', marker='o', markersize=8, alpha=0.8, ls='', lw=3, ecolor='tomato', markerfacecolor='none', markeredgecolor='tomato', markeredgewidth=3) # plot Filters for f in obs['filters']: w, t = f.wavelength.copy(), f.transmission.copy() t = t / t.max() t = 10**(0.2*(np.log10(ymax/ymin)))*t * ymin ax.loglog(w, t, lw=3, color='gray', alpha=0.7) # prettify ax.set_xlabel('Wavelength [A]') ax.set_ylabel('Flux Density [maggies]') ax.set_xlim([xmin, xmax]) ax.set_ylim([ymin, ymax]) ax.set_xscale("log") ax.set_yscale("log") ax.legend(loc='best', fontsize=20) fig.tight_layout() plt.show() """ # ---------------------------- from prospect.models import priors mass_param = { "name": "mass", # The mass parameter here is a scalar, so it has N=1 "N": 1, # We will be fitting for the mass, so it is a free parameter "isfree": True, # This is the initial value. For fixed parameters this is the # value that will always be used. "init": 1e8, # This sets the prior probability for the parameter "prior": priors.LogUniform(mini=1e6, maxi=1e12), # this sets the initial dispersion to use when generating # clouds of emcee "walkers". It is not required, but can be very helpful. "init_disp": 1e6, # this sets the minimum dispersion to use when generating #clouds of emcee "walkers". It is not required, but can be useful if # burn-in rounds leave the walker distribution too narrow for some reason. "disp_floor": 1e6, # This is not required, but can be helpful "units": "solar masses formed", } from prospect.models.templates import TemplateLibrary # Look at all the prepackaged parameter sets TemplateLibrary.show_contents() # Check a specific model. # This shows the defaults and params for the model. # parametric_sfh in this case (delayed-tau) # This is the one we will use TemplateLibrary.describe("parametric_sfh") run_params["object_redshift"] = 0.0 run_params["fixed_metallicity"] = None run_params["add_duste"] = True model = build_model(**run_params) print(model) print("\nInitial free parameter vector theta:\n {}\n".format(model.theta)) print("Initial parameter dictionary:\n{}".format(model.params)) run_params["zcontinuous"] = 1 sps = build_sps(**run_params) # Generate the model SED at the initial value of theta theta = model.theta.copy() initial_spec, initial_phot, initial_mfrac = model.sed(theta, obs=obs, sps=sps) #title_text = ','.join(["{}={}".format(p, model.params[p][0]) # for p in model.free_params]) a = 1.0 + model.params.get('zred', 0.0) # cosmological redshifting # photometric effective wavelengths wphot = obs["phot_wave"] # spectroscopic wavelengths if obs["wavelength"] is None: # *restframe* spectral wavelengths, since obs["wavelength"] is None wspec = sps.wavelengths wspec *= a #redshift them else: wspec = obs["wavelength"] # establish bounds xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8 temp = np.interp(np.linspace(xmin, xmax, 10000), wspec, initial_spec) ymin, ymax = temp.min() * 0.8, temp.max() / 0.4 """ fig = plt.figure(figsize=(10,5)) ax = fig.add_subplot(111) # plot model + data ax.loglog(wspec, initial_spec, label='Model spectrum', lw=0.7, color='navy', alpha=0.7) ax.errorbar(wphot, initial_phot, label='Model photometry', marker='s',markersize=10, alpha=0.8, ls='', lw=3, markerfacecolor='none', markeredgecolor='blue', markeredgewidth=3) ax.errorbar(wphot, obs['maggies'], yerr=obs['maggies_unc'], label='Observed photometry', marker='o', markersize=10, alpha=0.8, ls='', lw=3, ecolor='red', markerfacecolor='none', markeredgecolor='red', markeredgewidth=3) ax.set_title(title_text) # plot Filters for f in obs['filters']: w, t = f.wavelength.copy(), f.transmission.copy() t = t / t.max() t = 10**(0.2*(np.log10(ymax/ymin)))*t * ymin ax.loglog(w, t, lw=3, color='gray', alpha=0.7) # prettify ax.set_xlabel('Wavelength [A]') ax.set_ylabel('Flux Density [maggies]') ax.set_xlim([xmin, xmax]) ax.set_ylim([ymin, ymax]) ax.legend(loc='best', fontsize=20) fig.tight_layout() plt.show() """ verbose = True run_params["verbose"] = verbose # Here we will run all our building functions obs = build_obs(**run_params) sps = build_sps(**run_params) model = build_model(**run_params) # For fsps based sources it is useful to # know which stellar isochrone and spectral library # we are using print(sps.ssp.libraries) # --- start minimization ---- run_params["dynesty"] = False run_params["emcee"] = False run_params["optimize"] = True run_params["min_method"] = 'lm' # We'll start minimization from "nmin" separate places, # the first based on the current values of each parameter and the # rest drawn from the prior. Starting from these extra draws # can guard against local minima, or problems caused by # starting at the edge of a prior (e.g. dust2=0.0) run_params["nmin"] = 2 output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print("Done optmization in {}s".format(output["optimization"][1])) print(model.theta) (results, topt) = output["optimization"] # Find which of the minimizations gave the best result, # and use the parameter vector for that minimization ind_best = np.argmin([r.cost for r in results]) print(ind_best) theta_best = results[ind_best].x.copy() print(theta_best) # generate model prediction = model.mean_model(theta_best, obs=obs, sps=sps) pspec, pphot, pfrac = prediction """ fig = plt.figure(figsize=(10,5)) ax = fig.add_subplot(111) # plot Data, best fit model, and old models ax.loglog(wspec, initial_spec, label='Old model spectrum', lw=0.7, color='gray', alpha=0.5) ax.errorbar(wphot, initial_phot, label='Old model Photometry', marker='s', markersize=10, alpha=0.6, ls='', lw=3, markerfacecolor='none', markeredgecolor='gray', markeredgewidth=3) ax.loglog(wspec, pspec, label='Model spectrum', lw=0.7, color='slateblue', alpha=0.7) ax.errorbar(wphot, pphot, label='Model photometry', marker='s', markersize=10, alpha=0.8, ls='', lw=3, markerfacecolor='none', markeredgecolor='slateblue', markeredgewidth=3) ax.errorbar(wphot, obs['maggies'], yerr=obs['maggies_unc'], label='Observed photometry', marker='o', markersize=10, alpha=0.8, ls='', lw=3, ecolor='tomato', markerfacecolor='none', markeredgecolor='tomato', markeredgewidth=3) # plot filter transmission curves for f in obs['filters']: w, t = f.wavelength.copy(), f.transmission.copy() t = t / t.max() t = 10**(0.2*(np.log10(ymax/ymin)))*t * ymin ax.loglog(w, t, lw=3, color='gray', alpha=0.7) # Prettify ax.set_xlabel('Wavelength [A]') ax.set_ylabel('Flux Density [maggies]') ax.set_xlim([xmin, xmax]) ax.set_ylim([ymin, ymax]) ax.legend(loc='best', fontsize=20) fig.tight_layout() plt.show() """ # Set this to False if you don't want to do another optimization # before emcee sampling (but note that the "optimization" entry # in the output dictionary will be (None, 0.) in this case) # If set to true then another round of optmization will be performed # before sampling begins and the "optmization" entry of the output # will be populated. run_params["optimize"] = False run_params["emcee"] = True run_params["dynesty"] = False # Number of emcee walkers run_params["nwalkers"] = 128 # Number of iterations of the MCMC sampling run_params["niter"] = 512 # Number of iterations in each round of burn-in # After each round, the walkers are reinitialized based on the # locations of the highest probablity half of the walkers. run_params["nburn"] = [16, 32, 64] print("Now running with Emcee.") output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print('done emcee in {0}s'.format(output["sampling"][1])) hfile = "demo_emcee_mcmc.h5" writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) print('Finished') # ------------------- print("Now running with Dynesty.") run_params["dynesty"] = True run_params["optmization"] = False run_params["emcee"] = False run_params["nested_method"] = "rwalk" run_params["nlive_init"] = 400 run_params["nlive_batch"] = 200 run_params["nested_dlogz_init"] = 0.05 run_params["nested_posterior_thresh"] = 0.05 run_params["nested_maxcall"] = int(1e7) output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print('done dynesty in {0}s'.format(output["sampling"][1])) hfile = "demo_dynesty_mcmc.h5" writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) print('Finished') # ------------------------- # Visualizing results results_type = "dynesty" # "emcee" | "dynesty" # grab results (dictionary), the obs dictionary, and our corresponding models # When using parameter files set `dangerous=True` result, obs, _ = reader.results_from( "demo_{}_mcmc.h5".format(results_type), dangerous=False) #The following commented lines reconstruct the model and sps object, # if a parameter file continaing the `build_*` methods was saved along with the results #model = reader.get_model(result) #sps = reader.get_sps(result) # let's look at what's stored in the `result` dictionary print(result.keys()) if results_type == "emcee": chosen = np.random.choice(result["run_params"]["nwalkers"], size=10, replace=False) tracefig = reader.traceplot(result, figsize=(20, 10), chains=chosen) else: tracefig = reader.traceplot(result, figsize=(20, 10)) # maximum a posteriori (of the locations visited by the MCMC sampler) imax = np.argmax(result['lnprobability']) if results_type == "emcee": i, j = np.unravel_index(imax, result['lnprobability'].shape) theta_max = result['chain'][i, j, :].copy() thin = 5 else: theta_max = result["chain"][imax, :] thin = 1 print('Optimization value: {}'.format(theta_best)) print('MAP value: {}'.format(theta_max)) cornerfig = reader.subcorner(result, start=0, thin=thin, truths=theta_best, fig=plt.subplots(5, 5, figsize=(8, 8))[0]) plt.show() return None
# - Parser with default arguments - parser = prospect_args.get_parser() # - Add custom arguments - parser.add_argument('--zred', type=float, default=0.1, help="redshift for the model") parser.add_argument('--add_neb', action="store_true", help="If set, add nebular emission in the model") parser.add_argument('--add_noise', action="store_true", help="If set, noise up the mock") parser.add_argument('--snr', type=float, default=20, help="S/N ratio for the mock photometry") args = parser.parse_args() run_params = vars(args) obs, model, sps, noise = build_all(**run_params) run_params["param_file"] = __file__ hfile = setup_h5(model=model, obs=obs, **run_params) output = fit_model(obs, model, sps, noise, **run_params) writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) try: hfile.close() except(AttributeError): pass
# ------- # Sample # ------- if rp['verbose']: print('emcee sampling...') tstart = time.time() out = fitting.run_emcee_sampler(lnprobfn, initial_center, model, postkwargs=postkwargs, initial_prob=initial_prob, pool=pool, hdf5=hfile, **rp) esampler, burn_p0, burn_prob0 = out edur = time.time() - tstart if rp['verbose']: print('done emcee in {0}s'.format(edur)) # ------------------------- # Output pickles (and HDF5) # ------------------------- write_results.write_pickles(rp, model, obsdat, esampler, powell_guesses, outroot=outroot, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center, post_burnin_center=burn_p0, post_burnin_prob=burn_prob0) if hfile is None: hfile = hfilename write_results.write_hdf5(hfile, rp, model, obsdat, esampler, powell_guesses, toptimize=pdur, tsample=edur, sampling_initial_center=initial_center, post_burnin_center=burn_p0, post_burnin_prob=burn_prob0) halt('Finished')
def run_prospector(name_z): import time, sys, os import h5py import numpy as np import scipy from matplotlib.pyplot import * #%matplotlib inline import fsps import sedpy import prospect import emcee import astropy import math from prospect.fitting import fit_model from prospect.fitting import lnprobfn from func import * import matplotlib.pyplot as plt # --- Output files --- name = name_z[0] z = name_z[1] path_input = '/home/kk/work/data/red_magic/' + name path = '/home/kk/work/result/red_magic/' + name[:-4] '''try: os.mkdir(path) except: pass fe = open(path+'/para.txt','w')''' # --- Set up the essential parameter --- data_list = [ 'sdss_u', 'sdss_g', 'sdss_r', 'sdss_i', 'galex_fuv', 'galex_nuv', 'pan-starrs_ps1_i', 'pan-starrs_ps1_r', 'pan-starrs_ps1_g', 'pan-starrs_ps1_y', 'pan-starrs_ps1_z', 'sdss_z', 'ukidss_j', 'ukidss_h', 'ukidss_k', 'wise_w1', 'spitzer_irac_3.6', '_=3.6um', 'spitzer_irac_4.5', '_=4.5um', 'wise_w2', 'spitzer_irac_5.8', 'spitzer_irac_8.0', 'wise_w3', 'wise_w4', 'spitzer_mips_24', '2mass_h', '2mass_ks', '2mass_j', 'gaia_gaia2_gbp', 'gaia_gaia2_g', 'gaia_gaia2_grp', 'gaia_g', 'ukidss_y', 'vista_j', 'vista_h', 'vista_ks' ] run_params = {} run_params["snr"] = 10.0 run_params["object_redshift"] = z run_params["fixed_metallicity"] = None run_params["add_duste"] = True # --- Start build up the model --- fit_data = load_data(path_input, data_list) obs = build_obs(fit_data, **run_params) sps = build_sps(**run_params) model = build_model(**run_params) #fe.write(str(obs)) #print(fit_data) # --- Draw initial data plot --- wphot = obs["phot_wave"] # establish bounds xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8 ymin, ymax = obs["maggies"].min() * 0.8, obs["maggies"].max() / 0.4 figure(figsize=(16, 8)) # plot all the data plot(wphot, obs['maggies'], label='All observed photometry', marker='o', markersize=12, alpha=0.8, ls='', lw=3, color='slateblue') # overplot only the data we intend to fit mask = obs["phot_mask"] errorbar(wphot[mask], obs['maggies'][mask], yerr=obs['maggies_unc'][mask], label='Photometry to fit', marker='o', markersize=8, alpha=0.8, ls='', lw=3, ecolor='tomato', markerfacecolor='none', markeredgecolor='tomato', markeredgewidth=3) # prettify xlabel('Wavelength [A]') ylabel('Flux Density [maggies]') xlim([xmin, xmax]) ylim([ymin, ymax]) xscale("log") yscale("log") legend(loc='best', fontsize=20) #tight_layout() #plt.savefig(path+'/input.png') # run minimization and emcee with error handling condition = False while condition is False: try: # --- start minimization ---- run_params["dynesty"] = False run_params["emcee"] = False run_params["optimize"] = True run_params["min_method"] = 'lm' run_params["nmin"] = 5 output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print("Done optmization in {}s".format(output["optimization"][1])) # save theta_best for later use (results, topt) = output["optimization"] ind_best = np.argmin([r.cost for r in results]) theta_best = results[ind_best].x.copy() # --- start emcee --- run_params["optimize"] = False run_params["emcee"] = True run_params["dynesty"] = False run_params["nwalkers"] = 30 run_params["niter"] = 8000 run_params["nburn"] = [300, 800, 1600] output = fit_model(obs, model, sps, lnprobfn=lnprobfn, **run_params) print('done emcee in {0}s'.format(output["sampling"][1])) condition = True # sort out different errors and apply method to proceed except AssertionError as e: # error: sfr cannot be negative if str(e) == 'sfr cannot be negative.': obs = build_obs(fit_data, **run_params) sps = build_sps(**run_params) model = build_model(**run_params) # error: number of residules are less than variables except ValueError as e: if str( e ) == "Method 'lm' doesn't work when the number of residuals is less than the number of variables.": print(name) return # error: residule are not finite in the initial point elif str(e) == "Residuals are not finite in the initial point.": print(name) return # write final results to file from prospect.io import write_results as writer hfile = path + '_emcee.h5' print(hfile) writer.write_hdf5(hfile, run_params, model, obs, output["sampling"][0], output["optimization"][0], tsample=output["sampling"][1], toptimize=output["optimization"][1]) #fe.write(str(obs)) print('Finished') ############################################################################## import prospect.io.read_results as reader results_type = "emcee" # | "dynesty" # grab results (dictionary), the obs dictionary, and our corresponding models # When using parameter files set `dangerous=True` #tem_name = hfile[:-8]+'{}.h5' #result, obs, _ = reader.results_from(tem_name.format(results_type), dangerous=False) result, obs, _ = reader.results_from(hfile, dangerous=False) #The following commented lines reconstruct the model and sps object, # if a parameter file continaing the `build_*` methods was saved along with the results #model = reader.get_model(result) #sps = reader.get_sps(result) # let's look at what's stored in the `result` dictionary #print(result.keys()) # Make plot of data and model if results_type == "emcee": chosen = np.random.choice(result["run_params"]["nwalkers"], size=10, replace=False) tracefig = reader.traceplot(result, figsize=(20, 10), chains=chosen) else: tracefig = reader.traceplot(result, figsize=(20, 10)) #plt.savefig(path+'/trace.png') # maximum a posteriori (of the locations visited by the MCMC sampler) imax = np.argmax(result['lnprobability']) if results_type == "emcee": i, j = np.unravel_index(imax, result['lnprobability'].shape) theta_max = result['chain'][i, j, :].copy() thin = 5 else: theta_max = result["chain"][imax, :] thin = 1 #f.Write('Optimization value: {}'.format(theta_best)) #fe.write(str('MAP value: {}'.format(theta_max))) #fe.close() cornerfig = reader.subcorner(result, start=0, thin=thin, truths=theta_best, fig=subplots(15, 15, figsize=(27, 27))[0]) #plt.savefig(path+'/corner.png') # --- Draw final fitting curve --- a = 1.0 + model.params.get('zred', 0.0) # cosmological redshifting # photometric effective wavelengths wphot = obs["phot_wave"] # spectroscopic wavelengths if obs["wavelength"] is None: # *restframe* spectral wavelengths, since obs["wavelength"] is None wspec = sps.wavelengths wspec *= a #redshift them else: wspec = obs["wavelength"] # Get enssential data to calculate sfr z_fraction = theta_max[5:10] total_mass = theta_max[0] agebins = model.params['agebins'] x = np.zeros(6) for i in range(6): if i == 5: x[i] = (agebins[i][0] + agebins[i][1]) / 2 else: x[i] = (agebins[i][0] + agebins[i + 1][0]) / 2 sfr = prospect.models.transforms.zfrac_to_sfr(total_mass, z_fraction, agebins) # Calculate the 16% and 84% value from emcee results z_fraction1 = [] z_fraction2 = [] z_fraction3 = [] z_fraction4 = [] z_fraction5 = [] for i in range(30): for j in range(3000): z_fraction1.append(result['chain'][i][j][5]) z_fraction2.append(result['chain'][i][j][6]) z_fraction3.append(result['chain'][i][j][7]) z_fraction4.append(result['chain'][i][j][8]) z_fraction5.append(result['chain'][i][j][9]) zerr_1 = [np.percentile(z_fraction1, 16), np.percentile(z_fraction1, 84)] zerr_2 = [np.percentile(z_fraction2, 16), np.percentile(z_fraction2, 84)] zerr_3 = [np.percentile(z_fraction3, 16), np.percentile(z_fraction3, 84)] zerr_4 = [np.percentile(z_fraction4, 16), np.percentile(z_fraction4, 84)] zerr_5 = [np.percentile(z_fraction5, 16), np.percentile(z_fraction5, 84)] # Build upper and lower z_fraction array z_max = np.zeros(5) z_min = np.zeros(5) for i in range(5): z_min[i] = eval('zerr_{}[0]'.format(i + 1)) z_max[i] = eval('zerr_{}[1]'.format(i + 1)) #print(z_min) #print(z_max) # Build new sfr sfr_max = prospect.models.transforms.zfrac_to_sfr(total_mass, z_max, agebins) sfr_min = prospect.models.transforms.zfrac_to_sfr(total_mass, z_min, agebins) # randomly chosen parameters from chain randint = np.random.randint if results_type == "emcee": nwalkers, niter = run_params['nwalkers'], run_params['niter'] # Draw 100 random plots from mcmc figure(figsize=(16, 16)) ax = plt.subplot(211) for i in range(100): # Get data from any random chain theta = result['chain'][randint(nwalkers), randint(niter)] mspec_t, mphot_t, mextra = model.mean_model(theta, obs, sps=sps) # unit conversion to erg/cm^2/s mspec = (mspec_t * 1e-23) * 3e18 / wspec mphot = (mphot_t * 1e-23) * 3e18 / wphot # plot them out loglog(wspec, mspec, lw=0.7, color='grey', alpha=0.3) else: theta = result["chain"][randint(len(result["chain"]))] # now do it again for best fit model # sps = reader.get_sps(result) # this works if using parameter files mspec_map_t, mphot_map_t, _ = model.mean_model(theta_max, obs, sps=sps) # units conversion to erg/cm^2/s mspec_map = (mspec_map_t * 1e-23) * 3e18 / wspec mphot_map = (mphot_map_t * 1e-23) * 3e18 / wphot ob = (obs['maggies'] * 1e-23) * 3e18 / wphot ob_unc = (obs['maggies_unc'] * 1e-23) * 3e18 / wphot # calculate reduced chi^2 chi = 0 for i in range(len(wphot)): var = ob_unc[i] chi += (mphot_map[i] - ob[i])**2 / var**2 # Make plot of best fit model loglog(wspec, mspec_map, label='Model spectrum (MAP)', lw=0.7, color='green', alpha=0.7) errorbar(wphot, mphot_map, label='Model photometry (MAP)', marker='s', markersize=10, alpha=0.8, ls='', lw=3, markerfacecolor='none', markeredgecolor='green', markeredgewidth=3) errorbar(wphot, ob, yerr=ob_unc, label='Observed photometry', ecolor='red', marker='o', markersize=10, ls='', lw=3, alpha=0.8, markerfacecolor='none', markeredgecolor='red', markeredgewidth=3) text(0, 1, 'Chi-squared/Nphot =' + str(chi / len(wphot)), horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) # Setup bound xmin, xmax = np.min(wphot) * 0.6, np.max(wphot) / 0.8 temp = np.interp(np.linspace(xmin, xmax, 10000), wphot, mphot_map) ymin, ymax = temp.min() * 0.8, temp.max() * 10 xlabel('Wavelength [A]', fontsize=20) ylabel('Flux Density [erg/cm^2/s]', fontsize=20) xlim([xmin, xmax]) ylim([ymin, ymax]) legend(loc='lower right', fontsize=15) tight_layout() # plt.show() # plot sfr on the same canvas from mpl_toolkits.axes_grid.inset_locator import inset_axes in_axes = inset_axes( ax, width="25%", # width = 30% of parent_bbox height=2, # height : 1 inch loc='upper center') in_axes.scatter(10**(x - 9), sfr) #plt.fill_between(10**(x-9), sfr_min, sfr_max) xlabel('Age [Gyr]', fontsize=10) ylabel('SFR [M_sun/yr]', fontsize=10) #plt.xscale('log') plt.yscale('log') plt.savefig(path + '_fitting.png') # plot residue as subplot figure(figsize=(16, 8)) plt.subplot(212, sharex=ax) import pylab residue = abs(mphot_map - ob) scatter(wphot, abs(mphot_map - ob), label='Residue', lw=2, alpha=0.8) plt.xscale('log') plt.yscale('log') xmin, xmax = np.min(wphot) * 0.8, np.max(wphot) / 0.8 temp = np.interp(np.linspace(xmin, xmax, 10000), wphot, residue) ymin, ymax = temp.min() * 0.1, temp.max() / 0.8 xlabel('Wavelength [A]', fontsize=20) ylabel('Flux Change [erg/cm^2/s]', fontsize=20) xlim([xmin, xmax]) ylim([ymin, ymax]) legend(loc='best', fontsize=15) tight_layout()
# ------- if rp['verbose']: print('dynesty sampling...') tstart = time.time() # time it dynestyout = fitting.run_dynesty_sampler(lnprobfn, prior_transform, model.ndim, pool=pool, queue_size=nprocs, stop_function=stopping_function, wt_function=weight_function, **rp) ndur = time.time() - tstart print('done dynesty in {0}s'.format(ndur)) # ------------------------- # Output HDF5 (and pickles if asked for) # ------------------------- if rp.get("output_pickles", False): # Write the dynesty result object as a pickle import pickle with open(outroot + '_dns.pkl', 'w') as f: pickle.dump(dynestyout, f) # Write the model as a pickle partext = write_results.paramfile_string(**rp) write_results.write_model_pickle(outroot + '_model', model, powell=None, paramfile_text=partext) # Write HDF5 hfile = outroot + '_mcmc.h5' write_results.write_hdf5(hfile, rp, model, obs, dynestyout, None, tsample=ndur)