def test_read_noise_data(self): """ Read in the noise model from a cached file and test that selected values are as expected. """ ndata = read_noise_data(self.noise_trim_fname_cache) exp_keys = ["bias", "completeness", "error"] for ckey in ndata.keys(): assert ckey in exp_keys, f"{ckey} not in noise data expected keys" assert np.all((ndata["bias"] >= -1e-10) & (ndata["bias"] <= 1e-10) ), "bias values not between -1e-10 and 1e-10" assert np.all((ndata["error"] >= -1e-10) & (ndata["error"] <= 1e-10) ), "error values not between -1e-10 and 1e-10" assert np.all((ndata["completeness"] >= 0.0) & (ndata["completeness"] <= 1.0) ), "completeness values not between 0 and 1"
def megabeast_image(megabeast_input_file, verbose=True): """ Run the MegaBEAST on an image of BEAST results. The BEAST results are given as spatially-reordered BEAST outputs with a file of lnp results for each pixel in the image. Parameters ---------- megabeast_input_file : string Name of the file that contains settings, filenames, etc verbose : boolean (default=True) print extra info """ # read in the settings from the file params = read_input(megabeast_input_file) # use nstars image to setup for each pixel nstars_image, nstars_header = fits.getdata(params["nstars_filename"], header=True) n_x, n_y = nstars_image.shape # read in the beast data that is needed by all the pixels beast_data = {} # - SED data beast_data.update( read_sed_data(params["beast_seds_filename"], param_list=["Av"])) # - max completeness beast_data.update( read_noise_data( params["beast_noise_filename"], param_list=["completeness"], )) # completeness from toothpick model so n band completeness values # require only 1 completeness value for each model # max picked (may not be correct) beast_data["completeness"] = np.max(beast_data["completeness"], axis=1) # BEAST prior model beast_pmodel = {} beast_pmodel["AV"] = params["av_prior_model"] beast_pmodel["RV"] = params["rv_prior_model"] beast_pmodel["fA"] = params["fA_prior_model"] # setup for output pixel_fit_status = np.full((n_x, n_y), False, dtype=bool) n_fit_params = len(params["fit_param_names"]) best_fit_images = np.zeros((n_x, n_y, n_fit_params), dtype=float) + np.nan # loop over the pixels with non-zero entries in the nstars image for i in trange(n_x, desc="x pixels"): for j in trange(n_y, desc="y pixels", leave=False): if nstars_image[i, j] >= params["min_for_fit"]: pixel_fit_status[i, j] = True # filename with saved BEAST posteriors lnp_prefix = params["lnp_file_prefix"] lnp_filename = f"{lnp_prefix}_{j}_{i}_lnp.hd5" best_fit_params = fit_ensemble( beast_data, lnp_filename, beast_pmodel, nstars_expected=nstars_image[i, j], ) best_fit_images[i, j, :] = best_fit_params # output results (* = future) # - best fit # - *megabeast parameter 1D pPDFs # - *MCMC chain # Write the maps to disk master_header = nstars_header # check that the directory exists dpath = "./%s_megabeast/" % (params["projectname"]) if not os.path.exists(dpath): os.makedirs(dpath) for k, cname in enumerate(params["fit_param_names"]): hdu = fits.PrimaryHDU(best_fit_images[:, :, k], header=master_header) hdu.writeto( "%s_megabeast/%s_%s_bestfit.fits" % (params["projectname"], params["projectname"], cname), overwrite=True, )
def megabeast(megabeast_input_file, verbose=True): """ Run the MegaBEAST on each of the spatially-reordered BEAST outputs. Parameters ---------- megabeast_input_file : string Name of the file that contains settings, filenames, etc verbose : boolean (default=True) print extra info """ # read in the settings from the file mb_settings = read_megabeast_input(megabeast_input_file) # setup the megabeast model including defining the priors # - dust distribution model # - stellar populations model (later) # use nstars image to setup for each pixel nstars_image, nstars_header = fits.getdata(mb_settings["nstars_filename"], header=True) n_x, n_y = nstars_image.shape # read in the beast data that is needed by all the pixels beast_data = {} # - SED data beast_data.update( read_beast_data.read_sed_data( mb_settings["beast_seds_filename"], param_list=["Av"] # , "Rv", "f_A"] )) # - max completeness beast_data.update( read_beast_data.read_noise_data( mb_settings["beast_noise_filename"], param_list=["completeness"], )) beast_data["completeness"] = np.max(beast_data["completeness"], axis=1) # setup for output pixel_fit_status = np.full((n_x, n_y), False, dtype=bool) n_fit_params = len(mb_settings["fit_param_names"]) best_fit_images = np.zeros((n_x, n_y, n_fit_params), dtype=float) + np.nan # loop over the pixels with non-zero entries in the nstars image for i in trange(n_x, desc="x pixels"): for j in trange(n_y, desc="y pixels", leave=False): # for i in [6]: # for j in [6]: if verbose: print("working on (%i,%i)" % (i, j)) if nstars_image[i, j] >= mb_settings["min_for_fit"]: pixel_fit_status[i, j] = True # get the saved sparse likelihoods lnp_filename = mb_settings[ "lnp_file_prefix"] + "_{0}_{1}_lnp.hd5".format(j, i) lnp_data = read_beast_data.read_lnp_data( lnp_filename, nstars=nstars_image[i, j], shift_lnp=True, ) # get the completeness and BEAST model parameters for the # same grid points as the sparse likelihoods lnp_grid_vals = read_beast_data.get_lnp_grid_vals( beast_data, lnp_data) # initialize the ensemble model with the parameters used # for the saved BEAST model run results # currently only dust parameters allowed # for testing -> only Av avs = lnp_grid_vals["Av"] rvs = [3.1] # beast_data['Rv'] fAs = [1.0] # beast_data['f_A'] beast_dust_priors = PriorWeightsDust( avs, mb_settings["av_prior_model"], rvs, mb_settings["rv_prior_model"], fAs, mb_settings["fA_prior_model"], ) # standard minimization to find initial values def chi2(args): return -1.0 * lnprob(*args) result = op.minimize( chi2, [0.25, 2.0, 0.5, 0.5, 1], args=(beast_dust_priors, lnp_data, lnp_grid_vals), method="Nelder-Mead", ) best_fit_images[i, j, :] = result["x"] # print(result) # print(result['x']) # print(result['success']) # then run through MCMC to fully sample likelihood # include option not to run MCMC # output results # - best fit # - megabeast parameter 1D pPDFs # - MCMC chain master_header = nstars_header # Now, write the maps to disk # check that the directory exists if not os.path.exists("./" + mb_settings["projectname"] + "_megabeast/"): os.makedirs("./" + mb_settings["projectname"] + "_megabeast/") for k, cname in enumerate(mb_settings["fit_param_names"]): hdu = fits.PrimaryHDU(best_fit_images[:, :, k], header=master_header) # Save to FITS file hdu.writeto( "%s_megabeast/%s_%s_bestfit.fits" % (mb_settings["projectname"], mb_settings["projectname"], cname), overwrite=True, )
def plot_input_data(megabeast_input_file, chi2_plot=[], log_scale=False): """ Parameters ---------- megabeast_input_file : string Name of the file that contains settings, filenames, etc chi2_plot : list of floats (default=[]) Make A_V histogram(s) with chi2 less than each of the values in this list log_scale : boolean (default=False) If True, make the histogram x-axis a log scale (to visualize log-normal A_V distribution) """ # read in the settings from the file mb_settings = read_input(megabeast_input_file) # get the project name projectname = mb_settings["projectname"] # read in the beast data that is needed by all the pixels beast_data = {} # - SED data beast_data.update( read_beast_data.read_sed_data( mb_settings["beast_seds_filename"], param_list=["Av"] # , "Rv", "f_A"] )) # - max completeness beast_data.update( read_beast_data.read_noise_data( mb_settings["beast_noise_filename"], param_list=["completeness"], )) beast_data["completeness"] = np.max(beast_data["completeness"], axis=1) # read in the nstars image nstars_image, nstars_header = fits.getdata(mb_settings["nstars_filename"], header=True) # dimensions of images/plotting y_dimen = nstars_image.shape[0] x_dimen = nstars_image.shape[1] # set up multi-page figure if not log_scale: pp = PdfPages("{0}_megabeast/plot_input_data.pdf".format(projectname)) if log_scale: pp = PdfPages( "{0}_megabeast/plot_input_data_log.pdf".format(projectname)) # save the best-fit A_V best_av = [[[] for j in range(x_dimen)] for i in range(y_dimen)] best_av_chi2 = [[[] for j in range(x_dimen)] for i in range(y_dimen)] # ----------------- # Completeness vs A_V # ----------------- print("") print("Making completeness/Av plot") print("") # set up figure plt.figure(figsize=(6, 6)) plt.subplot(1, 1, 1) for i in tqdm(range(y_dimen), desc="y pixels"): for j in tqdm(range(x_dimen), desc="x pixels"): # for i in tqdm(range(int(y_dimen/3)), desc='y pixels'): # for j in tqdm(range(int(x_dimen/3)), desc='x pixels'): # for i in [0]: # for j in [12]: if nstars_image[i, j] > 20: # get info about the fits lnp_filename = mb_settings[ "lnp_file_prefix"] + "_{0}_{1}_lnp.hd5".format(j, i) lnp_data = read_beast_data.read_lnp_data( lnp_filename, nstars=nstars_image[i, j], shift_lnp=True, ) # get the completeness and BEAST model parameters for the # same grid points as the sparse likelihoods lnp_grid_vals = read_beast_data.get_lnp_grid_vals( beast_data, lnp_data) # grab the things we want to plot plot_av = lnp_grid_vals["Av"] plot_comp = lnp_grid_vals["completeness"] for n in range(nstars_image[i, j]): # plot a random subset of the AVs and completenesses if (i % 3 == 0) and (j % 3 == 0): plot_these = np.random.choice(plot_av[:, n].size, size=20, replace=False) plt.plot( plot_av[plot_these, n] + np.random.normal(scale=0.02, size=plot_these.size), plot_comp[plot_these, n], marker=".", c="black", ms=3, mew=0, linestyle="None", alpha=0.05, ) # also overplot the values for the best fit max_ind = np.where(lnp_data["vals"][:, n] == np.max( lnp_data["vals"][:, n]))[0][0] best_av[i][j].append(plot_av[max_ind, n]) best_av_chi2[i][j].append(-2 * np.max(lnp_data["vals"][:, n])) if (i % 3 == 0) and (j % 3 == 0): plt.plot( plot_av[max_ind, n] + np.random.normal(scale=0.01), plot_comp[max_ind, n], marker=".", c="magenta", ms=2, mew=0, linestyle="None", alpha=0.3, zorder=9999, ) ax = plt.gca() ax.set_xlabel(r"$A_V$") ax.set_ylabel("Completeness") pp.savefig() # ----------------- # histograms of AVs # ----------------- print("") print("Making Av Histograms") print("") # set up figure plt.figure(figsize=(x_dimen * 2, y_dimen * 2)) # flat list of A_V # https://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python flat_av = [i for sublist in best_av for item in sublist for i in item] # grab the max A_V of all of them # max_av = max(flat_av) # define bins if not log_scale: uniq_av = np.unique(flat_av) gap = np.min(np.diff(uniq_av)) bins = np.arange(uniq_av[0], uniq_av[-1], gap) if log_scale: uniq_av = np.unique(np.log10(flat_av)) gap = (uniq_av[-1] - uniq_av[0]) / len(uniq_av) bins = np.arange(uniq_av[0], uniq_av[-1], gap) for i in tqdm(range(y_dimen), desc="y pixels"): for j in tqdm(range(x_dimen), desc="x pixels"): # for i in [0]: # for j in [12]: if nstars_image[i, j] > 20: # set up the subplot plt.subplot(y_dimen, x_dimen, (y_dimen - i - 1) * (x_dimen) + j + 1) # make a histogram if best_av[i][j] != []: if not log_scale: plt.hist( best_av[i][j], bins=bins.size, range=(uniq_av[0] - gap / 2, uniq_av[-1] + gap / 2), facecolor="xkcd:azure", linewidth=0.25, edgecolor="xkcd:azure", ) if log_scale: plt.hist( np.log10(best_av[i][j]), bins=bins.size, range=(uniq_av[0] - gap / 2, uniq_av[-1] + gap / 2), facecolor="xkcd:azure", linewidth=0.25, edgecolor="xkcd:azure", ) # plt.xlim(xmax=max_av) plt.suptitle(r"Best-fit $A_V$ for each pixel", fontsize=40) pp.savefig() # ----------------- # histograms of AVs with a chi2 cut # ----------------- if len(chi2_plot) > 0: print("") print("Making Av Histograms with chi^2 cut") print("") for chi2_cut in chi2_plot: # set up figure plt.figure(figsize=(x_dimen * 2, y_dimen * 2)) for i in tqdm(range(y_dimen), desc="y pixels"): for j in tqdm(range(x_dimen), desc="x pixels"): # for i in [0]: # for j in [12]: if nstars_image[i, j] > 20: # set up the subplot plt.subplot(y_dimen, x_dimen, (y_dimen - i - 1) * (x_dimen) + j + 1) # make a histogram if best_av[i][j] != []: if not log_scale: plot_av = np.array(best_av[i][j])[ np.array(best_av_chi2[i][j]) < chi2_cut] if log_scale: plot_av = np.log10( np.array(best_av[i][j])[ np.array(best_av_chi2[i][j]) < chi2_cut]) if len(plot_av) != 0: plt.hist( plot_av, bins=bins.size, range=(uniq_av[0] - gap / 2, uniq_av[-1] + gap / 2), facecolor="xkcd:azure", linewidth=0.25, edgecolor="xkcd:azure", ) plt.suptitle( r"Best-fit $A_V$ for each pixel, but only using sources with $\chi^2 < $" + str(chi2_cut), fontsize=40, ) pp.savefig() # close PDF figure pp.close()