def test_read_noise_data(self):
"""
Read in the noise model from a cached file and test that selected values
are as expected.
"""
ndata = read_noise_data(self.noise_trim_fname_cache)
exp_keys = ["bias", "completeness", "error"]
for ckey in ndata.keys():
assert ckey in exp_keys, f"{ckey} not in noise data expected keys"
assert np.all((ndata["bias"] >= -1e-10) & (ndata["bias"] <= 1e-10)
), "bias values not between -1e-10 and 1e-10"
assert np.all((ndata["error"] >= -1e-10) & (ndata["error"] <= 1e-10)
), "error values not between -1e-10 and 1e-10"
assert np.all((ndata["completeness"] >= 0.0)
& (ndata["completeness"] <= 1.0)
), "completeness values not between 0 and 1"
def megabeast_image(megabeast_input_file, verbose=True):
"""
Run the MegaBEAST on an image of BEAST results. The BEAST results
are given as spatially-reordered BEAST outputs with a file of lnp results
for each pixel in the image.
Parameters
----------
megabeast_input_file : string
Name of the file that contains settings, filenames, etc
verbose : boolean (default=True)
print extra info
"""
# read in the settings from the file
params = read_input(megabeast_input_file)
# use nstars image to setup for each pixel
nstars_image, nstars_header = fits.getdata(params["nstars_filename"],
header=True)
n_x, n_y = nstars_image.shape
# read in the beast data that is needed by all the pixels
beast_data = {}
# - SED data
beast_data.update(
read_sed_data(params["beast_seds_filename"], param_list=["Av"]))
# - max completeness
beast_data.update(
read_noise_data(
params["beast_noise_filename"],
param_list=["completeness"],
))
# completeness from toothpick model so n band completeness values
# require only 1 completeness value for each model
# max picked (may not be correct)
beast_data["completeness"] = np.max(beast_data["completeness"], axis=1)
# BEAST prior model
beast_pmodel = {}
beast_pmodel["AV"] = params["av_prior_model"]
beast_pmodel["RV"] = params["rv_prior_model"]
beast_pmodel["fA"] = params["fA_prior_model"]
# setup for output
pixel_fit_status = np.full((n_x, n_y), False, dtype=bool)
n_fit_params = len(params["fit_param_names"])
best_fit_images = np.zeros((n_x, n_y, n_fit_params), dtype=float) + np.nan
# loop over the pixels with non-zero entries in the nstars image
for i in trange(n_x, desc="x pixels"):
for j in trange(n_y, desc="y pixels", leave=False):
if nstars_image[i, j] >= params["min_for_fit"]:
pixel_fit_status[i, j] = True
# filename with saved BEAST posteriors
lnp_prefix = params["lnp_file_prefix"]
lnp_filename = f"{lnp_prefix}_{j}_{i}_lnp.hd5"
best_fit_params = fit_ensemble(
beast_data,
lnp_filename,
beast_pmodel,
nstars_expected=nstars_image[i, j],
)
best_fit_images[i, j, :] = best_fit_params
# output results (* = future)
# - best fit
# - *megabeast parameter 1D pPDFs
# - *MCMC chain
# Write the maps to disk
master_header = nstars_header
# check that the directory exists
dpath = "./%s_megabeast/" % (params["projectname"])
if not os.path.exists(dpath):
os.makedirs(dpath)
for k, cname in enumerate(params["fit_param_names"]):
hdu = fits.PrimaryHDU(best_fit_images[:, :, k], header=master_header)
hdu.writeto(
"%s_megabeast/%s_%s_bestfit.fits" %
(params["projectname"], params["projectname"], cname),
overwrite=True,
)
def megabeast(megabeast_input_file, verbose=True):
"""
Run the MegaBEAST on each of the spatially-reordered BEAST outputs.
Parameters
----------
megabeast_input_file : string
Name of the file that contains settings, filenames, etc
verbose : boolean (default=True)
print extra info
"""
# read in the settings from the file
mb_settings = read_megabeast_input(megabeast_input_file)
# setup the megabeast model including defining the priors
# - dust distribution model
# - stellar populations model (later)
# use nstars image to setup for each pixel
nstars_image, nstars_header = fits.getdata(mb_settings["nstars_filename"],
header=True)
n_x, n_y = nstars_image.shape
# read in the beast data that is needed by all the pixels
beast_data = {}
# - SED data
beast_data.update(
read_beast_data.read_sed_data(
mb_settings["beast_seds_filename"],
param_list=["Av"] # , "Rv", "f_A"]
))
# - max completeness
beast_data.update(
read_beast_data.read_noise_data(
mb_settings["beast_noise_filename"],
param_list=["completeness"],
))
beast_data["completeness"] = np.max(beast_data["completeness"], axis=1)
# setup for output
pixel_fit_status = np.full((n_x, n_y), False, dtype=bool)
n_fit_params = len(mb_settings["fit_param_names"])
best_fit_images = np.zeros((n_x, n_y, n_fit_params), dtype=float) + np.nan
# loop over the pixels with non-zero entries in the nstars image
for i in trange(n_x, desc="x pixels"):
for j in trange(n_y, desc="y pixels", leave=False):
# for i in [6]:
# for j in [6]:
if verbose:
print("working on (%i,%i)" % (i, j))
if nstars_image[i, j] >= mb_settings["min_for_fit"]:
pixel_fit_status[i, j] = True
# get the saved sparse likelihoods
lnp_filename = mb_settings[
"lnp_file_prefix"] + "_{0}_{1}_lnp.hd5".format(j, i)
lnp_data = read_beast_data.read_lnp_data(
lnp_filename,
nstars=nstars_image[i, j],
shift_lnp=True,
)
# get the completeness and BEAST model parameters for the
# same grid points as the sparse likelihoods
lnp_grid_vals = read_beast_data.get_lnp_grid_vals(
beast_data, lnp_data)
# initialize the ensemble model with the parameters used
# for the saved BEAST model run results
# currently only dust parameters allowed
# for testing -> only Av
avs = lnp_grid_vals["Av"]
rvs = [3.1] # beast_data['Rv']
fAs = [1.0] # beast_data['f_A']
beast_dust_priors = PriorWeightsDust(
avs,
mb_settings["av_prior_model"],
rvs,
mb_settings["rv_prior_model"],
fAs,
mb_settings["fA_prior_model"],
)
# standard minimization to find initial values
def chi2(args):
return -1.0 * lnprob(*args)
result = op.minimize(
chi2,
[0.25, 2.0, 0.5, 0.5, 1],
args=(beast_dust_priors, lnp_data, lnp_grid_vals),
method="Nelder-Mead",
)
best_fit_images[i, j, :] = result["x"]
# print(result)
# print(result['x'])
# print(result['success'])
# then run through MCMC to fully sample likelihood
# include option not to run MCMC
# output results
# - best fit
# - megabeast parameter 1D pPDFs
# - MCMC chain
master_header = nstars_header
# Now, write the maps to disk
# check that the directory exists
if not os.path.exists("./" + mb_settings["projectname"] + "_megabeast/"):
os.makedirs("./" + mb_settings["projectname"] + "_megabeast/")
for k, cname in enumerate(mb_settings["fit_param_names"]):
hdu = fits.PrimaryHDU(best_fit_images[:, :, k], header=master_header)
# Save to FITS file
hdu.writeto(
"%s_megabeast/%s_%s_bestfit.fits" %
(mb_settings["projectname"], mb_settings["projectname"], cname),
overwrite=True,
)
def plot_input_data(megabeast_input_file, chi2_plot=[], log_scale=False):
"""
Parameters
----------
megabeast_input_file : string
Name of the file that contains settings, filenames, etc
chi2_plot : list of floats (default=[])
Make A_V histogram(s) with chi2 less than each of the values in this list
log_scale : boolean (default=False)
If True, make the histogram x-axis a log scale (to visualize log-normal
A_V distribution)
"""
# read in the settings from the file
mb_settings = read_input(megabeast_input_file)
# get the project name
projectname = mb_settings["projectname"]
# read in the beast data that is needed by all the pixels
beast_data = {}
# - SED data
beast_data.update(
read_beast_data.read_sed_data(
mb_settings["beast_seds_filename"],
param_list=["Av"] # , "Rv", "f_A"]
))
# - max completeness
beast_data.update(
read_beast_data.read_noise_data(
mb_settings["beast_noise_filename"],
param_list=["completeness"],
))
beast_data["completeness"] = np.max(beast_data["completeness"], axis=1)
# read in the nstars image
nstars_image, nstars_header = fits.getdata(mb_settings["nstars_filename"],
header=True)
# dimensions of images/plotting
y_dimen = nstars_image.shape[0]
x_dimen = nstars_image.shape[1]
# set up multi-page figure
if not log_scale:
pp = PdfPages("{0}_megabeast/plot_input_data.pdf".format(projectname))
if log_scale:
pp = PdfPages(
"{0}_megabeast/plot_input_data_log.pdf".format(projectname))
# save the best-fit A_V
best_av = [[[] for j in range(x_dimen)] for i in range(y_dimen)]
best_av_chi2 = [[[] for j in range(x_dimen)] for i in range(y_dimen)]
# -----------------
# Completeness vs A_V
# -----------------
print("")
print("Making completeness/Av plot")
print("")
# set up figure
plt.figure(figsize=(6, 6))
plt.subplot(1, 1, 1)
for i in tqdm(range(y_dimen), desc="y pixels"):
for j in tqdm(range(x_dimen), desc="x pixels"):
# for i in tqdm(range(int(y_dimen/3)), desc='y pixels'):
# for j in tqdm(range(int(x_dimen/3)), desc='x pixels'):
# for i in [0]:
# for j in [12]:
if nstars_image[i, j] > 20:
# get info about the fits
lnp_filename = mb_settings[
"lnp_file_prefix"] + "_{0}_{1}_lnp.hd5".format(j, i)
lnp_data = read_beast_data.read_lnp_data(
lnp_filename,
nstars=nstars_image[i, j],
shift_lnp=True,
)
# get the completeness and BEAST model parameters for the
# same grid points as the sparse likelihoods
lnp_grid_vals = read_beast_data.get_lnp_grid_vals(
beast_data, lnp_data)
# grab the things we want to plot
plot_av = lnp_grid_vals["Av"]
plot_comp = lnp_grid_vals["completeness"]
for n in range(nstars_image[i, j]):
# plot a random subset of the AVs and completenesses
if (i % 3 == 0) and (j % 3 == 0):
plot_these = np.random.choice(plot_av[:, n].size,
size=20,
replace=False)
plt.plot(
plot_av[plot_these, n] +
np.random.normal(scale=0.02, size=plot_these.size),
plot_comp[plot_these, n],
marker=".",
c="black",
ms=3,
mew=0,
linestyle="None",
alpha=0.05,
)
# also overplot the values for the best fit
max_ind = np.where(lnp_data["vals"][:, n] == np.max(
lnp_data["vals"][:, n]))[0][0]
best_av[i][j].append(plot_av[max_ind, n])
best_av_chi2[i][j].append(-2 *
np.max(lnp_data["vals"][:, n]))
if (i % 3 == 0) and (j % 3 == 0):
plt.plot(
plot_av[max_ind, n] + np.random.normal(scale=0.01),
plot_comp[max_ind, n],
marker=".",
c="magenta",
ms=2,
mew=0,
linestyle="None",
alpha=0.3,
zorder=9999,
)
ax = plt.gca()
ax.set_xlabel(r"$A_V$")
ax.set_ylabel("Completeness")
pp.savefig()
# -----------------
# histograms of AVs
# -----------------
print("")
print("Making Av Histograms")
print("")
# set up figure
plt.figure(figsize=(x_dimen * 2, y_dimen * 2))
# flat list of A_V
# https://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
flat_av = [i for sublist in best_av for item in sublist for i in item]
# grab the max A_V of all of them
# max_av = max(flat_av)
# define bins
if not log_scale:
uniq_av = np.unique(flat_av)
gap = np.min(np.diff(uniq_av))
bins = np.arange(uniq_av[0], uniq_av[-1], gap)
if log_scale:
uniq_av = np.unique(np.log10(flat_av))
gap = (uniq_av[-1] - uniq_av[0]) / len(uniq_av)
bins = np.arange(uniq_av[0], uniq_av[-1], gap)
for i in tqdm(range(y_dimen), desc="y pixels"):
for j in tqdm(range(x_dimen), desc="x pixels"):
# for i in [0]:
# for j in [12]:
if nstars_image[i, j] > 20:
# set up the subplot
plt.subplot(y_dimen, x_dimen,
(y_dimen - i - 1) * (x_dimen) + j + 1)
# make a histogram
if best_av[i][j] != []:
if not log_scale:
plt.hist(
best_av[i][j],
bins=bins.size,
range=(uniq_av[0] - gap / 2,
uniq_av[-1] + gap / 2),
facecolor="xkcd:azure",
linewidth=0.25,
edgecolor="xkcd:azure",
)
if log_scale:
plt.hist(
np.log10(best_av[i][j]),
bins=bins.size,
range=(uniq_av[0] - gap / 2,
uniq_av[-1] + gap / 2),
facecolor="xkcd:azure",
linewidth=0.25,
edgecolor="xkcd:azure",
)
# plt.xlim(xmax=max_av)
plt.suptitle(r"Best-fit $A_V$ for each pixel", fontsize=40)
pp.savefig()
# -----------------
# histograms of AVs with a chi2 cut
# -----------------
if len(chi2_plot) > 0:
print("")
print("Making Av Histograms with chi^2 cut")
print("")
for chi2_cut in chi2_plot:
# set up figure
plt.figure(figsize=(x_dimen * 2, y_dimen * 2))
for i in tqdm(range(y_dimen), desc="y pixels"):
for j in tqdm(range(x_dimen), desc="x pixels"):
# for i in [0]:
# for j in [12]:
if nstars_image[i, j] > 20:
# set up the subplot
plt.subplot(y_dimen, x_dimen,
(y_dimen - i - 1) * (x_dimen) + j + 1)
# make a histogram
if best_av[i][j] != []:
if not log_scale:
plot_av = np.array(best_av[i][j])[
np.array(best_av_chi2[i][j]) < chi2_cut]
if log_scale:
plot_av = np.log10(
np.array(best_av[i][j])[
np.array(best_av_chi2[i][j]) < chi2_cut])
if len(plot_av) != 0:
plt.hist(
plot_av,
bins=bins.size,
range=(uniq_av[0] - gap / 2,
uniq_av[-1] + gap / 2),
facecolor="xkcd:azure",
linewidth=0.25,
edgecolor="xkcd:azure",
)
plt.suptitle(
r"Best-fit $A_V$ for each pixel, but only using sources with $\chi^2 < $"
+ str(chi2_cut),
fontsize=40,
)
pp.savefig()
# close PDF figure
pp.close()
