def split_and_check(grid_fname, num_subgrids):
complete_g = SEDGrid(grid_fname)
sub_fnames = subgridding_tools.split_grid(grid_fname, num_subgrids)
# count the number of grid cells
sub_seds = []
sub_grids = []
for sub_fname in sub_fnames:
sub_g = SEDGrid(sub_fname)
sub_seds.append(sub_g.seds)
sub_grids.append(sub_g.grid)
np.testing.assert_equal(complete_g.lamb, sub_g.lamb)
if not complete_g.grid.colnames == sub_g.grid.colnames:
raise AssertionError()
sub_seds_reconstructed = np.concatenate(sub_seds)
np.testing.assert_equal(sub_seds_reconstructed, complete_g.seds)
sub_grids_reconstructed = np.concatenate(sub_grids)
np.testing.assert_equal(sub_grids_reconstructed, complete_g.grid)
# the split method skips anything that already exists, so if we
# want to use this function multiple times for the same test
# grid, we need to do this.
for f in sub_fnames:
os.remove(f)
def test_trim_grid(self):
"""
Generate trim the sed grid and noise model using cached versions of the
both and compare the result to a cached version.
"""
# read in the observed data
obsdata = Observations(self.obs_fname_cache, self.settings.filters,
self.settings.obs_colnames)
# get the modesedgrid
modelsedgrid = SEDGrid(self.seds_fname_cache)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(self.noise_fname_cache)
# trim the model sedgrid
seds_trim_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name
noise_trim_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name
trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
seds_trim_fname,
noise_trim_fname,
sigma_fac=3.0,
)
# compare the new to the cached version
compare_hdf5(self.seds_trim_fname_cache, seds_trim_fname, ctype="seds")
compare_hdf5(self.noise_trim_fname_cache,
noise_trim_fname,
ctype="noise")
def test_simobs(self):
"""
Simulate observations using cached versions of the sed grid and noise model
and compare the result to a cached version.
"""
# download files specific to this test
simobs_fname_cache = download_rename("beast_example_phat_simobs.fits")
# get the physics model grid - includes priors
modelsedgrid = SEDGrid(self.seds_fname_cache)
# read in the noise model - includes bias, unc, and completeness
noisegrid = noisemodel.get_noisemodelcat(self.noise_fname_cache)
table_new = gen_SimObs_from_sedgrid(
modelsedgrid,
noisegrid,
nsim=100,
compl_filter="max",
ranseed=1234,
)
# check that the simobs files are exactly the same
table_cache = Table.read(simobs_fname_cache)
# to avoid issues with uppercase vs lowercase column names, make them all
# the same before comparing
for col in table_new.colnames:
table_new[col].name = col.upper()
for col in table_cache.colnames:
table_cache[col].name = col.upper()
compare_tables(table_cache, table_new)
def gen_obsmodel(modelsedgridfile, source_density=None, use_rate=True):
"""
Code to create filenames and run the toothpick noise model
Parameters
----------
modelsedgridfile : string
path+name of the physics model grid file
source_density : string (default=None)
set to None if there's no source density info, otherwise set to
a string of the form "#-#"
use_rate : boolean (default=True)
Choose whether to use the rate or magnitude when creating the noise
model. This should always be True, but is currently an option to be
compatible with the phat_small example (which has no rate info).
When that gets fixed, please remove this option!
Returns
-------
noisefile : string
name of the created noise file
"""
print("")
# noise and AST file names
noisefile = modelsedgridfile.replace("seds", "noisemodel")
astfile = datamodel.astfile
# If we are treating regions with different
# backgrounds/source densities separately, pick one of the
# split ast files, and name noise file accordingly
if source_density is not None:
noisefile = noisefile.replace("noisemodel",
"noisemodel_bin" + source_density)
astfile = datamodel.astfile.replace(
".fits", "_bin" + source_density.replace("_", "-") + ".fits")
# only create noise file if it doesn't already exist
if not os.path.isfile(noisefile):
print("creating " + noisefile)
modelsedgrid = SEDGrid(modelsedgridfile)
noisemodel.make_toothpick_noise_model(
noisefile,
astfile,
modelsedgrid,
absflux_a_matrix=datamodel.absflux_a_matrix,
use_rate=use_rate,
)
else:
print(noisefile + " already exists")
return noisefile # (same as noisefile)
def gen_obsmodel(settings, modelsedgridfile, source_density=None):
"""
Code to create filenames and run the toothpick noise model
Parameters
----------
settings : beast.tools.beast_settings.beast_settings instance
object with the beast settings
modelsedgridfile : string
path+name of the physics model grid file
source_density : string (default=None)
set to None if there's no source density info, otherwise set to
a string of the form "#-#"
Returns
-------
noisefile : string
name of the created noise file
"""
print("")
# noise and AST file names
noisefile = modelsedgridfile.replace("seds", "noisemodel")
astfile = settings.astfile
# If we are treating regions with different
# backgrounds/source densities separately, pick one of the
# split ast files, and name noise file accordingly
if source_density is not None:
noisefile = noisefile.replace("noisemodel",
"noisemodel_bin" + source_density)
astfile = settings.astfile.replace(
".fits", "_bin" + source_density.replace("_", "-") + ".fits")
# only create noise file if it doesn't already exist
if not os.path.isfile(noisefile):
print("creating " + noisefile)
modelsedgrid = SEDGrid(modelsedgridfile)
noisemodel.make_toothpick_noise_model(
noisefile,
astfile,
modelsedgrid,
absflux_a_matrix=settings.absflux_a_matrix,
)
else:
print(noisefile + " already exists")
return noisefile # (same as noisefile)
def test_trim_grid():
# download the needed files
vega_fname = download_rename("vega.hd5")
seds_fname = download_rename("beast_example_phat_seds.grid.hd5")
noise_fname = download_rename("beast_example_phat_noisemodel.grid.hd5")
obs_fname = download_rename("b15_4band_det_27_A.fits")
# download cached version of noisemodel on the sed grid
noise_trim_fname_cache = download_rename(
"beast_example_phat_noisemodel_trim.grid.hd5")
seds_trim_fname_cache = download_rename(
"beast_example_phat_seds_trim.grid.hd5")
################
# read in the observed data
filters = [
"HST_WFC3_F275W",
"HST_WFC3_F336W",
"HST_ACS_WFC_F475W",
"HST_ACS_WFC_F814W",
"HST_WFC3_F110W",
"HST_WFC3_F160W",
]
basefilters = ["F275W", "F336W", "F475W", "F814W", "F110W", "F160W"]
obs_colnames = [f.lower() + "_rate" for f in basefilters]
obsdata = Observations(obs_fname,
filters,
obs_colnames,
vega_fname=vega_fname)
# get the modesedgrid
modelsedgrid = SEDGrid(seds_fname)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(noise_fname)
# trim the model sedgrid
seds_trim_fname = "beast_example_phat_seds_trim.grid.hd5"
noise_trim_fname = seds_trim_fname.replace("_seds", "_noisemodel")
trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
seds_trim_fname,
noise_trim_fname,
sigma_fac=3.0,
)
# compare the new to the cached version
compare_hdf5(seds_trim_fname_cache, seds_trim_fname, ctype="seds")
compare_hdf5(noise_trim_fname_cache, noise_trim_fname, ctype="noise")
def split_and_check(grid_fname, num_subgrids):
"""
Split a sed grid into subgrids and test the contents of the subgrids
are as expected and concatenating the subgrid components (seds, grid)
gives the full sed grid.
Parameters
----------
grid_fname : str
filename for the sed grid
num_subgrids : int
number of subgrids to split the sed grid into
"""
complete_g = SEDGrid(grid_fname)
sub_fnames = subgridding_tools.split_grid(grid_fname, num_subgrids)
# count the number of grid cells
sub_seds = []
sub_grids = []
for sub_fname in sub_fnames:
sub_g = SEDGrid(sub_fname)
sub_seds.append(sub_g.seds)
sub_grids.append(sub_g.grid)
np.testing.assert_equal(complete_g.lamb, sub_g.lamb)
if not complete_g.grid.colnames == sub_g.grid.colnames:
raise AssertionError()
sub_seds_reconstructed = np.concatenate(sub_seds)
np.testing.assert_equal(sub_seds_reconstructed, complete_g.seds)
sub_grids_reconstructed = np.concatenate(sub_grids)
np.testing.assert_equal(sub_grids_reconstructed, complete_g.grid)
# the split method skips anything that already exists, so if we
# want to use this function multiple times for the same test
# grid, we need to do this.
for f in sub_fnames:
os.remove(f)
def merge_grids(seds_fname, sub_names):
"""
Merges a set of grids into one big grid. The grids need to have the
same columns
Parameters
----------
seds_fname: string
path for the output file
sub_names: list of strings
paths for the input grids
"""
if not os.path.isfile(seds_fname):
for n in sub_names:
print("Appending {} to {}".format(n, seds_fname))
g = SEDGrid(n)
g.write(seds_fname, append=True)
else:
print("{} already exists".format(seds_fname))
def split_grid(grid_fname, num_subgrids, overwrite=False):
"""
Splits a spectral or sed grid (they are the same class actually)
according to grid point index (so basically, arbitrarily).
Parameters
----------
grid_fname: string
file name of the existing grid to be split up
num_subgrids: integer
the number of parts the grid should be split into
overwrite: bool
any subgrids that already exist will be deleted if set to True.
If set to False, skip over any grids that are already there.
Returns
-------
list of string
the names of the newly created subgrid files
"""
g = SEDGrid(grid_fname, backend="disk")
fnames = []
num_seds = len(g.seds)
slices = uniform_slices(num_seds, num_subgrids)
for i, slc in enumerate(slices):
subgrid_fname = grid_fname.replace(".hd5", "sub{}.hd5".format(i))
fnames.append(subgrid_fname)
if os.path.isfile(subgrid_fname):
if overwrite:
os.remove(subgrid_fname)
else:
print("{} already exists. Skipping.".format(subgrid_fname))
continue
print("constructing subgrid " + str(i))
# Load a slice as a SEDGrid object
sub_g = SEDGrid(
g.lamb[:],
seds=g.seds[slc],
grid=Table(g.grid[slc]),
backend="memory",
)
if g.filters is not None:
sub_g.header["filters"] = " ".join(g.filters)
# Save it to a new file
sub_g.write(subgrid_fname, append=False)
return fnames
def test_toothpick_noisemodel():
# download the needed files
asts_fname = download_rename("fake_stars_b15_27_all.hd5")
filter_fname = download_rename("filters.hd5")
vega_fname = download_rename("vega.hd5")
hst_fname = download_rename("hst_whitedwarf_frac_covar.fits")
seds_fname = download_rename("beast_example_phat_seds.grid.hd5")
# download cached version of noisemodel on the sed grid
noise_fname_cache = download_rename(
"beast_example_phat_noisemodel.grid.hd5")
################
# get the modesedgrid on which to generate the noisemodel
modelsedgrid = SEDGrid(seds_fname)
# absflux calibration covariance matrix for HST specific filters (AC)
filters = [
"HST_WFC3_F275W",
"HST_WFC3_F336W",
"HST_ACS_WFC_F475W",
"HST_ACS_WFC_F814W",
"HST_WFC3_F110W",
"HST_WFC3_F160W",
]
absflux_a_matrix = hst_frac_matrix(filters,
hst_fname=hst_fname,
filterLib=filter_fname)
# generate the AST noise model
noise_fname = "/tmp/beast_example_phat_noisemodel.grid.hd5"
noisemodel.make_toothpick_noise_model(
noise_fname,
asts_fname,
modelsedgrid,
absflux_a_matrix=absflux_a_matrix,
vega_fname=vega_fname,
use_rate=False,
)
# compare the new to the cached version
compare_hdf5(noise_fname_cache, noise_fname)
def test_toothpick_noisemodel(self):
"""
Generate the nosiemodel (aka observationmodel) using a cached version of
the artifical star test results (ASTs) and compare the result to a cached
version.
"""
# get the modelsedgrid on which to generate the noisemodel
modelsedgrid = SEDGrid(self.seds_fname_cache)
# generate the AST noise model
noise_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name
noisemodel.make_toothpick_noise_model(
noise_fname,
self.asts_fname_cache,
modelsedgrid,
absflux_a_matrix=self.settings.absflux_a_matrix,
)
# compare the new to the cached version
compare_hdf5(self.noise_fname_cache, noise_fname)
def test_simobs():
# download the needed files
vega_fname = download_rename("vega.hd5")
seds_fname = download_rename("beast_example_phat_seds.grid.hd5")
noise_fname = download_rename("beast_example_phat_noisemodel.grid.hd5")
# download cached version of noisemodel on the sed grid
simobs_fname_cache = download_rename("beast_example_phat_simobs.fits")
################
# get the physics model grid - includes priors
modelsedgrid = SEDGrid(seds_fname)
# read in the noise model - includes bias, unc, and completeness
noisegrid = noisemodel.get_noisemodelcat(noise_fname)
table_new = gen_SimObs_from_sedgrid(
modelsedgrid,
noisegrid,
nsim=100,
compl_filter="f475w",
ranseed=1234,
vega_fname=vega_fname,
)
# check that the simobs files are exactly the same
table_cache = Table.read(simobs_fname_cache)
# to avoid issues with uppercase vs lowercase column names, make them all
# the same before comparing
for col in table_new.colnames:
table_new[col].name = col.upper()
for col in table_cache.colnames:
table_cache[col].name = col.upper()
compare_tables(table_cache, table_new)
def supplement_ast(
sedgrid_fname,
filters,
nAST=1000,
existingASTfile=None,
outASTfile=None,
outASTfile_params=None,
mag_cuts=None,
color_cuts=None,
):
"""
Creates an additional fake star catalog from a BEAST model grid
that fulfills the customized conditions to supplement input ASTs.
If the existing input AST parameter file is given, already selected
models will be excluded from this process. The input artificial
stars are picked randomly from the remaining models.
Parameters
----------
sedgrid_fname: string
BEAST model grid from which the models are picked (hdf5 file)
filters: list of string
Names of the filters
nAST: int
Number of unique additional ASTs per source density bin
existingASTfile: string (optional, default=None)
Name of the existing input AST parameter file. If not None,
the models that were already listed in the existing list Will
be removed by default
outASTfile: string (optional, default=None)
Output file name for the chosen models
outASTfile_params: string (optional, default=None)
If a file name is given, the physical parameters associated with
each model will be written to disk
mag_cut: dictionary (optional, default=None)
Dictionary of bright and faint magnitude limits for given filters.
The way to specify the cuts is by updating the "ast_suppl_maglimit" key
in the beast_settings file. This is a dictionary that includes information
for the magnitude cuts as a function of the filters included in observation.
For example, for a field observed with HST_WFC3_F336W, HST_WFC3_F475W,
and HST_WFC3_F814W, to set a magnitude range limit of 16<HST_WFC3_F475W<28 mag,
and 15<HST_WFC3_F814W<27 mag you need to set the following within the beast_settings file:
# specify that the ast_supplement mode should be on
ast_supplement = True
# initialize and populate the dictionary of desired magnitude limits
ast_suppl_maglimits = {}
# the magntidue limits are defined by the filter and a list of the limits in magnitudes
ast_suppl_maglimits["HST_WFC3_F475W"] = [16,28]
ast_suppl_maglimits["HST_WFC3_F814W"] = [15,27]
# set the key word
ast_suppl_maglimit = ast_suppl_maglimits
color_cut: dictionary (optional, default=None)
Dictionary of red color limits for given filters.
The way to specify the cuts is by updating the "ast_suppl_colorlimit" key
in the beast_settings file. This is a dictionary that includes information
for the color cuts as a function of the filters included in observation.
For example, for a field observed with HST_WFC3_F336W, HST_WFC3_F475W,
and HST_WFC3_F814W, to set a color range limit of HST_WFC3_F475W-HST_WFC3_F814W<6,
HST_WFC3_F336W-HST_WFC3_F475W<5 and HST_WFC3_F336W-HST_WFC3_F814W<4, you need
to set the following within the beast_settings file:
# specify that the ast_supplement mode should be on
ast_supplement = True
# initialize the dictionary of desired magnitude limits
ast_suppl_colorlimits = {}
# the color limits are defined by the first filter in the color (e.g, X for X-Y),
# and the input is a list including the second filter (e.g., Y for X-Y) and the
# color limit in magnitudes
ast_suppl_colorlimits["HST_WFC3_F475W"] = [["HST_WFC3_F814W",6]]
ast_suppl_colorlimits["HST_WFC3_F336W"] = [["HST_WFC3_F475W",5], ["HST_WFC3_F814W",4]]
# set the key word
ast_suppl_colorlimit = ast_suppl_colorlimits
Returns
-------
sedsMags: astropy Table
A table containing the selected model seds (columns are named
after the filters)
"""
with Vega() as v:
vega_f, vega_flux, lambd = v.getFlux(filters)
modelsedgrid = SEDGrid(sedgrid_fname)
# Convert to Vega mags
sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux)
Nseds = sedsMags.shape[0]
sedsIndx = np.arange(Nseds)
if existingASTfile is not None and os.path.isfile(existingASTfile):
print("{} exists. Will attempt to load SEDs for ASTs from there \
and remove those SEDs from the SED grid".format(existingASTfile))
print("existing AST file", existingASTfile)
t = Table.read(existingASTfile, format="fits")
sedsMags = np.delete(sedsMags, t["sedgrid_indx"], axis=0)
sedsIndx = np.delete(sedsIndx, t["sedgrid_indx"])
Nseds = sedsMags.shape[0]
# Apply selection conditions if supplied
# Just magnitude cuts
print("mag_cuts", mag_cuts)
print("color_cuts", color_cuts)
if mag_cuts is not None:
cond = np.ones(Nseds, dtype=bool)
for key in list(mag_cuts.keys()):
idx_filter = [i for i, iflt in enumerate(filters) if key in iflt]
bright_cut = mag_cuts[key][0]
faint_cut = mag_cuts[key][1]
tmp_cond = np.logical_and(
(sedsMags[:, idx_filter] >= bright_cut),
(sedsMags[:, idx_filter] <= faint_cut),
)
if color_cuts is not None:
if key in color_cuts:
for limit in color_cuts[key]:
idx_color_filter = [
i for i, iflt in enumerate(filters)
if limit[0] in iflt
]
tmp_cond = np.logical_and(
tmp_cond,
(sedsMags[:, idx_filter] -
sedsMags[:, idx_color_filter] <= limit[1]),
)
cond = np.logical_and(cond, tmp_cond.ravel())
sedsMags = sedsMags[cond, :]
sedsIndx = sedsIndx[cond]
# Randomly select models
# Supplementing ASTs does not need to follow
# the toothpick-way selection
chosen_idxs = np.random.choice(len(sedsIndx), nAST)
sedsIndx = sedsIndx[chosen_idxs]
# Gather the selected model seds in a table
sedsMags = Table(sedsMags[chosen_idxs, :], names=filters)
if outASTfile is not None:
ascii.write(
sedsMags,
outASTfile,
overwrite=True,
formats={k: "%.5f"
for k in sedsMags.colnames},
)
# if chosen, save the corresponding model parameters
if outASTfile_params is not None:
grid_dict = {}
for key in list(modelsedgrid.grid.keys()):
grid_dict[key] = modelsedgrid.grid[key][sedsIndx]
grid_dict["sedgrid_indx"] = sedsIndx
ast_params = Table(grid_dict)
ast_params.write(outASTfile_params, overwrite=True)
return sedsMags
def plot_noisemodel(
sed_file,
noise_file_list,
plot_file,
samp=100,
color=["black", "red", "gold", "lime", "xkcd:azure"],
label=None,
):
"""
Make a plot of the noise model: for each of the bandsm make plots of bias
and uncertainty as a function of flux
If there are multiple files in noise_file_list, each of them will be
overplotted in each panel.
Parameters
----------
sed_file : string
path+name of the SED grid file
noise_file_list : list of strings
path+name of the noise model file(s)
plot_file : string
name of the file to save the plot
samp : int (default=100)
plotting all of the SED points takes a long time for a viewer to load,
so set this to plot every Nth point
color : list of strings (default=['black','red','gold','lime','xkcd:azure'])
colors to cycle through when making plots
label : list of strings (default=None)
if set, use these labels in a legend for each item in noise_file_list
"""
# read in the SED grid
print("* reading SED grid file")
sed_object = SEDGrid(sed_file)
if hasattr(sed_object.seds, "read"):
sed_grid = sed_object.seds.read()
else:
sed_grid = sed_object.seds
filter_list = sed_object.filters
n_filter = len(filter_list)
# figure
fig, ax = plt.subplots(nrows=3, ncols=n_filter, figsize=(25, 15))
# setup the plots
fontsize = 12
font = {"size": fontsize}
plt.rc("font", **font)
plt.rc("lines", linewidth=2)
plt.rc("axes", linewidth=2)
plt.rc("xtick.major", width=2)
plt.rc("ytick.major", width=2)
# go through noise files
for n, nfile in enumerate(np.atleast_1d(noise_file_list)):
print("* reading " + nfile)
# read in the values
noisemodel_vals = noisemodel.get_noisemodelcat(nfile)
# extract error and bias
noise_err = noisemodel_vals["error"]
noise_bias = noisemodel_vals["bias"]
noise_compl = noisemodel_vals["completeness"]
# plot things
for f, filt in enumerate(filter_list):
# error is negative where it's been extrapolated -> trim those
good_err = np.where(noise_err[:, f] > 0)[0]
plot_sed = sed_grid[good_err, f][::samp]
plot_err = noise_err[good_err, f][::samp]
plot_bias = noise_bias[good_err, f][::samp]
plot_compl = noise_compl[good_err, f][::samp]
# bias
bax = ax[0, f]
bax.plot(
np.log10(plot_sed),
plot_bias / plot_sed,
marker="o",
linestyle="none",
mew=0,
ms=2,
color=color[n % len(color)],
alpha=0.1,
)
if label is not None:
bax.set_label(label[n])
bax.tick_params(axis="both", which="major")
# ax.set_xlim(ax.get_xlim()[::-1])
bax.set_xlabel("log " + filt)
bax.set_ylabel(r"Bias ($\mu$/F)")
# error
eax = ax[1, f]
eax.plot(
np.log10(plot_sed),
plot_err / plot_sed,
marker="o",
linestyle="none",
mew=0,
ms=2,
color=color[n % len(color)],
alpha=0.1,
)
if label is not None:
eax.set_label(label[n])
eax.tick_params(axis="both", which="major")
# ax.set_xlim(ax.get_xlim()[::-1])
eax.set_xlabel("log " + filt)
eax.set_ylabel(r"Error ($\sigma$/F)")
# completeness
cax = ax[2, f]
cax.plot(
np.log10(plot_sed),
plot_compl,
marker="o",
linestyle="none",
mew=0,
ms=2,
color=color[n % len(color)],
alpha=0.1,
)
if label is not None:
cax.set_label(label[n])
cax.tick_params(axis="both", which="major")
# ax.set_xlim(ax.get_xlim()[::-1])
cax.set_xlabel("log " + filt)
cax.set_ylabel(r"Completeness")
# do a legend if this is
# (a) the leftmost panel
# (b) the last line to be added
# (c) there are labels set
if (f == 0) and (n == len(noise_file_list) - 1) and (label
is not None):
leg = bax.legend(fontsize=12)
for lh in leg.legendHandles:
lh._legmarker.set_alpha(1)
leg = eax.legend(fontsize=12)
for lh in leg.legendHandles:
lh._legmarker.set_alpha(1)
plt.tight_layout()
fig.savefig(plot_file)
plt.close(fig)
def calc_depth(
physgrid_list,
noise_model_list,
completeness_value=0.5,
vega_mag=True,
vega_fname=None,
):
"""
Calculate the observation depth of a field using the completeness. Some
fields have low completeness at both faint and bright fluxes; this finds
the faintest flux at which the completeness exceeds the given value(s).
Parameters
----------
physgrid_list : string or list of strings
Name of the physics model file. If there are multiple physics model
grids (i.e., if there are subgrids), list them all here.
noise_model_list : string or list of strings
Name of the noise model file. If there are multiple files for
physgrid_list (because of subgrids), list the noise model file
associated with each physics model file.
completeness_value : float or list of floats
The completeness(es) at which to evaluate the depth. Completeness is
defined in the range 0.0 to 1.0.
vega_mag : boolean (default=True)
If True, return results in Vega mags. Otherwise returns flux in
erg/s/cm^2/A.
vega_fname : string
filename for the vega info (useful for testing)
Returns
-------
depth_dict : dictionary
keys are the filters present in the grid, each value is the flux or Vega
mag for each of the requested completeness values
"""
# ------ Reading in data
# If there are subgrids, we can't read them all into memory. Therefore,
# we'll go through each one and just grab the relevant parts.
compl_table_list = []
# make a table for each physics model + noise model
for physgrid, noise_model in zip(
np.atleast_1d(physgrid_list), np.atleast_1d(noise_model_list)
):
# get the physics model grid - includes priors
modelsedgrid = SEDGrid(str(physgrid))
if hasattr(modelsedgrid.seds, "read"):
sed_grid = modelsedgrid.seds.read()
else:
sed_grid = modelsedgrid.seds
# get list of filters
filter_list = modelsedgrid.filters
# read in the noise model
noisegrid = noisemodel.get_noisemodelcat(str(noise_model))
# get the completeness
model_compl = noisegrid["completeness"]
# put it all into a table
table_dict = {filt: sed_grid[:, f] for f, filt in enumerate(filter_list)}
table_dict.update(
{filt + "_compl": model_compl[:, f] for f, filt in enumerate(filter_list)}
)
# append to the list
compl_table_list.append(Table(table_dict))
# stack all the tables into one
compl_table = vstack(compl_table_list)
# if chosen, get the vega fluxes for the filters
if vega_mag:
_, vega_flux, _ = Vega(source=vega_fname).getFlux(filter_list)
# ------ Calculate depth
# initialize dictionary to hold results
depth_dict = {filt: [] for filt in filter_list}
# grab numbers for each filter
for f, filt in enumerate(filter_list):
use_sed = compl_table[filt]
use_comp = compl_table[filt + "_compl"]
# get sorted versions of data
sort_ind = np.argsort(use_sed)
sort_sed = use_sed[sort_ind]
sort_comp = use_comp[sort_ind]
# grab depths
for compl in np.atleast_1d(completeness_value):
# first check whether the noise model even covers this completeness
# (in case there weren't sufficient ASTs)
if (compl < np.min(sort_comp)) or (compl > np.max(sort_comp)):
depth_dict[filt].append(np.nan)
continue
# find first instance of completeness > N
first_ind = np.where(sort_comp > compl)[0][0]
# corresponding flux
comp_flux = sort_sed[first_ind]
# save it
if vega_mag:
depth_dict[filt].append(-2.5 * np.log10(comp_flux / vega_flux[f]))
else:
depth_dict[filt].append(comp_flux)
# return the results
return depth_dict
def plot_noisemodel(
sed_file,
noise_file_list,
plot_file,
samp=100,
cmap_name='viridis',
):
"""
Make a plot of the noise model: for each of the bandsm make plots of bias
and uncertainty as a function of flux
If there are multiple files in noise_file_list, each of them will be
overplotted in each panel.
Parameters
----------
sed_file : string
path+name of the SED grid file
noise_file_list : list of strings
path+name of the noise model file(s)
plot_file : string
name of the file to save the plot
samp : int (default=100)
plotting all of the SED points takes a long time for a viewer to load,
so set this to plot every Nth point
cmap_name : string (default=plt.cm.viridis)
name of a color map to use
"""
# read in the SED grid
print("* reading SED grid file")
sed_object = SEDGrid(sed_file)
if hasattr(sed_object.seds, "read"):
sed_grid = sed_object.seds.read()
else:
sed_grid = sed_object.seds
filter_list = sed_object.filters
n_filter = len(filter_list)
# figure
fig, ax = plt.subplots(nrows=3, ncols=n_filter, figsize=(25, 15))
# setup the plots
fontsize = 12
font = {"size": fontsize}
plt.rc("font", **font)
plt.rc("lines", linewidth=2)
plt.rc("axes", linewidth=2)
plt.rc("xtick.major", width=2)
plt.rc("ytick.major", width=2)
plt.set_cmap(cmap_name)
# go through noise files after sorting them according to
# their SD bin number
noise_file_list.sort(key=lambda f: int(''.join(filter(str.isdigit, f))))
bin_label = [re.findall(r"bin\d+", x)[0] for x in noise_file_list]
for n, nfile in enumerate(np.atleast_1d(noise_file_list)):
print("* reading " + nfile)
# read in the values
noisemodel_vals = noisemodel.get_noisemodelcat(nfile)
# extract error and bias
noise_err = noisemodel_vals["error"]
noise_bias = noisemodel_vals["bias"]
noise_compl = noisemodel_vals["completeness"]
# plot things
for f, filt in enumerate(filter_list):
# error is negative where it's been extrapolated -> trim those
good_err = np.where(noise_err[:, f] > 0)[0]
plot_sed = sed_grid[good_err, f][::samp]
plot_err = noise_err[good_err, f][::samp]
plot_bias = noise_bias[good_err, f][::samp]
plot_compl = noise_compl[good_err, f][::samp]
# bias
bax = ax[0, f]
bax.plot(
np.log10(plot_sed),
plot_bias / plot_sed,
marker="o",
linestyle="none",
mew=0,
ms=2,
alpha=0.1,
label='SD %s' % (bin_label[n]),
)
bax.tick_params(axis="both", which="major")
bax.set_xlabel("log " + filt)
bax.set_ylabel(r"Bias ($\mu$/F)")
leg = bax.legend(loc='lower right', markerscale=3)
for lh in leg.legendHandles:
lh._legmarker.set_alpha(1)
# error
eax = ax[1, f]
eax.plot(
np.log10(plot_sed),
plot_err / plot_sed,
marker="o",
linestyle="none",
mew=0,
ms=2,
alpha=0.1,
)
eax.tick_params(axis="both", which="major")
eax.set_xlabel("log " + filt)
eax.set_ylabel(r"Error ($\sigma$/F)")
# completeness
cax = ax[2, f]
cax.plot(
np.log10(plot_sed),
plot_compl,
marker="o",
linestyle="none",
mew=0,
ms=2,
alpha=0.1,
)
cax.tick_params(axis="both", which="major")
cax.set_xlabel("log " + filt)
cax.set_ylabel(r"Completeness")
plt.tight_layout()
fig.savefig(plot_file, dpi=300)
plt.close(fig)
"trimfile", help="file with modelgrid, obsfiles, filebase to use"
)
args = parser.parse_args()
start_time = time.clock()
# read in trim file
f = open(args.trimfile, "r")
file_lines = list(f)
# physics model grid name
modelfile = file_lines[0].rstrip()
# get the modesedgrid on which to generate the noisemodel
print("Reading the model grid files = ", modelfile)
modelsedgrid = SEDGrid(modelfile)
new_time = time.clock()
print("time to read: ", (new_time - start_time) / 60.0, " min")
old_noisefile = ""
for k in range(1, len(file_lines)):
print("\n\n")
# file names
noisefile, obsfile, filebase = file_lines[k].split()
# make sure the proper directories exist
if not os.path.isdir(os.path.dirname(filebase)):
os.makedirs(os.path.dirname(filebase))
def make_extinguished_sed_grid(
project,
specgrid,
filters,
av=[0.0, 5, 0.1],
rv=[0.0, 5, 0.2],
fA=None,
av_prior_model={"name": "flat"},
rv_prior_model={"name": "flat"},
fA_prior_model={"name": "flat"},
extLaw=None,
add_spectral_properties_kwargs=None,
absflux_cov=False,
verbose=True,
seds_fname=None,
filterLib=None,
info_fname=None,
**kwargs,
):
"""
Create SED model grid integrated with filters and dust extinguished
Parameters
----------
project: str
project name
specgrid: SpectralGrid object
spectral grid to transform
filters: sequence
ordered sequence of filters to use to extract the photometry
filter names are the full names in core.filters
av: sequence
sequence of Av values to sample
av_prior_model: dict
dict including prior model name and parameters
rv: sequence
sequence of Rv values to sample
rv_prior_model: dict
dict including prior model name and parameters
fA: sequence (optional)
sequence of fA values to sample (depending on extLaw definition)
fA_prior_model: dict
dict including prior model name and parameters
extLaw: extinction.ExtLaw
extinction law to use during the process
add_spectral_properties_kwargs: dict
keyword arguments to call :func:`add_spectral_properties`
to add model properties from the spectra into the grid property table
asbflux_cov: boolean
set to calculate the absflux covariance matrices for each model
(can be very slow!!! But it is the right thing to do)
seds_fname: str
full filename to save the sed grid into
filterLib: str
full filename to the filter library hd5 file
info_fname : str
Set to specify the filename to save beast info to, otherwise
saved to project/project_beast_info.asdf
Returns
-------
fname: str
name of saved file
g: SpectralGrid object
spectral grid to transform
"""
# create the dust grid arrays
avs = np.arange(av[0], av[1] + 0.5 * av[2], av[2])
rvs = np.arange(rv[0], rv[1] + 0.5 * rv[2], rv[2])
if fA is not None:
fAs = np.arange(fA[0], fA[1] + 0.5 * fA[2], fA[2])
else:
fAs = [1.0]
# create SED file name if needed
if seds_fname is None:
seds_fname = "%s/%s_seds.grid.hd5" % (project, project)
# generate extinguished grids if SED file doesn't exist
if not os.path.isfile(seds_fname):
extLaw = extLaw or extinction.Cardelli()
if verbose:
print("Make SEDS")
if fA is not None:
g = creategrid.make_extinguished_grid(
specgrid,
filters,
extLaw,
avs,
rvs,
fAs,
av_prior_model=av_prior_model,
rv_prior_model=rv_prior_model,
fA_prior_model=fA_prior_model,
add_spectral_properties_kwargs=add_spectral_properties_kwargs,
absflux_cov=absflux_cov,
filterLib=filterLib,
)
else:
g = creategrid.make_extinguished_grid(
specgrid,
filters,
extLaw,
avs,
rvs,
av_prior_model=av_prior_model,
rv_prior_model=rv_prior_model,
add_spectral_properties_kwargs=add_spectral_properties_kwargs,
absflux_cov=absflux_cov,
)
# write to disk
if hasattr(g, "write"):
g.write(seds_fname)
else:
for gk in g:
gk.write(seds_fname, append=True)
# save info to the beast info file
info = {
"av_input": av,
"rv_input": rv,
"fA_input": fA,
"avs": avs,
"rvs": rvs,
"fAs": fAs,
"av_prior_model": av_prior_model,
"rv_prior_model": rv_prior_model,
"fA_prior_model": fA_prior_model,
}
if info_fname is None:
info_fname = f"{project}/{project}_beast_info.asdf"
add_to_beast_info_file(info_fname, info)
g = SEDGrid(seds_fname, backend="memory")
return (seds_fname, g)
def subgrid_info(grid_fname, noise_fname=None):
"""
Generates a list of mins and maxes of all the quantities in the given grid
Parameters
----------
grid_fname: string
path to a beast grid file (hd5 format)
noise_fname: string
Path to the noise model file for the given grid (hd5 format)
(optional). If this is given, the mins/maxes for the full model
fluxes are added too, under the name 'log'+filter+'_wd_bias'
(needs to conform to the name used in fit.py).
Returns
-------
info_dict: dictionary
{name of quantity [string]: {'min': min, 'max': max, 'unique': unique values}}
"""
# Use the disk backend to minimize the memory usage
sedgrid = SEDGrid(grid_fname, backend="disk")
seds = sedgrid.seds
info_dict = {}
qnames = sedgrid.keys()
for q in qnames:
qvals = sedgrid[q]
qmin = np.amin(qvals)
qmax = np.amax(qvals)
qunique = np.unique(qvals)
info_dict[q] = {}
info_dict[q]["min"] = qmin
info_dict[q]["max"] = qmax
info_dict[q]["unique"] = qunique
if noise_fname is not None:
noisemodel = get_noisemodelcat(noise_fname)
# The following is also in fit.py, so we're kind of doing double
# work here, but it's necessary if we want to know the proper
# ranges for these values.
full_model_flux = seds[:] + noisemodel["bias"]
logtempseds = np.array(full_model_flux)
full_model_flux = (np.sign(logtempseds) *
np.log1p(np.abs(logtempseds * math.log(10))) /
math.log(10))
filters = sedgrid.filters
for i, f in enumerate(filters):
f_fluxes = full_model_flux[:, i]
# Be sure to cut out the -100's in the calculation of the minimum
qmin = np.amin(f_fluxes[f_fluxes > -99.99])
qmax = np.amax(f_fluxes)
qunique = np.unique(qvals)
q = "symlog" + f + "_wd_bias"
info_dict[q] = {}
info_dict[q]["min"] = qmin
info_dict[q]["max"] = qmax
info_dict[q]["unique"] = qunique
print("Gathered grid info for {}".format(grid_fname))
return info_dict
def plot_completeness(
physgrid_list,
noise_model_list,
output_plot_filename,
param_list=["Av", "Rv", "logA", "f_A", "M_ini", "Z", "distance"],
compl_filter="F475W",
):
"""
Make visualization of the completeness
Parameters
----------
physgrid_list : string or list of strings
Name of the physics model file. If there are multiple physics model
grids (i.e., if there are subgrids), list them all here.
noise_model_list : string or list of strings
Name of the noise model file. If there are multiple files for
physgrid_list (because of subgrids), list the noise model file
associated with each physics model file.
param_list : list of strings
names of the parameters to plot
compl_filter : str
filter to use for completeness (required for toothpick model)
output_plot_filename : string
name of the file in which to save the output plot
"""
n_params = len(param_list)
# If there are subgrids, we can't read them all into memory. Therefore,
# we'll go through each one and just grab the relevant parts.
compl_table_list = []
# make a table for each physics model + noise model
for physgrid, noise_model in zip(np.atleast_1d(physgrid_list),
np.atleast_1d(noise_model_list)):
# get the physics model grid - includes priors
modelsedgrid = SEDGrid(str(physgrid))
# get list of filters
short_filters = [
filter.split(sep="_")[-1].upper()
for filter in modelsedgrid.filters
]
if compl_filter.upper() not in short_filters:
raise ValueError("requested completeness filter not present")
filter_k = short_filters.index(compl_filter.upper())
print("Completeness from {0}".format(modelsedgrid.filters[filter_k]))
# read in the noise model
noisegrid = noisemodel.get_noisemodelcat(str(noise_model))
# get the completeness
model_compl = noisegrid["completeness"]
# close the file to save memory
noisegrid.close()
# put it all into a table
table_dict = {x: modelsedgrid[x] for x in param_list}
table_dict["compl"] = model_compl[:, filter_k]
# append to the list
compl_table_list.append(Table(table_dict))
# stack all the tables into one
compl_table = vstack(compl_table_list)
# import pdb; pdb.set_trace()
# figure
fig = plt.figure(figsize=(4 * n_params, 4 * n_params))
# label font sizes
label_font = 25
tick_font = 22
# load in color map
cmap = matplotlib.cm.get_cmap("magma")
# iterate through the panels
for i, pi in enumerate(param_list):
for j, pj in enumerate(param_list[i:], i):
print("plotting {0} and {1}".format(pi, pj))
# not along diagonal
if i != j:
# set up subplot
plt.subplot(n_params, n_params, i + j * (n_params) + 1)
ax = plt.gca()
# create image and labels
x_col, x_bins, x_label = setup_axis(compl_table, pi)
y_col, y_bins, y_label = setup_axis(compl_table, pj)
compl_image, _, _, _ = binned_statistic_2d(
x_col,
y_col,
compl_table["compl"],
statistic="mean",
bins=(x_bins, y_bins),
)
# plot points
im = plt.imshow(
compl_image.T,
# np.random.random((4,4)),
extent=(
np.min(x_bins),
np.max(x_bins),
np.min(y_bins),
np.max(y_bins),
),
cmap="magma",
vmin=0,
vmax=1,
aspect="auto",
origin="lower",
)
ax.tick_params(
axis="both",
which="both",
direction="in",
labelsize=tick_font,
bottom=True,
top=True,
left=True,
right=True,
)
# axis labels and ticks
if i == 0:
ax.set_ylabel(y_label, fontsize=label_font)
# ax.get_yaxis().set_label_coords(-0.35,0.5)
else:
ax.set_yticklabels([])
if j == n_params - 1:
ax.set_xlabel(x_label, fontsize=label_font)
plt.xticks(rotation=-45)
else:
ax.set_xticklabels([])
# along diagonal
if i == j:
# set up subplot
plt.subplot(n_params, n_params, i + j * (n_params) + 1)
ax = plt.gca()
# create histogram and labels
x_col, x_bins, x_label = setup_axis(compl_table, pi)
compl_hist, _, _ = binned_statistic(
x_col,
compl_table["compl"],
statistic="mean",
bins=x_bins,
)
# make histogram
_, _, patches = plt.hist(x_bins[:-1],
x_bins,
weights=compl_hist)
# color each bar by its completeness
for c, comp in enumerate(compl_hist):
patches[c].set_color(cmap(comp))
patches[c].set_linewidth = 0.1
# make a black outline so it stands out as a histogram
plt.hist(x_bins[:-1],
x_bins,
weights=compl_hist,
histtype="step",
color="k")
# axis ranges
plt.xlim(np.min(x_bins), np.max(x_bins))
plt.ylim(0, 1.05)
ax.tick_params(axis="y",
which="both",
length=0,
labelsize=tick_font)
ax.tick_params(axis="x",
which="both",
direction="in",
labelsize=tick_font)
# axis labels and ticks
ax.set_yticklabels([])
if i < n_params - 1:
ax.set_xticklabels([])
if i == n_params - 1:
ax.set_xlabel(x_label, fontsize=label_font)
plt.xticks(rotation=-45)
# plt.subplots_adjust(wspace=0.05, hspace=0.05)
plt.tight_layout()
# add a colorbar
gs = GridSpec(nrows=20, ncols=n_params)
cax = fig.add_subplot(gs[0, 2:])
cbar = plt.colorbar(im, cax=cax, orientation="horizontal")
cbar.set_label("Completeness", fontsize=label_font)
cbar.ax.tick_params(labelsize=tick_font)
gs.tight_layout(fig)
fig.savefig(output_plot_filename)
plt.close(fig)
def make_extinguished_grid(
spec_grid,
filter_names,
extLaw,
avs,
rvs,
fAs=None,
av_prior_model={"name": "flat"},
rv_prior_model={"name": "flat"},
fA_prior_model={"name": "flat"},
chunksize=0,
add_spectral_properties_kwargs=None,
absflux_cov=False,
filterLib=None,
):
"""
Extinguish spectra and extract an SEDGrid through given series of filters
(all wavelengths in stellar SEDs and filter response functions are assumed
to be in Angstroms)
Parameters
----------
spec_grid: string or grid.SpectralGrid
if string:
spec_grid is the filename to the grid file with stellar spectra
the backend to load this grid will be the minimal invasive: 'HDF'
if possible, 'cache' otherwise.
if not a string, expecting the corresponding SpectralGrid instance
(backend already setup)
filter_names: list
list of filter names according to the filter lib
Avs: sequence
Av values to iterate over
av_prior_model: list
list including prior model name and parameters
Rvs: sequence
Rv values to iterate over
rv_prior_model: list
list including prior model name and parameters
fAs: sequence (optional)
f_A values to iterate over
f_A can be omitted if the extinction Law does not use it or allow
fixed values
fA_prior_model: list
list including prior model name and parameters
chunksize: int, optional (default=0)
number of extinction model variations to generate at each cycle.
Note that this means len(spec_grid * chunksize)
If default <= 0, all models will be returned at once.
filterLib: str
full filename to the filter library hd5 file
add_spectral_properties_kwargs: dict
keyword arguments to call :func:`add_spectral_properties` at each
iteration to add model properties from the spectra into the grid
property table
asbflux_cov: boolean
set to calculate the absflux covariance matrices for each model
(can be very slow!!! But it is the right thing to do)
Returns
-------
g: grid.SpectralGrid
final grid of reddened SEDs and models
"""
# Check inputs
# ============
# get the stellar grid (no dust yet)
# if string is provided try to load the most memory efficient backend
# otherwise use a cache-type backend (load only when needed)
if isinstance(spec_grid, str):
ext = spec_grid.split(".")[-1]
if ext in ["hdf", "hd5", "hdf5"]:
g0 = SpectralGrid(spec_grid, backend="disk")
else:
g0 = SpectralGrid(spec_grid, backend="cache")
else:
helpers.type_checker("spec_grid", spec_grid, SpectralGrid)
g0 = spec_grid
# Tag fA usage
if fAs is None:
with_fA = False
else:
with_fA = True
# get the min/max R(V) values necessary for the grid point definition
min_Rv = min(rvs)
max_Rv = max(rvs)
# Create the sampling mesh
# ========================
# basically the dot product from all input 1d vectors
# setup interation over the full dust parameter grid
if with_fA:
dustpriors = PriorWeightsDust(avs, av_prior_model, rvs, rv_prior_model,
fAs, fA_prior_model)
it = np.nditer(np.ix_(avs, rvs, fAs))
niter = np.size(avs) * np.size(rvs) * np.size(fAs)
npts, pts = _make_dust_fA_valid_points_generator(it, min_Rv, max_Rv)
# Pet the user
print("""number of initially requested points = {0:d}
number of valid points = {1:d} (based on restrictions in R(V)
versus f_A plane)
""".format(niter, npts))
if npts == 0:
raise AttributeError("No valid points")
else:
dustpriors = PriorWeightsDust(avs, av_prior_model, rvs, rv_prior_model,
[1.0], fA_prior_model)
it = np.nditer(np.ix_(avs, rvs))
npts = np.size(avs) * np.size(rvs)
pts = ((float(ak), float(rk)) for ak, rk in it)
# Generate the Grid
# =================
N0 = len(g0.grid)
N = N0 * npts
if chunksize <= 0:
print("Generating a final grid of {0:d} points".format(N))
else:
print("Generating a final grid of {0:d} points in {1:d} pieces".format(
N, int(float(N0) / chunksize + 1.0)))
if chunksize <= 0:
chunksize = npts
if add_spectral_properties_kwargs is not None:
nameformat = add_spectral_properties_kwargs.pop("nameformat",
"{0:s}") + "_wd"
for chunk_pts in helpers.chunks(pts, chunksize):
# iter over chunks of models
# setup chunk outputs
cols = {"Av": np.zeros(N, dtype=float), "Rv": np.zeros(N, dtype=float)}
if with_fA:
cols["Rv_A"] = np.zeros(N, dtype=float)
cols["f_A"] = np.zeros(N, dtype=float)
keys = list(g0.keys())
for key in keys:
cols[key] = np.zeros(N, dtype=float)
n_filters = len(filter_names)
_seds = np.zeros((N, n_filters), dtype=float)
if absflux_cov:
n_offdiag = ((n_filters**2) - n_filters) / 2
_cov_diag = np.zeros((N, n_filters), dtype=float)
_cov_offdiag = np.zeros((N, n_offdiag), dtype=float)
for count, pt in enumerate(tqdm(chunk_pts, desc="SED grid")):
if with_fA:
Av, Rv, f_A = pt
dust_prior_weight = dustpriors.get_weight(Av, Rv, f_A)
Rv_MW = extLaw.get_Rv_A(Rv, f_A)
r = g0.applyExtinctionLaw(extLaw,
Av=Av,
Rv=Rv,
f_A=f_A,
inplace=False)
# add extra "spectral bands" if requested
if add_spectral_properties_kwargs is not None:
r = add_spectral_properties(
r,
nameformat=nameformat,
filterLib=filterLib,
**add_spectral_properties_kwargs)
temp_results = r.getSEDs(filter_names, filterLib=filterLib)
# adding the dust parameters to the models
cols["Av"][N0 * count:N0 * (count + 1)] = Av
cols["Rv"][N0 * count:N0 * (count + 1)] = Rv
cols["f_A"][N0 * count:N0 * (count + 1)] = f_A
cols["Rv_A"][N0 * count:N0 * (count + 1)] = Rv_MW
else:
Av, Rv = pt
dust_prior_weight = dustpriors.get_weight(Av, Rv, 1.0)
r = g0.applyExtinctionLaw(extLaw, Av=Av, Rv=Rv, inplace=False)
if add_spectral_properties_kwargs is not None:
r = add_spectral_properties(
r,
nameformat=nameformat,
filterLib=filterLib,
**add_spectral_properties_kwargs)
temp_results = r.getSEDs(filter_names, filterLib=filterLib)
# adding the dust parameters to the models
cols["Av"][N0 * count:N0 * (count + 1)] = Av
cols["Rv"][N0 * count:N0 * (count + 1)] = Rv
# get new attributes if exist
for key in list(temp_results.grid.keys()):
if key not in keys:
k1 = N0 * count
k2 = N0 * (count + 1)
cols.setdefault(key, np.zeros(
N, dtype=float))[k1:k2] = temp_results.grid[key]
# compute the fractional absflux covariance matrices
if absflux_cov:
absflux_covmats = calc_absflux_cov_matrices(
r, temp_results, filter_names)
_cov_diag[N0 * count:N0 * (count + 1)] = absflux_covmats[0]
_cov_offdiag[N0 * count:N0 * (count + 1)] = absflux_covmats[1]
# assign the extinguished SEDs to the output object
_seds[N0 * count:N0 * (count + 1)] = temp_results.seds[:]
# copy the rest of the parameters
for key in keys:
cols[key][N0 * count:N0 * (count + 1)] = g0.grid[key]
# multiply existing prior weights by the dust prior weight
cols["weight"][N0 * count:N0 * (count + 1)] *= dust_prior_weight
cols["prior_weight"][N0 * count:N0 *
(count + 1)] *= dust_prior_weight
if count == 0:
cols["lamb"] = temp_results.lamb[:]
_lamb = cols.pop("lamb")
# free the memory of temp_results
# del temp_results
# del tempgrid
# Ship
if absflux_cov:
g = SEDGrid(
_lamb,
seds=_seds,
cov_diag=_cov_diag,
cov_offdiag=_cov_offdiag,
grid=Table(cols),
backend="memory",
)
else:
g = SEDGrid(_lamb, seds=_seds, grid=Table(cols), backend="memory")
g.header["filters"] = " ".join(filter_names)
yield g
def test_grid_warnings():
with pytest.raises(ValueError) as exc:
SEDGrid(backend="hdf")
assert exc.value.args[0] == "hdf backend not supported"
with pytest.raises(ValueError) as exc:
SEDGrid("test.txt")
assert exc.value.args[0] == "txt file type not supported"
# define grid contents
n_bands = 3
# filter_names = ["BAND1", "BAND2", "BAND3"]
n_models = 100
lamb = [1.0, 2.0, 3.0]
seds = np.zeros((n_models, n_bands))
# cov_diag = np.full((n_models, n_bands), 0.1)
# n_offdiag = ((n_bands ** 2) - n_bands) // 2
# cov_offdiag = np.full((n_models, n_offdiag), 1.0)
cols = {"Av": [1.0, 1.1, 1.3], "Rv": [2.0, 3.0, 4.0]}
header = {"Origin": "test_code"}
gtable = Table(cols)
gtable.meta = header
with pytest.raises(ValueError) as exc:
SEDGrid(lamb)
assert exc.value.args[0] == "seds or grid not passed"
for ftype in ["fits", "hdf"]:
with pytest.raises(ValueError) as exc:
a = SEDGrid(lamb, seds=seds, grid=gtable)
a.grid = cols
a.write(f"testgridwriteerror.{ftype}")
assert exc.value.args[0] == "Only astropy.Table are supported"
with pytest.raises(ValueError) as exc:
a = SEDGrid(lamb, seds=seds, grid=gtable)
a.grid = None
a.write(f"testgridwriteerror.{ftype}")
assert exc.value.args[
0] == "Full data set not specified (lamb, seds, grid)"
def test_sedgrid(cformat, cback, copygrid):
"""
Tests of the SEDGrid class
"""
n_bands = 3
filter_names = ["BAND1", "BAND2", "BAND3"]
n_models = 100
lamb = [1.0, 2.0, 3.0]
seds = np.zeros((n_models, n_bands))
cov_diag = np.full((n_models, n_bands), 0.1)
n_offdiag = ((n_bands**2) - n_bands) // 2
cov_offdiag = np.full((n_models, n_offdiag), 1.0)
cols = {"Av": [1.0, 1.1, 1.3], "Rv": [2.0, 3.0, 4.0]}
header = {"Origin": "test_code"}
gtable = Table(cols)
gtable.meta = header
tgrid = SEDGrid(
lamb,
seds=seds,
grid=gtable,
header=header,
cov_diag=cov_diag,
cov_offdiag=cov_offdiag,
backend="memory",
)
tgrid.header["filters"] = " ".join(filter_names)
# check that the grid has the expected properties
expected_props = [
"lamb",
"seds",
"cov_diag",
"cov_offdiag",
"grid",
"nbytes",
"filters",
"header",
"keys",
]
for cprop in expected_props:
assert hasattr(tgrid, cprop), f"missing {cprop} property"
np.testing.assert_allclose(tgrid.lamb, lamb, err_msg="lambdas not equal")
np.testing.assert_allclose(tgrid.seds, seds, err_msg="seds not equal")
np.testing.assert_allclose(tgrid.cov_diag,
cov_diag,
err_msg="covdiag not equal")
np.testing.assert_allclose(tgrid.cov_offdiag,
cov_offdiag,
err_msg="covoffdiag not equal")
assert isinstance(tgrid.nbytes,
(int, np.integer)), "grid nbytes property not integer"
compare_tables(tgrid.grid, gtable)
assert tgrid.grid.keys() == list(cols.keys()), "colnames of grid not equal"
assert tgrid.filters == filter_names, "filters of grid not equal"
# test writing and reading to disk
print(f"testing {cformat} file format")
tfile = NamedTemporaryFile(suffix=cformat)
# write the file
tgrid.write(tfile.name)
# read in the file using different backends
if (cback == "disk") and (cformat == ".fits"): # not supported
return True
print(f" testing {cback} backend")
dgrid_in = SEDGrid(tfile.name, backend=cback)
# test making a copy
print(f" testing copygrid={copygrid}")
if copygrid:
dgrid = dgrid_in.copy()
else:
dgrid = dgrid_in
print(dgrid)
for cprop in expected_props:
assert hasattr(dgrid, cprop), f"missing {cprop} property"
# check that the grid has the expected values
# this test is having a problem in the online travis ci
# it someone manages to access another file with HST filter names!
# no idea way. Works fine offline.
# assert dgrid.filters == filter_names, "{cformat} file filters not equal"
assert len(dgrid) == n_bands, f"{cformat} file len not equal"
np.testing.assert_allclose(
dgrid.lamb, lamb, err_msg=f"{cformat} file grid lambdas not equal")
np.testing.assert_allclose(dgrid.seds,
seds,
err_msg=f"{cformat} file grid seds not equal")
np.testing.assert_allclose(
dgrid.cov_diag,
cov_diag,
err_msg=f"{cformat} file grid cov_diag not equal",
)
np.testing.assert_allclose(
dgrid.cov_offdiag,
cov_offdiag,
err_msg=f"{cformat} file grid cov_offdiag not equal",
)
assert isinstance(
dgrid.nbytes,
(int, np.integer)), f"{cformat} file grid nbytes property not integer"
dTable = dgrid.grid
if (cback == "disk") and (cformat == ".hdf"):
dTable = read_table_hdf5(dgrid.grid)
compare_tables(dTable, gtable, otag=f"{cformat} file")
assert dTable.keys() == list(
cols.keys()), f"{cformat} file colnames of grid not equal"
assert dgrid.keys() == tgrid.keys(
), f"{cformat} file colnames of grid not equal"
# final copy - needed for disk backend to get the now defined variables
print(dgrid)
dgrid_fin = dgrid.copy()
print(dgrid_fin)
def plot_toothpick_details(asts_filename, seds_filename, savefig=False):
"""
Plot the details of the toothpick noisemodel creation for each filter.
These plots show the individual AST results as points as
(flux_in - flux_out)/flux_in. In addition, the binned values of these
points are plotted giving the bias term in the observation model.
Error bars around the binned bias values give the binned sigma term of
the observation model. Finally, as a separate column of plots the
binned completeness in each filter is plotted.
Parameters
----------
asts_filename : str
filename with the AST results
seds_filename : str
filename with the SED grid (used just for the filter information)
savefig : str (default=False)
to save the figure, set this to the file extension (e.g., 'png', 'pdf')
"""
sedgrid = SEDGrid(seds_filename, backend="cache")
# read in AST results
model = toothpick.MultiFilterASTs(asts_filename, sedgrid.filters)
# set the column mappings as the external file is BAND_VEGA or BAND_IN
model.set_data_mappings(upcase=True,
in_pair=("in", "in"),
out_pair=("out", "rate"))
# compute binned biases, uncertainties, and completeness as a function of band flux
model.fit_bins(nbins=50, completeness_mag_cut=-10)
nfilters = len(sedgrid.filters)
fig, ax = plt.subplots(nrows=nfilters,
ncols=2,
figsize=(14, 10),
sharex=True)
set_params()
for i, cfilter in enumerate(sedgrid.filters):
mag_in = model.data[model.filter_aliases[cfilter + "_in"]]
flux_out = model.data[model.filter_aliases[cfilter + "_out"]]
flux_in = (10**(-0.4 * mag_in)) * model.vega_flux[i]
flux_out *= model.vega_flux[i]
gvals = flux_out != 0.0
ax[i, 0].plot(
flux_in[gvals],
(flux_in[gvals] - flux_out[gvals]) / flux_in[gvals],
"ko",
alpha=0.1,
markersize=2,
)
# not all bins are filled with good data
ngbins = model._nasts[i]
ax[i, 0].plot(
model._fluxes[0:ngbins, i],
model._biases[0:ngbins, i] / model._fluxes[0:ngbins, i],
"b-",
)
ax[i, 0].errorbar(
model._fluxes[0:ngbins, i],
model._biases[0:ngbins, i] / model._fluxes[0:ngbins, i],
yerr=model._sigmas[0:ngbins, i] / model._fluxes[0:ngbins, i],
fmt="bo",
markersize=2,
alpha=0.5,
)
ax[i, 0].set_ylim(-5e0, 5e0)
ax[i, 0].set_xscale("log")
ax[i, 0].set_ylabel(r"$(F_i - F_o)/F_i$")
ax[i, 1].plot(
model._fluxes[0:ngbins, i],
model._compls[0:ngbins, i],
"b-",
)
ax[i, 1].yaxis.tick_right()
ax[i, 1].yaxis.set_label_position("right")
ax[i, 1].set_ylim(0, 1)
ax[i, 1].set_xscale("log")
sfilt = cfilter.split("_")[-1]
ax[i, 1].set_ylabel(f"C({sfilt})")
ax[nfilters - 1, 0].set_xlabel(r"$F_i$")
ax[nfilters - 1, 1].set_xlabel(r"$F_i$")
# add in the zero line
# do after all the data has been plotted to get the full x range
pxrange = ax[0, 0].get_xlim()
for i, cfilter in enumerate(sedgrid.filters):
ax[i, 0].plot(pxrange, [0.0, 0.0], "k--", alpha=0.5)
# figname
basename = asts_filename.replace(".fits", "_plot")
fig.tight_layout()
# save or show fig
if savefig:
fig.savefig("{}.{}".format(basename, savefig))
else:
plt.show()
def fit_submodel(
settings,
photometry_file,
modelsedgrid_file,
noise_file,
pdf_max_nbins,
stats_file,
pdf_file,
pdf2d_file,
pdf2d_param_list,
lnp_file,
grid_info_file=None,
resume=False,
):
"""
Code to run the SED fitting
Parameters
----------
settings : beast.tools.beast_settings.beast_settings instance
object with the beast settings
photometry_file : string
path+name of the photometry file
modelsedgrid_file : string
path+name of the physics model grid file
noise_file : string
path+name of the noise model file
pdf_max_nbins : int
Maxiumum number of bins to use for the 1D and 2D PDFs
stats_file : string
path+name of the file to contain stats output
pdf_file : string
path+name of the file to contain 1D PDF output
pdf2d_file : string
path+name of the file to contain 2D PDF output
pdf2d_param_list: list of strings or None
parameters for which to make 2D PDFs (or None)
lnp_file : string
path+name of the file to contain log likelihood output
grid_info_file : string (default=None)
path+name for pickle file that contains dictionary with subgrid
min/max/n_unique (required for a run with subgrids)
resume : boolean (default=False)
choose whether to resume existing run or start over
Returns
-------
noisefile : string
name of the created noise file
"""
# read in the photometry catalog
obsdata = Observations(photometry_file,
settings.filters,
obs_colnames=settings.obs_colnames)
# check if it's a subgrid run by looking in the file name
if "gridsub" in modelsedgrid_file:
subgrid_run = True
print("loading grid_info_dict from " + grid_info_file)
with open(grid_info_file, "rb") as p:
grid_info_dict = pickle.loads(p.read())
else:
subgrid_run = False
# load the SED grid and noise model
modelsedgrid = SEDGrid(modelsedgrid_file)
noisemodel_vals = noisemodel.get_noisemodelcat(noise_file)
if subgrid_run:
fit.summary_table_memory(
obsdata,
noisemodel_vals,
modelsedgrid,
resume=resume,
threshold=-10.0,
save_every_npts=100,
lnp_npts=500,
max_nbins=pdf_max_nbins,
stats_outname=stats_file,
pdf1d_outname=pdf_file,
pdf2d_outname=pdf2d_file,
pdf2d_param_list=pdf2d_param_list,
grid_info_dict=grid_info_dict,
lnp_outname=lnp_file,
do_not_normalize=True,
surveyname=settings.surveyname,
)
print("Done fitting on grid " + modelsedgrid_file)
else:
fit.summary_table_memory(
obsdata,
noisemodel_vals,
modelsedgrid,
resume=resume,
threshold=-10.0,
save_every_npts=100,
lnp_npts=500,
max_nbins=pdf_max_nbins,
stats_outname=stats_file,
pdf1d_outname=pdf_file,
pdf2d_outname=pdf2d_file,
pdf2d_param_list=pdf2d_param_list,
lnp_outname=lnp_file,
surveyname=settings.surveyname,
)
print("Done fitting on grid " + modelsedgrid_file)
def pick_models(
sedgrid_fname,
filters,
mag_cuts,
Nfilter=3,
N_stars=70,
Nrealize=20,
outfile=None,
outfile_params=None,
bright_cut=None,
vega_fname=None,
ranseed=None,
):
"""Creates a fake star catalog from a BEAST model grid
Parameters
----------
sedgrid_fname: string
BEAST model grid from which the models are picked (hdf5 file)
filters: list of string
Names of the filters
mag_cuts: list
List of magnitude limits for each filter
Nfilter: Integer
In how many filters, you want a fake star to be brighter
than the limit (mag_cut) (default = 3)
N_stars: Integer
Number of stellar models picked per a single log(age)
(default=70)
Nrealize: Integer
Number of realization of each models (default = 20)
outfile: str
If a file name is given, the selected models will be written to
disk
outfile_params: str
If a file name is given, the physical parameters associated with
each model will be written to disk
bright_cut: list of float
Same as mag_cuts, but for the bright end
vega_fname: str
filename of vega file
ranseed : int
used to set the seed to make the results reproducable
useful for testing
Returns
-------
astropy Table of selected models
- and optionally -
ascii file: A list of selected models, written to 'outfile'
fits file: the corresponding physical parameters, written to 'outfile_params'
"""
with Vega(source=vega_fname) as v: # Get the vega fluxes
vega_f, vega_flux, lamb = v.getFlux(filters)
modelsedgrid = SEDGrid(sedgrid_fname)
# Convert to Vega mags
sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux)
# make sure Nfilters isn't larger than the total number of filters
if Nfilter > len(filters):
Nfilter = len(filters)
# Select the models above the magnitude limits in N filters
idxs = mag_limits(sedsMags,
mag_cuts,
Nfilter=Nfilter,
bright_cut=bright_cut)
cols = {}
for key in list(modelsedgrid.grid.keys()):
cols[key] = modelsedgrid.grid[key][idxs]
grid_cut = Table(cols)
# Sample the model grid uniformly
prime_params = np.column_stack(
(grid_cut["logA"], grid_cut["M_ini"], grid_cut["Av"]))
search_age = np.unique(prime_params[:, 0])
N_sample = N_stars
model_ind = [] # indices for the model grid
ast_params = grid_cut[[]] # the corresponding model parameters
# set the random seed - mainly for testing
if not None:
np.random.seed(ranseed)
for iage in search_age:
(tmp, ) = np.where(prime_params[:, 0] == iage)
new_ind = np.random.choice(tmp, N_sample)
model_ind.append(new_ind)
[ast_params.add_row(grid_cut[new_ind[i]]) for i in range(len(new_ind))]
index = np.repeat(idxs[np.array(model_ind).reshape((-1))], Nrealize)
sedsMags = Table(sedsMags[index, :], names=filters)
if outfile is not None:
ascii.write(
sedsMags,
outfile,
overwrite=True,
formats={k: "%.5f"
for k in sedsMags.colnames},
)
if outfile_params is not None:
ast_params.write(outfile_params, overwrite=True)
return sedsMags
def plot_ast(ast_file, sed_grid_file=None):
"""
Make a histogram of the AST fluxes. If an SED grid is given, also plot
a comparison histogram of the SED fluxes.
The histogram bins are set by the bins originally used to create the ASTs
(using the flux bin method), which are saved in
ast_file.replace('inputAST','ASTfluxbins')
and are automatically read in.
Output plot is saved in the same location/name as ast_file, but with a .png
instead of .txt.
Parameters
----------
ast_file : string
name of AST input file
sed_grid_file : string (default=None)
name of SED grid file
"""
# read in AST info
ast_table = Table.read(ast_file, format="ascii")
ast_fluxbins = Table.read(ast_file.replace("inputAST", "ASTfluxbins"),
format="ascii")
# get filter names (and it's even in order!)
filter_cols = [col for col in ast_table.colnames if "_" in col]
filter_list = [col[-5:] for col in filter_cols]
n_filter = len(filter_list)
# if chosen, read in model grid
if sed_grid_file is not None:
modelsedgrid = SEDGrid(sed_grid_file)
with Vega() as v:
_, vega_flux, _ = v.getFlux(filter_cols)
sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux)
# make a histogram for each filter
fig = plt.figure(figsize=(7, 4 * n_filter))
for f, filt in enumerate(filter_list):
# names of table columns with bin values
min_bin_col = [
b for b in ast_fluxbins.colnames if ("mins" in b and filt in b)
][0]
max_bin_col = [
b for b in ast_fluxbins.colnames if ("maxs" in b and filt in b)
][0]
# use those to make a bin list
bin_list = np.append(ast_fluxbins[min_bin_col],
ast_fluxbins[max_bin_col][-1])
# make histograms
ax = plt.subplot(n_filter, 1, f + 1)
ast_col = [b for b in ast_table.colnames if filt in b][0]
plt.hist(
ast_table[ast_col],
bins=bin_list,
density=True,
facecolor="black",
edgecolor="none",
alpha=0.3,
label="ASTs",
)
if sed_grid_file is not None:
plt.hist(
sedsMags[:, f],
bins=bin_list,
density=True,
histtype="step",
facecolor="none",
edgecolor="black",
label="Model grid",
)
# labels
ax.tick_params(axis="both", which="major", labelsize=13)
ax.set_xlim(ax.get_xlim()[::-1])
plt.xlabel(filt + " (Vega mag)", fontsize=14)
plt.ylabel("Normalized Histogram", fontsize=14)
# add legend in first plot
if f == 0:
ax.legend(fontsize=14)
plt.tight_layout()
fig.savefig(ast_file.replace(".txt", ".png"))
plt.close(fig)
def pick_models_toothpick_style(
sedgrid_fname,
filters,
Nfilter,
N_fluxes,
min_N_per_flux,
outfile=None,
outfile_params=None,
bins_outfile=None,
bright_cut=None,
):
"""
Creates a fake star catalog from a BEAST model grid. The chosen seds
are optimized for the toothpick model, by working with a given
number of flux bins, and making sure that every flux bin is covered
by at least a given number of models (for each filter individually,
which is how the toothpick model works).
Parameters
----------
sedgrid_fname: string
BEAST model grid from which the models are picked (hdf5 file)
filters: list of string
Names of the filters, to be used as columns of the output table
Nfilter: integer
In how many filters a fake star needs to be brighter than the
mag_cut value
N_fluxes: integer
The number of flux bins into which the dynamic range of the
model grid in each filter is divided
min_N_per_flux: integer
Minimum number of model seds that need to fall into each bin
outfile: string
Output path for the models (optional). If this file already
exists, the chosen seds are loaded from this file instead.
outfile_params: string (default=None)
If a file name is given, the physical parameters associated with
each model will be written to disk
bins_outfile: string
Output path for a file containing the flux bin limits for each
filter, and the number of samples for each (optional)
bright_cut: list of float
List of magnitude limits for each filter (won't sample model
SEDs that are too bright)
Returns
-------
sedsMags: astropy Table
A table containing the selected model seds (columns are named
after the filters)
"""
if outfile is not None and os.path.isfile(outfile):
print(
"{} already exists. Will attempt to load SEDs for ASTs from there."
.format(outfile))
t = Table.read(outfile, format="ascii")
return t
with Vega() as v:
vega_f, vega_flux, lambd = v.getFlux(filters)
modelsedgrid = SEDGrid(sedgrid_fname)
sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux)
Nseds = sedsMags.shape[0]
Nf = sedsMags.shape[1]
idxs = np.arange(Nseds)
# Check if logL=-9.999 model points sliently sneak through
if min(modelsedgrid.grid["logL"]) < -9:
warnings.warn("There are logL=-9.999 model points in the SED grid!")
print("Excluding those SED models from selecting input ASTs")
idxs = np.where(modelsedgrid.grid["logL"] > -9)[0]
sedsMags = sedsMags[idxs]
# Set up a number of flux bins for each filter
maxes = np.amax(sedsMags, axis=0)
mins = np.amin(sedsMags, axis=0)
bin_edges = np.zeros((N_fluxes + 1, Nf)) # indexed on [fluxbin, nfilters]
for f in range(Nf):
bin_edges[:, f] = np.linspace(mins[f], maxes[f], N_fluxes + 1)
bin_mins = bin_edges[:-1, :]
bin_maxs = bin_edges[1:, :]
if not len(bin_mins) == len(bin_maxs) == N_fluxes:
raise AssertionError()
bin_count = np.zeros((N_fluxes, Nf))
chosen_idxs = []
counter = 0
successes = 0
include_mask = np.full(idxs.shape, True, dtype=bool)
chunksize = 100000
while True:
counter += 1
# pick some random models
rand_idx = np.random.choice(idxs[include_mask], size=chunksize)
randomseds = sedsMags[rand_idx, :]
# Find in which bin each model belongs, for each filter
fluxbins = np.zeros(randomseds.shape, dtype=int)
for fltr in range(Nf):
fluxbins[:, fltr] = np.digitize(randomseds[:, fltr],
bin_maxs[:, fltr])
# Clip in place (models of which the flux is equal to the max
# are assigned bin nr N_fluxes. Move these down to bin nr
# N_fluxes - 1)
np.clip(fluxbins, a_min=0, a_max=N_fluxes - 1, out=fluxbins)
add_these = np.full((len(rand_idx)), False, dtype=bool)
for r in range(len(rand_idx)):
# If any of the flux bins that this model falls into does
# not have enough samples yet, add it to the list of model
# spectra to be output
if (bin_count[fluxbins[r, :], range(Nf)] < min_N_per_flux).any():
bin_count[fluxbins[r, :], range(Nf)] += 1
successes += 1
add_these[r] = True
# If all these bins are full...
else:
# ... do not include this model again, since we will reject it
# anyway.
include_mask[idxs == rand_idx] = False
# Add the approved models
chosen_idxs.extend(rand_idx[add_these])
# If some of the randomly picked models were not added
if not add_these.any():
# ... check if we have enough samples everywhere, or if all
# the models have been exhausted (and hence the bins are
# impossible to fill).
enough_samples = (bin_count.flatten() >= min_N_per_flux).all()
still_models_left = include_mask.any()
if enough_samples or not still_models_left:
break
if not counter % 10:
print("Sampled {} models. {} successfull seds. Ratio = {}".format(
counter * chunksize, successes,
successes / counter / chunksize))
print("Bin array:")
print(bin_count)
# Gather the selected model seds in a table
sedsMags = Table(sedsMags[chosen_idxs, :], names=filters)
if outfile is not None:
ascii.write(
sedsMags,
outfile,
overwrite=True,
formats={k: "%.5f"
for k in sedsMags.colnames},
)
# if chosen, save the corresponding model parameters
if outfile_params is not None:
grid_dict = {}
for key in list(modelsedgrid.grid.keys()):
grid_dict[key] = modelsedgrid.grid[key][chosen_idxs]
grid_dict["sedgrid_indx"] = chosen_idxs
ast_params = Table(grid_dict)
ast_params.write(outfile_params, overwrite=True)
if bins_outfile is not None:
bin_info_table = Table()
col_bigarrays = [bin_mins, bin_maxs, bin_count]
col_basenames = ["bin_mins_", "bin_maxs_", "bin_count_"]
for fltr, filter_name in enumerate(filters):
for bigarray, basename in zip(col_bigarrays, col_basenames):
bin_info_table.add_column(
Column(bigarray[:, fltr], name=basename + filter_name))
ascii.write(bin_info_table, bins_outfile, overwrite=True)
return sedsMags
def simulate_obs(
physgrid_list,
noise_model_list,
output_catalog,
nsim=100,
compl_filter="F475W",
weight_to_use='weight',
ranseed=None,
):
"""
Wrapper for creating a simulated photometry.
Parameters
----------
physgrid_list : list of strings
Name of the physics model file. If there are multiple physics model
grids (i.e., if there are subgrids), list them all here, and they will
each be sampled nsim/len(physgrid_list) times.
noise_model_list : list of strings
Name of the noise model file. If there are multiple files for
physgrid_list (because of subgrids), list the noise model file
associated with each physics model file.
output_catalog : string
Name of the output simulated photometry catalog
n_sim : int (default=100)
Number of simulated objects to create. If nsim/len(physgrid_list) isn't
an integer, this will be increased so that each grid has the same
number of samples.
compl_filter : string (default='F475W')
filter name to use for completeness
weight_to_use : string (default='weight')
Set to either 'weight' (prior+grid), 'prior_weight', or 'grid_weight' to
choose the weighting for SED selection.
ranseed : int
seed for random number generator
"""
# numbers of samples to do
# (ensure there are enough for even sampling of multiple model grids)
n_phys = len(physgrid_list)
samples_per_grid = int(np.ceil(nsim / n_phys))
# list to hold all simulation tables
simtable_list = []
# make a table for each physics model + noise model
for physgrid, noise_model in zip(np.atleast_1d(physgrid_list),
np.atleast_1d(noise_model_list)):
# get the physics model grid - includes priors
modelsedgrid = SEDGrid(str(physgrid))
# read in the noise model - includes bias, unc, and completeness
noisegrid = noisemodel.get_noisemodelcat(str(noise_model))
# generate the table
simtable = gen_SimObs_from_sedgrid(
modelsedgrid,
noisegrid,
nsim=samples_per_grid,
compl_filter=compl_filter,
weight_to_use=weight_to_use,
ranseed=ranseed,
)
# append to the list
simtable_list.append(simtable)
# stack all the tables into one and write it out
vstack(simtable_list).write(output_catalog, overwrite=True)
def generate_files_for_tests(run_beast=True, run_tools=True):
"""
Use the metal_small example to generate a full set of files for the BEAST
regression tests.
Parameters
----------
run_beast : boolean (default=True)
if True, run the BEAST
run_tools : boolean (default=True)
if True, run the code to generate things for tools
"""
# read in BEAST settings
settings_orig = beast_settings.beast_settings("beast_settings.txt")
# also make a version with subgrids
settings_subgrids = copy.deepcopy(settings_orig)
settings_subgrids.n_subgrid = 2
settings_subgrids.project = f"{settings_orig.project}_subgrids"
# ==========================================
# run the beast for each set of settings
# ==========================================
if run_beast:
for settings in [settings_orig, settings_subgrids]:
# -----------------
# physics model
# -----------------
create_physicsmodel.create_physicsmodel(
settings,
nsubs=settings.n_subgrid,
nprocs=1,
)
# -----------------
# ASTs
# -----------------
# currently only works for no subgrids
if settings.n_subgrid == 1:
make_ast_inputs.make_ast_inputs(settings,
pick_method="flux_bin_method")
# -----------------
# obs model
# -----------------
create_obsmodel.create_obsmodel(
settings,
use_sd=False,
nsubs=settings.n_subgrid,
nprocs=1,
use_rate=True,
)
# -----------------
# trimming
# -----------------
# make file names
file_dict = create_filenames.create_filenames(
settings, use_sd=False, nsubs=settings.n_subgrid)
# read in the observed data
obsdata = Observations(settings.obsfile, settings.filters,
settings.obs_colnames)
for i in range(settings.n_subgrid):
# get the modesedgrid on which to generate the noisemodel
modelsedgridfile = file_dict["modelsedgrid_files"][i]
modelsedgrid = SEDGrid(modelsedgridfile)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(
file_dict["noise_files"][i])
# trim the model sedgrid
sed_trimname = file_dict["modelsedgrid_trim_files"][i]
noisemodel_trimname = file_dict["noise_trim_files"][i]
trim_grid.trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
sed_trimname,
noisemodel_trimname,
sigma_fac=3.0,
)
# -----------------
# fitting
# -----------------
run_fitting.run_fitting(
settings,
use_sd=False,
nsubs=settings.n_subgrid,
nprocs=1,
pdf2d_param_list=["Av", "M_ini", "logT"],
pdf_max_nbins=200,
)
# -----------------
# merging
# -----------------
# it'll automatically skip for no subgrids
merge_files.merge_files(settings,
use_sd=False,
nsubs=settings.n_subgrid)
print("\n\n")
# ==========================================
# reference files for assorted tools
# ==========================================
if run_tools:
# -----------------
# compare_spec_type
# -----------------
# the input settings
input = {
"spec_ra": [72.67213351],
"spec_dec": [-67.71720515],
"spec_type": ["A"],
"spec_subtype": [0],
"lumin_class": ["IV"],
"match_radius": 0.2,
}
# run it
output = compare_spec_type.compare_spec_type(
settings_orig.obsfile,
"{0}/{0}_stats.fits".format(settings_orig.project),
**input,
)
# save the inputs and outputs
asdf.AsdfFile({
"input": input,
"output": output
}).write_to("{0}/{0}_compare_spec_type.asdf".format(
settings_orig.project))
# -----------------
# star_type_probability
# -----------------
# input settings
input = {
"output_filebase": None,
"ext_O_star_params": {
"min_M_ini": 10,
"min_Av": 0.5,
"max_Av": 5
},
}
# run it
output = star_type_probability.star_type_probability(
"{0}/{0}_pdf1d.fits".format(settings_orig.project),
"{0}/{0}_pdf2d.fits".format(settings_orig.project),
**input,
)
# save the inputs and outputs
asdf.AsdfFile({
"input": input,
"output": output
}).write_to("{0}/{0}_star_type_probability.asdf".format(
settings_orig.project))
# ==========================================
# asdf file permissions
# ==========================================
# for unknown reasons, asdf currently writes files with permissions set
# to -rw-------. This changes it to -rw-r--r-- (like the rest of the
# BEAST files) so Karl can easily copy them over to the cached file
# website.
# list of asdf files
asdf_files = glob.glob("*/*.asdf")
# go through each one to change permissions
for fname in asdf_files:
os.chmod(fname,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
