def test_trim_grid(self):
"""
Generate trim the sed grid and noise model using cached versions of the
both and compare the result to a cached version.
"""
# read in the observed data
obsdata = Observations(self.obs_fname_cache, self.settings.filters,
self.settings.obs_colnames)
# get the modesedgrid
modelsedgrid = SEDGrid(self.seds_fname_cache)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(self.noise_fname_cache)
# trim the model sedgrid
seds_trim_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name
noise_trim_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name
trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
seds_trim_fname,
noise_trim_fname,
sigma_fac=3.0,
)
# compare the new to the cached version
compare_hdf5(self.seds_trim_fname_cache, seds_trim_fname, ctype="seds")
compare_hdf5(self.noise_trim_fname_cache,
noise_trim_fname,
ctype="noise")
def trim_submodel(modelsedgridfile):
modelsedgrid = FileSEDGrid(modelsedgridfile)
noisefile = modelsedgridfile.replace("seds", "noisemodel")
sed_trimname = modelsedgridfile.replace("seds", "seds_trim")
noisemodel_trimname = sed_trimname.replace("seds", "noisemodel")
# When working with density bins, we nees to work in a subfolder
if args.dens_bin is not None:
noisefile = os.path.join(bin_subfolder, noisefile)
sed_trimname = os.path.join(bin_subfolder, sed_trimname)
noisemodel_trimname = os.path.join(bin_subfolder,
noisemodel_trimname)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(noisefile)
# trim the model sedgrid
trim_grid.trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
sed_trimname,
noisemodel_trimname,
sigma_fac=3.0,
)
def test_trim_grid():
# download the needed files
vega_fname = download_rename("vega.hd5")
seds_fname = download_rename("beast_example_phat_seds.grid.hd5")
noise_fname = download_rename("beast_example_phat_noisemodel.grid.hd5")
obs_fname = download_rename("b15_4band_det_27_A.fits")
# download cached version of noisemodel on the sed grid
noise_trim_fname_cache = download_rename(
"beast_example_phat_noisemodel_trim.grid.hd5")
seds_trim_fname_cache = download_rename(
"beast_example_phat_seds_trim.grid.hd5")
################
# read in the observed data
filters = [
"HST_WFC3_F275W",
"HST_WFC3_F336W",
"HST_ACS_WFC_F475W",
"HST_ACS_WFC_F814W",
"HST_WFC3_F110W",
"HST_WFC3_F160W",
]
basefilters = ["F275W", "F336W", "F475W", "F814W", "F110W", "F160W"]
obs_colnames = [f.lower() + "_rate" for f in basefilters]
obsdata = Observations(obs_fname,
filters,
obs_colnames,
vega_fname=vega_fname)
# get the modesedgrid
modelsedgrid = SEDGrid(seds_fname)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(noise_fname)
# trim the model sedgrid
seds_trim_fname = "beast_example_phat_seds_trim.grid.hd5"
noise_trim_fname = seds_trim_fname.replace("_seds", "_noisemodel")
trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
seds_trim_fname,
noise_trim_fname,
sigma_fac=3.0,
)
# compare the new to the cached version
compare_hdf5(seds_trim_fname_cache, seds_trim_fname, ctype="seds")
compare_hdf5(noise_trim_fname_cache, noise_trim_fname, ctype="noise")
def trim_submodel(modelsedgridfile):
modelsedgrid = FileSEDGrid(modelsedgridfile)
# read in the noise model just created
noisefile = modelsedgridfile.replace('seds', 'noisemodel')
noisemodel_vals = noisemodel.get_noisemodelcat(noisefile)
# trim the model sedgrid
sed_trimname = modelsedgridfile.replace('seds', 'seds_trim')
noisemodel_trimname = sed_trimname.replace('seds', 'noisemodel')
trim_grid.trim_models(modelsedgrid, noisemodel_vals, obsdata,
sed_trimname, noisemodel_trimname, sigma_fac=3.)
def run_beast_production(basename,
physicsmodel=False,
ast=False,
observationmodel=False,
trim=False,
fitting=False,
resume=False,
source_density='',
sub_source_density=''):
"""
Turns the original command-line version of run_beast_production.py into
something callable from within a function
Parameters
----------
basename : string
name of the gst file (assuming it's located in ./data/)
For the info related to the other inputs, see the argparse info at the bottom
"""
# before doing ANYTHING, force datamodel to re-import (otherwise, any
# changes within this python session will not be loaded!)
importlib.reload(datamodel)
# check input parameters, print what is the problem, stop run_beast
verify_params.verify_input_format(datamodel)
# update the filenames as needed for production
# - photometry sub-file
datamodel.obsfile = basename.replace(
'.fits',
'_with_sourceden' + '_SD_' + source_density.replace('_', '-') +
'_sub' + sub_source_density + '.fits')
# - stats files
stats_filebase = "%s/%s"%(datamodel.project,datamodel.project) \
+ '_sd' + source_density.replace('_','-') \
+ '_sub' + sub_source_density
sed_trimname = stats_filebase + '_sed_trim.grid.hd5'
# - trimmed noise model
noisemodel_trimname = stats_filebase + '_noisemodel_trim.hd5'
# - SED grid
#modelsedgrid_filename = "%s/%s_seds.grid.hd5"%(datamodel.project,
# datamodel.project)
modelsedgrid_filename = "METAL_seds.grid.hd5"
print("***run information***")
print(" project = " + datamodel.project)
print(" obsfile = " + datamodel.obsfile)
print(" astfile = " + datamodel.astfile)
print(" noisefile = " + datamodel.noisefile)
print(" trimmed sedfile = " + sed_trimname)
print("trimmed noisefiles = " + noisemodel_trimname)
print(" stats filebase = " + stats_filebase)
# make sure the project directory exists
pdir = create_project_dir(datamodel.project)
if physicsmodel:
# download and load the isochrones
(iso_fname, oiso) = make_iso_table(datamodel.project,
oiso=datamodel.oiso,
logtmin=datamodel.logt[0],
logtmax=datamodel.logt[1],
dlogt=datamodel.logt[2],
z=datamodel.z)
if hasattr(datamodel, 'add_spectral_properties_kwargs'):
extra_kwargs = datamodel.add_spectral_properties_kwargs
else:
extra_kwargs = None
if hasattr(datamodel, 'velocity'):
redshift = (datamodel.velocity / const.c).decompose().value
else:
redshift = 0
# generate the spectral library (no dust extinction)
(spec_fname, g_spec) = make_spectral_grid(
datamodel.project,
oiso,
osl=datamodel.osl,
redshift=redshift,
distance=datamodel.distances,
distance_unit=datamodel.distance_unit,
add_spectral_properties_kwargs=extra_kwargs)
# add the stellar priors as weights
# also computes the grid weights for the stellar part
(pspec_fname, g_pspec) = add_stellar_priors(datamodel.project, g_spec)
# generate the SED grid by integrating the filter response functions
# effect of dust extinction applied before filter integration
# also computes the dust priors as weights
(seds_fname, g_seds) = make_extinguished_sed_grid(
datamodel.project,
g_pspec,
datamodel.filters,
extLaw=datamodel.extLaw,
av=datamodel.avs,
rv=datamodel.rvs,
fA=datamodel.fAs,
rv_prior_model=datamodel.rv_prior_model,
av_prior_model=datamodel.av_prior_model,
fA_prior_model=datamodel.fA_prior_model,
spec_fname=modelsedgrid_filename,
add_spectral_properties_kwargs=extra_kwargs)
if ast:
N_models = datamodel.ast_models_selected_per_age
Nfilters = datamodel.ast_bands_above_maglimit
Nrealize = datamodel.ast_realization_per_model
mag_cuts = datamodel.ast_maglimit
obsdata = datamodel.get_obscat(basename, datamodel.filters)
if len(mag_cuts) == 1:
tmp_cuts = mag_cuts
min_mags = np.zeros(len(datamodel.filters))
for k, filtername in enumerate(obsdata.filters):
sfiltername = obsdata.data.resolve_alias(filtername)
sfiltername = sfiltername.replace('rate', 'vega')
sfiltername = sfiltername.replace('RATE', 'VEGA')
keep, = np.where(obsdata[sfiltername] < 99.)
min_mags[k] = np.percentile(obsdata[keep][sfiltername], 90.)
# max. mags from the gst observation cat.
mag_cuts = min_mags + tmp_cuts
outfile = './' + datamodel.project + '/' + datamodel.project + '_inputAST.txt'
outfile_params = './' + datamodel.project + '/' + datamodel.project + '_ASTparams.fits'
chosen_seds = pick_models(modelsedgrid_filename,
datamodel.filters,
mag_cuts,
Nfilter=Nfilters,
N_stars=N_models,
Nrealize=Nrealize,
outfile=outfile,
outfile_params=outfile_params)
if datamodel.ast_with_positions == True:
separation = datamodel.ast_pixel_distribution
filename = datamodel.project + '/' + datamodel.project + '_inputAST.txt'
if datamodel.ast_reference_image is not None:
# With reference image, use the background or source density map if available
if datamodel.ast_density_table is not None:
pick_positions_from_map(
obsdata,
chosen_seds,
datamodel.ast_density_table,
datamodel.ast_N_bins,
datamodel.ast_realization_per_model,
outfile=filename,
refimage=datamodel.ast_reference_image,
refimage_hdu=0,
Nrealize=1,
set_coord_boundary=datamodel.ast_coord_boundary)
else:
pick_positions(obsdata,
filename,
separation,
refimage=datamodel.ast_reference_image)
else:
# Without reference image, we can only use this function
if datamodel.ast_density_table is None:
pick_positions(obsdata, filename, separation)
else:
print(
"To use ast_density_table, ast_reference_image must be specified."
)
if observationmodel:
print('Generating noise model from ASTs and absflux A matrix')
# get the modesedgrid on which to generate the noisemodel
modelsedgrid = FileSEDGrid(modelsedgrid_filename)
# generate the AST noise model
noisemodel.make_toothpick_noise_model( \
datamodel.noisefile,
datamodel.astfile,
modelsedgrid,
use_rate=True,
absflux_a_matrix=datamodel.absflux_a_matrix)
if trim:
print('Trimming the model and noise grids')
# read in the observed data
obsdata = datamodel.get_obscat(basename, datamodel.filters)
# get the modesedgrid on which to generate the noisemodel
modelsedgrid = FileSEDGrid(modelsedgrid_filename)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(datamodel.noisefile)
# trim the model sedgrid
trim_grid.trim_models(modelsedgrid,
noisemodel_vals,
obsdata,
sed_trimname,
noisemodel_trimname,
sigma_fac=3.)
if fitting:
start_time = time.clock()
# read in the the AST noise model
noisemodel_vals = noisemodel.get_noisemodelcat(noisemodel_trimname)
# read in the observed data
obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters)
# output files
statsfile = stats_filebase + '_stats.fits'
pdf1dfile = statsfile.replace('stats.fits', 'pdf1d.fits')
lnpfile = statsfile.replace('stats.fits', 'lnp.hd5')
fit.summary_table_memory(obsdata,
noisemodel_vals,
sed_trimname,
resume=resume,
threshold=-10.,
save_every_npts=100,
lnp_npts=500,
stats_outname=statsfile,
pdf1d_outname=pdf1dfile,
lnp_outname=lnpfile,
surveyname=datamodel.surveyname)
new_time = time.clock()
print('time to fit: ', (new_time - start_time) / 60., ' min')
if noisefile == old_noisefile:
print("not reading noisefile - same as last")
# print(noisefile)
else:
print("reading noisefile")
# read in the noise model
noisemodel_vals = noisemodel.get_noisemodelcat(noisefile)
old_noisefile = noisefile
# read in the observed data
print("getting the observed data")
obsdata = Observations(
obsfile, modelsedgrid.filters, obs_colnames=datamodel.obs_colnames
)
# trim the model sedgrid
# set n_detected = 0 to disable the trimming of models based on
# the ASTs (e.g. extrapolations are ok)
# this is needed as the ASTs in the NIR bands do not go faint enough
trim_grid.trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
sed_trimname,
noisemodel_trimname,
sigma_fac=3.0,
)
new_time = time.clock()
print("time to trim: ", (new_time - start_time) / 60.0, " min")
def generate_files_for_tests(run_beast=True, run_tools=True):
"""
Use the metal_small example to generate a full set of files for the BEAST
regression tests.
Parameters
----------
run_beast : boolean (default=True)
if True, run the BEAST
run_tools : boolean (default=True)
if True, run the code to generate things for tools
"""
# read in BEAST settings
settings_orig = beast_settings.beast_settings("beast_settings.txt")
# also make a version with subgrids
settings_subgrids = copy.deepcopy(settings_orig)
settings_subgrids.n_subgrid = 2
settings_subgrids.project = f"{settings_orig.project}_subgrids"
# ==========================================
# run the beast for each set of settings
# ==========================================
if run_beast:
for settings in [settings_orig, settings_subgrids]:
# -----------------
# physics model
# -----------------
create_physicsmodel.create_physicsmodel(
settings,
nsubs=settings.n_subgrid,
nprocs=1,
)
# -----------------
# ASTs
# -----------------
# currently only works for no subgrids
if settings.n_subgrid == 1:
make_ast_inputs.make_ast_inputs(settings,
pick_method="flux_bin_method")
# -----------------
# obs model
# -----------------
create_obsmodel.create_obsmodel(
settings,
use_sd=False,
nsubs=settings.n_subgrid,
nprocs=1,
use_rate=True,
)
# -----------------
# trimming
# -----------------
# make file names
file_dict = create_filenames.create_filenames(
settings, use_sd=False, nsubs=settings.n_subgrid)
# read in the observed data
obsdata = Observations(settings.obsfile, settings.filters,
settings.obs_colnames)
for i in range(settings.n_subgrid):
# get the modesedgrid on which to generate the noisemodel
modelsedgridfile = file_dict["modelsedgrid_files"][i]
modelsedgrid = SEDGrid(modelsedgridfile)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(
file_dict["noise_files"][i])
# trim the model sedgrid
sed_trimname = file_dict["modelsedgrid_trim_files"][i]
noisemodel_trimname = file_dict["noise_trim_files"][i]
trim_grid.trim_models(
modelsedgrid,
noisemodel_vals,
obsdata,
sed_trimname,
noisemodel_trimname,
sigma_fac=3.0,
)
# -----------------
# fitting
# -----------------
run_fitting.run_fitting(
settings,
use_sd=False,
nsubs=settings.n_subgrid,
nprocs=1,
pdf2d_param_list=["Av", "M_ini", "logT"],
pdf_max_nbins=200,
)
# -----------------
# merging
# -----------------
# it'll automatically skip for no subgrids
merge_files.merge_files(settings,
use_sd=False,
nsubs=settings.n_subgrid)
print("\n\n")
# ==========================================
# reference files for assorted tools
# ==========================================
if run_tools:
# -----------------
# compare_spec_type
# -----------------
# the input settings
input = {
"spec_ra": [72.67213351],
"spec_dec": [-67.71720515],
"spec_type": ["A"],
"spec_subtype": [0],
"lumin_class": ["IV"],
"match_radius": 0.2,
}
# run it
output = compare_spec_type.compare_spec_type(
settings_orig.obsfile,
"{0}/{0}_stats.fits".format(settings_orig.project),
**input,
)
# save the inputs and outputs
asdf.AsdfFile({
"input": input,
"output": output
}).write_to("{0}/{0}_compare_spec_type.asdf".format(
settings_orig.project))
# -----------------
# star_type_probability
# -----------------
# input settings
input = {
"output_filebase": None,
"ext_O_star_params": {
"min_M_ini": 10,
"min_Av": 0.5,
"max_Av": 5
},
}
# run it
output = star_type_probability.star_type_probability(
"{0}/{0}_pdf1d.fits".format(settings_orig.project),
"{0}/{0}_pdf2d.fits".format(settings_orig.project),
**input,
)
# save the inputs and outputs
asdf.AsdfFile({
"input": input,
"output": output
}).write_to("{0}/{0}_star_type_probability.asdf".format(
settings_orig.project))
# ==========================================
# asdf file permissions
# ==========================================
# for unknown reasons, asdf currently writes files with permissions set
# to -rw-------. This changes it to -rw-r--r-- (like the rest of the
# BEAST files) so Karl can easily copy them over to the cached file
# website.
# list of asdf files
asdf_files = glob.glob("*/*.asdf")
# go through each one to change permissions
for fname in asdf_files:
os.chmod(fname,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
