def gen_obsmodel(modelsedgridfile, source_density=None, use_rate=True):
"""
Code to create filenames and run the toothpick noise model
Parameters
----------
modelsedgridfile : string
path+name of the physics model grid file
source_density : string (default=None)
set to None if there's no source density info, otherwise set to
a string of the form "#-#"
use_rate : boolean (default=True)
Choose whether to use the rate or magnitude when creating the noise
model. This should always be True, but is currently an option to be
compatible with the phat_small example (which has no rate info).
When that gets fixed, please remove this option!
Returns
-------
noisefile : string
name of the created noise file
"""
print("")
# noise and AST file names
noisefile = modelsedgridfile.replace("seds", "noisemodel")
astfile = datamodel.astfile
# If we are treating regions with different
# backgrounds/source densities separately, pick one of the
# split ast files, and name noise file accordingly
if source_density is not None:
noisefile = noisefile.replace("noisemodel",
"noisemodel_bin" + source_density)
astfile = datamodel.astfile.replace(
".fits", "_bin" + source_density.replace("_", "-") + ".fits")
# only create noise file if it doesn't already exist
if not os.path.isfile(noisefile):
print("creating " + noisefile)
modelsedgrid = FileSEDGrid(modelsedgridfile)
noisemodel.make_toothpick_noise_model(
noisefile,
astfile,
modelsedgrid,
absflux_a_matrix=datamodel.absflux_a_matrix,
use_rate=use_rate,
)
else:
print(noisefile + " already exists")
return noisefile # (same as noisefile)
def gen_subobsmodel(modelsedgridfile):
modelsedgrid = FileSEDGrid(modelsedgridfile)
# generate the AST noise model
noisefile = modelsedgridfile.replace('seds', 'noisemodel')
noisemodel.make_toothpick_noise_model(
noisefile,
datamodel.astfile,
modelsedgrid,
absflux_a_matrix=datamodel.absflux_a_matrix)
def gen_obsmodel(settings, modelsedgridfile, source_density=None):
"""
Code to create filenames and run the toothpick noise model
Parameters
----------
settings : beast.tools.beast_settings.beast_settings instance
object with the beast settings
modelsedgridfile : string
path+name of the physics model grid file
source_density : string (default=None)
set to None if there's no source density info, otherwise set to
a string of the form "#-#"
Returns
-------
noisefile : string
name of the created noise file
"""
print("")
# noise and AST file names
noisefile = modelsedgridfile.replace("seds", "noisemodel")
astfile = settings.astfile
# If we are treating regions with different
# backgrounds/source densities separately, pick one of the
# split ast files, and name noise file accordingly
if source_density is not None:
noisefile = noisefile.replace("noisemodel",
"noisemodel_bin" + source_density)
astfile = settings.astfile.replace(
".fits", "_bin" + source_density.replace("_", "-") + ".fits")
# only create noise file if it doesn't already exist
if not os.path.isfile(noisefile):
print("creating " + noisefile)
modelsedgrid = SEDGrid(modelsedgridfile)
noisemodel.make_toothpick_noise_model(
noisefile,
astfile,
modelsedgrid,
absflux_a_matrix=settings.absflux_a_matrix,
)
else:
print(noisefile + " already exists")
return noisefile # (same as noisefile)
def test_toothpick_noisemodel():
# download the needed files
asts_fname = download_rename("fake_stars_b15_27_all.hd5")
filter_fname = download_rename("filters.hd5")
vega_fname = download_rename("vega.hd5")
hst_fname = download_rename("hst_whitedwarf_frac_covar.fits")
seds_fname = download_rename("beast_example_phat_seds.grid.hd5")
# download cached version of noisemodel on the sed grid
noise_fname_cache = download_rename(
"beast_example_phat_noisemodel.grid.hd5")
################
# get the modesedgrid on which to generate the noisemodel
modelsedgrid = SEDGrid(seds_fname)
# absflux calibration covariance matrix for HST specific filters (AC)
filters = [
"HST_WFC3_F275W",
"HST_WFC3_F336W",
"HST_ACS_WFC_F475W",
"HST_ACS_WFC_F814W",
"HST_WFC3_F110W",
"HST_WFC3_F160W",
]
absflux_a_matrix = hst_frac_matrix(filters,
hst_fname=hst_fname,
filterLib=filter_fname)
# generate the AST noise model
noise_fname = "/tmp/beast_example_phat_noisemodel.grid.hd5"
noisemodel.make_toothpick_noise_model(
noise_fname,
asts_fname,
modelsedgrid,
absflux_a_matrix=absflux_a_matrix,
vega_fname=vega_fname,
use_rate=False,
)
# compare the new to the cached version
compare_hdf5(noise_fname_cache, noise_fname)
def test_toothpick_noisemodel(self):
"""
Generate the nosiemodel (aka observationmodel) using a cached version of
the artifical star test results (ASTs) and compare the result to a cached
version.
"""
# get the modelsedgrid on which to generate the noisemodel
modelsedgrid = SEDGrid(self.seds_fname_cache)
# generate the AST noise model
noise_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name
noisemodel.make_toothpick_noise_model(
noise_fname,
self.asts_fname_cache,
modelsedgrid,
absflux_a_matrix=self.settings.absflux_a_matrix,
)
# compare the new to the cached version
compare_hdf5(self.noise_fname_cache, noise_fname)
def gen_subobsmodel(modelsedgridfile):
modelsedgrid = FileSEDGrid(modelsedgridfile)
# generate the AST noise model
noisefile = modelsedgridfile.replace("seds", "noisemodel")
astfile = settings.astfile
# If we are treating regions with different
# backgrounds/source densities separately, pick one of the
# split ast files, and put the results in a subfolder.
if args.dens_bin is not None:
noisefile = os.path.join(bin_subfolder, noisefile)
create_project_dir(os.path.dirname(noisefile))
astfile = subcatalog_fname(astfile, args.dens_bin)
outname = noisemodel.make_toothpick_noise_model(
noisefile,
astfile,
modelsedgrid,
absflux_a_matrix=settings.absflux_a_matrix,
)
return outname
def run_beast_production(basename,
physicsmodel=False,
ast=False,
observationmodel=False,
trim=False,
fitting=False,
resume=False,
source_density='',
sub_source_density=''):
"""
Turns the original command-line version of run_beast_production.py into
something callable from within a function
Parameters
----------
basename : string
name of the gst file (assuming it's located in ./data/)
For the info related to the other inputs, see the argparse info at the bottom
"""
# before doing ANYTHING, force datamodel to re-import (otherwise, any
# changes within this python session will not be loaded!)
importlib.reload(datamodel)
# check input parameters, print what is the problem, stop run_beast
verify_params.verify_input_format(datamodel)
# update the filenames as needed for production
# - photometry sub-file
datamodel.obsfile = basename.replace(
'.fits',
'_with_sourceden' + '_SD_' + source_density.replace('_', '-') +
'_sub' + sub_source_density + '.fits')
# - stats files
stats_filebase = "%s/%s"%(datamodel.project,datamodel.project) \
+ '_sd' + source_density.replace('_','-') \
+ '_sub' + sub_source_density
sed_trimname = stats_filebase + '_sed_trim.grid.hd5'
# - trimmed noise model
noisemodel_trimname = stats_filebase + '_noisemodel_trim.hd5'
# - SED grid
#modelsedgrid_filename = "%s/%s_seds.grid.hd5"%(datamodel.project,
# datamodel.project)
modelsedgrid_filename = "METAL_seds.grid.hd5"
print("***run information***")
print(" project = " + datamodel.project)
print(" obsfile = " + datamodel.obsfile)
print(" astfile = " + datamodel.astfile)
print(" noisefile = " + datamodel.noisefile)
print(" trimmed sedfile = " + sed_trimname)
print("trimmed noisefiles = " + noisemodel_trimname)
print(" stats filebase = " + stats_filebase)
# make sure the project directory exists
pdir = create_project_dir(datamodel.project)
if physicsmodel:
# download and load the isochrones
(iso_fname, oiso) = make_iso_table(datamodel.project,
oiso=datamodel.oiso,
logtmin=datamodel.logt[0],
logtmax=datamodel.logt[1],
dlogt=datamodel.logt[2],
z=datamodel.z)
if hasattr(datamodel, 'add_spectral_properties_kwargs'):
extra_kwargs = datamodel.add_spectral_properties_kwargs
else:
extra_kwargs = None
if hasattr(datamodel, 'velocity'):
redshift = (datamodel.velocity / const.c).decompose().value
else:
redshift = 0
# generate the spectral library (no dust extinction)
(spec_fname, g_spec) = make_spectral_grid(
datamodel.project,
oiso,
osl=datamodel.osl,
redshift=redshift,
distance=datamodel.distances,
distance_unit=datamodel.distance_unit,
add_spectral_properties_kwargs=extra_kwargs)
# add the stellar priors as weights
# also computes the grid weights for the stellar part
(pspec_fname, g_pspec) = add_stellar_priors(datamodel.project, g_spec)
# generate the SED grid by integrating the filter response functions
# effect of dust extinction applied before filter integration
# also computes the dust priors as weights
(seds_fname, g_seds) = make_extinguished_sed_grid(
datamodel.project,
g_pspec,
datamodel.filters,
extLaw=datamodel.extLaw,
av=datamodel.avs,
rv=datamodel.rvs,
fA=datamodel.fAs,
rv_prior_model=datamodel.rv_prior_model,
av_prior_model=datamodel.av_prior_model,
fA_prior_model=datamodel.fA_prior_model,
spec_fname=modelsedgrid_filename,
add_spectral_properties_kwargs=extra_kwargs)
if ast:
N_models = datamodel.ast_models_selected_per_age
Nfilters = datamodel.ast_bands_above_maglimit
Nrealize = datamodel.ast_realization_per_model
mag_cuts = datamodel.ast_maglimit
obsdata = datamodel.get_obscat(basename, datamodel.filters)
if len(mag_cuts) == 1:
tmp_cuts = mag_cuts
min_mags = np.zeros(len(datamodel.filters))
for k, filtername in enumerate(obsdata.filters):
sfiltername = obsdata.data.resolve_alias(filtername)
sfiltername = sfiltername.replace('rate', 'vega')
sfiltername = sfiltername.replace('RATE', 'VEGA')
keep, = np.where(obsdata[sfiltername] < 99.)
min_mags[k] = np.percentile(obsdata[keep][sfiltername], 90.)
# max. mags from the gst observation cat.
mag_cuts = min_mags + tmp_cuts
outfile = './' + datamodel.project + '/' + datamodel.project + '_inputAST.txt'
outfile_params = './' + datamodel.project + '/' + datamodel.project + '_ASTparams.fits'
chosen_seds = pick_models(modelsedgrid_filename,
datamodel.filters,
mag_cuts,
Nfilter=Nfilters,
N_stars=N_models,
Nrealize=Nrealize,
outfile=outfile,
outfile_params=outfile_params)
if datamodel.ast_with_positions == True:
separation = datamodel.ast_pixel_distribution
filename = datamodel.project + '/' + datamodel.project + '_inputAST.txt'
if datamodel.ast_reference_image is not None:
# With reference image, use the background or source density map if available
if datamodel.ast_density_table is not None:
pick_positions_from_map(
obsdata,
chosen_seds,
datamodel.ast_density_table,
datamodel.ast_N_bins,
datamodel.ast_realization_per_model,
outfile=filename,
refimage=datamodel.ast_reference_image,
refimage_hdu=0,
Nrealize=1,
set_coord_boundary=datamodel.ast_coord_boundary)
else:
pick_positions(obsdata,
filename,
separation,
refimage=datamodel.ast_reference_image)
else:
# Without reference image, we can only use this function
if datamodel.ast_density_table is None:
pick_positions(obsdata, filename, separation)
else:
print(
"To use ast_density_table, ast_reference_image must be specified."
)
if observationmodel:
print('Generating noise model from ASTs and absflux A matrix')
# get the modesedgrid on which to generate the noisemodel
modelsedgrid = FileSEDGrid(modelsedgrid_filename)
# generate the AST noise model
noisemodel.make_toothpick_noise_model( \
datamodel.noisefile,
datamodel.astfile,
modelsedgrid,
use_rate=True,
absflux_a_matrix=datamodel.absflux_a_matrix)
if trim:
print('Trimming the model and noise grids')
# read in the observed data
obsdata = datamodel.get_obscat(basename, datamodel.filters)
# get the modesedgrid on which to generate the noisemodel
modelsedgrid = FileSEDGrid(modelsedgrid_filename)
# read in the noise model just created
noisemodel_vals = noisemodel.get_noisemodelcat(datamodel.noisefile)
# trim the model sedgrid
trim_grid.trim_models(modelsedgrid,
noisemodel_vals,
obsdata,
sed_trimname,
noisemodel_trimname,
sigma_fac=3.)
if fitting:
start_time = time.clock()
# read in the the AST noise model
noisemodel_vals = noisemodel.get_noisemodelcat(noisemodel_trimname)
# read in the observed data
obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters)
# output files
statsfile = stats_filebase + '_stats.fits'
pdf1dfile = statsfile.replace('stats.fits', 'pdf1d.fits')
lnpfile = statsfile.replace('stats.fits', 'lnp.hd5')
fit.summary_table_memory(obsdata,
noisemodel_vals,
sed_trimname,
resume=resume,
threshold=-10.,
save_every_npts=100,
lnp_npts=500,
stats_outname=statsfile,
pdf1d_outname=pdf1dfile,
lnp_outname=lnpfile,
surveyname=datamodel.surveyname)
new_time = time.clock()
print('time to fit: ', (new_time - start_time) / 60., ' min')
refimage=datamodel.ast_reference_image)
else:
pick_positions(filename, separation)
if args.observationmodel:
print('Generating noise model from ASTs and absflux A matrix')
# get the modesedgrid on which to generate the noisemodel
modelsedgridfile = datamodel.project + '/' + datamodel.project + \
'_seds.grid.hd5'
modelsedgrid = FileSEDGrid(modelsedgridfile)
# generate the AST noise model
noisemodel.make_toothpick_noise_model( \
datamodel.noisefile,
datamodel.astfile,
modelsedgrid,
absflux_a_matrix=datamodel.absflux_a_matrix)
if args.trim:
print('Trimming the model and noise grids')
# read in the observed data
obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters)
# get the modesedgrid on which to generate the noisemodel
modelsedgridfile = datamodel.project + '/' + datamodel.project + \
'_seds.grid.hd5'
modelsedgrid = FileSEDGrid(modelsedgridfile)
# read in the noise model just created
pick_positions(obsdata, filename, separation)
else:
print("To use ast_density_table, ast_reference_image must be specified.")
if args.observationmodel:
print('Generating noise model from ASTs and absflux A matrix')
# get the modesedgrid on which to generate the noisemodel
modelsedgridfile = datamodel.project + '/' + datamodel.project + \
'_seds.grid.hd5'
modelsedgrid = FileSEDGrid(modelsedgridfile)
# generate the AST noise model using the toothpick model
noisemodel.make_toothpick_noise_model(
datamodel.noisefile,
datamodel.astfile,
modelsedgrid,
absflux_a_matrix=datamodel.absflux_a_matrix,
use_rate=False)
# in the absence of ASTs, the splinter noise model can be used
# instead of the toothpick model above
# **warning** not very realistic
# import beast.observationmodel.noisemodel.splinter as noisemodel
#
# noisemodel.make_splinter_noise_model(
# datamodel.noisefile,
# modelsedgrid,
# absflux_a_matrix=datamodel.absflux_a_matrix)
if args.trim:
print('Trimming the model and noise grids')
