def test_trim_grid(self): """ Generate trim the sed grid and noise model using cached versions of the both and compare the result to a cached version. """ # read in the observed data obsdata = Observations(self.obs_fname_cache, self.settings.filters, self.settings.obs_colnames) # get the modesedgrid modelsedgrid = SEDGrid(self.seds_fname_cache) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(self.noise_fname_cache) # trim the model sedgrid seds_trim_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name noise_trim_fname = tempfile.NamedTemporaryFile(suffix=".hd5").name trim_models( modelsedgrid, noisemodel_vals, obsdata, seds_trim_fname, noise_trim_fname, sigma_fac=3.0, ) # compare the new to the cached version compare_hdf5(self.seds_trim_fname_cache, seds_trim_fname, ctype="seds") compare_hdf5(self.noise_trim_fname_cache, noise_trim_fname, ctype="noise")
def trim_submodel(modelsedgridfile): modelsedgrid = FileSEDGrid(modelsedgridfile) noisefile = modelsedgridfile.replace("seds", "noisemodel") sed_trimname = modelsedgridfile.replace("seds", "seds_trim") noisemodel_trimname = sed_trimname.replace("seds", "noisemodel") # When working with density bins, we nees to work in a subfolder if args.dens_bin is not None: noisefile = os.path.join(bin_subfolder, noisefile) sed_trimname = os.path.join(bin_subfolder, sed_trimname) noisemodel_trimname = os.path.join(bin_subfolder, noisemodel_trimname) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(noisefile) # trim the model sedgrid trim_grid.trim_models( modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.0, )
def test_trim_grid(): # download the needed files vega_fname = download_rename("vega.hd5") seds_fname = download_rename("beast_example_phat_seds.grid.hd5") noise_fname = download_rename("beast_example_phat_noisemodel.grid.hd5") obs_fname = download_rename("b15_4band_det_27_A.fits") # download cached version of noisemodel on the sed grid noise_trim_fname_cache = download_rename( "beast_example_phat_noisemodel_trim.grid.hd5") seds_trim_fname_cache = download_rename( "beast_example_phat_seds_trim.grid.hd5") ################ # read in the observed data filters = [ "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_ACS_WFC_F475W", "HST_ACS_WFC_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", ] basefilters = ["F275W", "F336W", "F475W", "F814W", "F110W", "F160W"] obs_colnames = [f.lower() + "_rate" for f in basefilters] obsdata = Observations(obs_fname, filters, obs_colnames, vega_fname=vega_fname) # get the modesedgrid modelsedgrid = SEDGrid(seds_fname) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(noise_fname) # trim the model sedgrid seds_trim_fname = "beast_example_phat_seds_trim.grid.hd5" noise_trim_fname = seds_trim_fname.replace("_seds", "_noisemodel") trim_models( modelsedgrid, noisemodel_vals, obsdata, seds_trim_fname, noise_trim_fname, sigma_fac=3.0, ) # compare the new to the cached version compare_hdf5(seds_trim_fname_cache, seds_trim_fname, ctype="seds") compare_hdf5(noise_trim_fname_cache, noise_trim_fname, ctype="noise")
def trim_submodel(modelsedgridfile): modelsedgrid = FileSEDGrid(modelsedgridfile) # read in the noise model just created noisefile = modelsedgridfile.replace('seds', 'noisemodel') noisemodel_vals = noisemodel.get_noisemodelcat(noisefile) # trim the model sedgrid sed_trimname = modelsedgridfile.replace('seds', 'seds_trim') noisemodel_trimname = sed_trimname.replace('seds', 'noisemodel') trim_grid.trim_models(modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.)
def run_beast_production(basename, physicsmodel=False, ast=False, observationmodel=False, trim=False, fitting=False, resume=False, source_density='', sub_source_density=''): """ Turns the original command-line version of run_beast_production.py into something callable from within a function Parameters ---------- basename : string name of the gst file (assuming it's located in ./data/) For the info related to the other inputs, see the argparse info at the bottom """ # before doing ANYTHING, force datamodel to re-import (otherwise, any # changes within this python session will not be loaded!) importlib.reload(datamodel) # check input parameters, print what is the problem, stop run_beast verify_params.verify_input_format(datamodel) # update the filenames as needed for production # - photometry sub-file datamodel.obsfile = basename.replace( '.fits', '_with_sourceden' + '_SD_' + source_density.replace('_', '-') + '_sub' + sub_source_density + '.fits') # - stats files stats_filebase = "%s/%s"%(datamodel.project,datamodel.project) \ + '_sd' + source_density.replace('_','-') \ + '_sub' + sub_source_density sed_trimname = stats_filebase + '_sed_trim.grid.hd5' # - trimmed noise model noisemodel_trimname = stats_filebase + '_noisemodel_trim.hd5' # - SED grid #modelsedgrid_filename = "%s/%s_seds.grid.hd5"%(datamodel.project, # datamodel.project) modelsedgrid_filename = "METAL_seds.grid.hd5" print("***run information***") print(" project = " + datamodel.project) print(" obsfile = " + datamodel.obsfile) print(" astfile = " + datamodel.astfile) print(" noisefile = " + datamodel.noisefile) print(" trimmed sedfile = " + sed_trimname) print("trimmed noisefiles = " + noisemodel_trimname) print(" stats filebase = " + stats_filebase) # make sure the project directory exists pdir = create_project_dir(datamodel.project) if physicsmodel: # download and load the isochrones (iso_fname, oiso) = make_iso_table(datamodel.project, oiso=datamodel.oiso, logtmin=datamodel.logt[0], logtmax=datamodel.logt[1], dlogt=datamodel.logt[2], z=datamodel.z) if hasattr(datamodel, 'add_spectral_properties_kwargs'): extra_kwargs = datamodel.add_spectral_properties_kwargs else: extra_kwargs = None if hasattr(datamodel, 'velocity'): redshift = (datamodel.velocity / const.c).decompose().value else: redshift = 0 # generate the spectral library (no dust extinction) (spec_fname, g_spec) = make_spectral_grid( datamodel.project, oiso, osl=datamodel.osl, redshift=redshift, distance=datamodel.distances, distance_unit=datamodel.distance_unit, add_spectral_properties_kwargs=extra_kwargs) # add the stellar priors as weights # also computes the grid weights for the stellar part (pspec_fname, g_pspec) = add_stellar_priors(datamodel.project, g_spec) # generate the SED grid by integrating the filter response functions # effect of dust extinction applied before filter integration # also computes the dust priors as weights (seds_fname, g_seds) = make_extinguished_sed_grid( datamodel.project, g_pspec, datamodel.filters, extLaw=datamodel.extLaw, av=datamodel.avs, rv=datamodel.rvs, fA=datamodel.fAs, rv_prior_model=datamodel.rv_prior_model, av_prior_model=datamodel.av_prior_model, fA_prior_model=datamodel.fA_prior_model, spec_fname=modelsedgrid_filename, add_spectral_properties_kwargs=extra_kwargs) if ast: N_models = datamodel.ast_models_selected_per_age Nfilters = datamodel.ast_bands_above_maglimit Nrealize = datamodel.ast_realization_per_model mag_cuts = datamodel.ast_maglimit obsdata = datamodel.get_obscat(basename, datamodel.filters) if len(mag_cuts) == 1: tmp_cuts = mag_cuts min_mags = np.zeros(len(datamodel.filters)) for k, filtername in enumerate(obsdata.filters): sfiltername = obsdata.data.resolve_alias(filtername) sfiltername = sfiltername.replace('rate', 'vega') sfiltername = sfiltername.replace('RATE', 'VEGA') keep, = np.where(obsdata[sfiltername] < 99.) min_mags[k] = np.percentile(obsdata[keep][sfiltername], 90.) # max. mags from the gst observation cat. mag_cuts = min_mags + tmp_cuts outfile = './' + datamodel.project + '/' + datamodel.project + '_inputAST.txt' outfile_params = './' + datamodel.project + '/' + datamodel.project + '_ASTparams.fits' chosen_seds = pick_models(modelsedgrid_filename, datamodel.filters, mag_cuts, Nfilter=Nfilters, N_stars=N_models, Nrealize=Nrealize, outfile=outfile, outfile_params=outfile_params) if datamodel.ast_with_positions == True: separation = datamodel.ast_pixel_distribution filename = datamodel.project + '/' + datamodel.project + '_inputAST.txt' if datamodel.ast_reference_image is not None: # With reference image, use the background or source density map if available if datamodel.ast_density_table is not None: pick_positions_from_map( obsdata, chosen_seds, datamodel.ast_density_table, datamodel.ast_N_bins, datamodel.ast_realization_per_model, outfile=filename, refimage=datamodel.ast_reference_image, refimage_hdu=0, Nrealize=1, set_coord_boundary=datamodel.ast_coord_boundary) else: pick_positions(obsdata, filename, separation, refimage=datamodel.ast_reference_image) else: # Without reference image, we can only use this function if datamodel.ast_density_table is None: pick_positions(obsdata, filename, separation) else: print( "To use ast_density_table, ast_reference_image must be specified." ) if observationmodel: print('Generating noise model from ASTs and absflux A matrix') # get the modesedgrid on which to generate the noisemodel modelsedgrid = FileSEDGrid(modelsedgrid_filename) # generate the AST noise model noisemodel.make_toothpick_noise_model( \ datamodel.noisefile, datamodel.astfile, modelsedgrid, use_rate=True, absflux_a_matrix=datamodel.absflux_a_matrix) if trim: print('Trimming the model and noise grids') # read in the observed data obsdata = datamodel.get_obscat(basename, datamodel.filters) # get the modesedgrid on which to generate the noisemodel modelsedgrid = FileSEDGrid(modelsedgrid_filename) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(datamodel.noisefile) # trim the model sedgrid trim_grid.trim_models(modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.) if fitting: start_time = time.clock() # read in the the AST noise model noisemodel_vals = noisemodel.get_noisemodelcat(noisemodel_trimname) # read in the observed data obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters) # output files statsfile = stats_filebase + '_stats.fits' pdf1dfile = statsfile.replace('stats.fits', 'pdf1d.fits') lnpfile = statsfile.replace('stats.fits', 'lnp.hd5') fit.summary_table_memory(obsdata, noisemodel_vals, sed_trimname, resume=resume, threshold=-10., save_every_npts=100, lnp_npts=500, stats_outname=statsfile, pdf1d_outname=pdf1dfile, lnp_outname=lnpfile, surveyname=datamodel.surveyname) new_time = time.clock() print('time to fit: ', (new_time - start_time) / 60., ' min')
if noisefile == old_noisefile: print("not reading noisefile - same as last") # print(noisefile) else: print("reading noisefile") # read in the noise model noisemodel_vals = noisemodel.get_noisemodelcat(noisefile) old_noisefile = noisefile # read in the observed data print("getting the observed data") obsdata = Observations( obsfile, modelsedgrid.filters, obs_colnames=datamodel.obs_colnames ) # trim the model sedgrid # set n_detected = 0 to disable the trimming of models based on # the ASTs (e.g. extrapolations are ok) # this is needed as the ASTs in the NIR bands do not go faint enough trim_grid.trim_models( modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.0, ) new_time = time.clock() print("time to trim: ", (new_time - start_time) / 60.0, " min")
def generate_files_for_tests(run_beast=True, run_tools=True): """ Use the metal_small example to generate a full set of files for the BEAST regression tests. Parameters ---------- run_beast : boolean (default=True) if True, run the BEAST run_tools : boolean (default=True) if True, run the code to generate things for tools """ # read in BEAST settings settings_orig = beast_settings.beast_settings("beast_settings.txt") # also make a version with subgrids settings_subgrids = copy.deepcopy(settings_orig) settings_subgrids.n_subgrid = 2 settings_subgrids.project = f"{settings_orig.project}_subgrids" # ========================================== # run the beast for each set of settings # ========================================== if run_beast: for settings in [settings_orig, settings_subgrids]: # ----------------- # physics model # ----------------- create_physicsmodel.create_physicsmodel( settings, nsubs=settings.n_subgrid, nprocs=1, ) # ----------------- # ASTs # ----------------- # currently only works for no subgrids if settings.n_subgrid == 1: make_ast_inputs.make_ast_inputs(settings, pick_method="flux_bin_method") # ----------------- # obs model # ----------------- create_obsmodel.create_obsmodel( settings, use_sd=False, nsubs=settings.n_subgrid, nprocs=1, use_rate=True, ) # ----------------- # trimming # ----------------- # make file names file_dict = create_filenames.create_filenames( settings, use_sd=False, nsubs=settings.n_subgrid) # read in the observed data obsdata = Observations(settings.obsfile, settings.filters, settings.obs_colnames) for i in range(settings.n_subgrid): # get the modesedgrid on which to generate the noisemodel modelsedgridfile = file_dict["modelsedgrid_files"][i] modelsedgrid = SEDGrid(modelsedgridfile) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat( file_dict["noise_files"][i]) # trim the model sedgrid sed_trimname = file_dict["modelsedgrid_trim_files"][i] noisemodel_trimname = file_dict["noise_trim_files"][i] trim_grid.trim_models( modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.0, ) # ----------------- # fitting # ----------------- run_fitting.run_fitting( settings, use_sd=False, nsubs=settings.n_subgrid, nprocs=1, pdf2d_param_list=["Av", "M_ini", "logT"], pdf_max_nbins=200, ) # ----------------- # merging # ----------------- # it'll automatically skip for no subgrids merge_files.merge_files(settings, use_sd=False, nsubs=settings.n_subgrid) print("\n\n") # ========================================== # reference files for assorted tools # ========================================== if run_tools: # ----------------- # compare_spec_type # ----------------- # the input settings input = { "spec_ra": [72.67213351], "spec_dec": [-67.71720515], "spec_type": ["A"], "spec_subtype": [0], "lumin_class": ["IV"], "match_radius": 0.2, } # run it output = compare_spec_type.compare_spec_type( settings_orig.obsfile, "{0}/{0}_stats.fits".format(settings_orig.project), **input, ) # save the inputs and outputs asdf.AsdfFile({ "input": input, "output": output }).write_to("{0}/{0}_compare_spec_type.asdf".format( settings_orig.project)) # ----------------- # star_type_probability # ----------------- # input settings input = { "output_filebase": None, "ext_O_star_params": { "min_M_ini": 10, "min_Av": 0.5, "max_Av": 5 }, } # run it output = star_type_probability.star_type_probability( "{0}/{0}_pdf1d.fits".format(settings_orig.project), "{0}/{0}_pdf2d.fits".format(settings_orig.project), **input, ) # save the inputs and outputs asdf.AsdfFile({ "input": input, "output": output }).write_to("{0}/{0}_star_type_probability.asdf".format( settings_orig.project)) # ========================================== # asdf file permissions # ========================================== # for unknown reasons, asdf currently writes files with permissions set # to -rw-------. This changes it to -rw-r--r-- (like the rest of the # BEAST files) so Karl can easily copy them over to the cached file # website. # list of asdf files asdf_files = glob.glob("*/*.asdf") # go through each one to change permissions for fname in asdf_files: os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)