def split_and_check(grid_fname, num_subgrids): complete_g = FileSEDGrid(grid_fname) sub_fnames = subgridding_tools.split_grid(grid_fname, num_subgrids) # count the number of grid cells sub_seds = [] sub_grids = [] for sub_fname in sub_fnames: sub_g = FileSEDGrid(sub_fname) sub_seds.append(sub_g.seds) sub_grids.append(sub_g.grid.data) np.testing.assert_equal(complete_g.lamb, sub_g.lamb) if not complete_g.grid.columns.items() == sub_g.grid.columns.items(): raise AssertionError() sub_seds_reconstructed = np.concatenate(sub_seds) np.testing.assert_equal(sub_seds_reconstructed, complete_g.seds) sub_grids_reconstructed = np.concatenate(sub_grids) np.testing.assert_equal(sub_grids_reconstructed, complete_g.grid.data) # the split method skips anything that already exists, so if we # want to use this function multiple times for the same test # grid, we need to do this. for f in sub_fnames: os.remove(f)
def fit_submodel(modelsedgridfile): # input files trimmed_modelsedgridfile = modelsedgridfile.replace( 'seds', 'seds_trim') trimmed_noisemodelfile = trimmed_modelsedgridfile.replace( 'seds', 'noisemodel') # output files lnpfile = modelsedgridfile.replace('seds', 'lnp') statsfile = modelsedgridfile.replace('seds', 'stats') statsfile = statsfile.replace('.hd5', '.fits') pdf1dfile = statsfile.replace('stats', 'pdf1d') # load the subgrid seds and subgrid noisemodel modelsedgrid = FileSEDGrid(trimmed_modelsedgridfile) noisemodel_vals = noisemodel.get_noisemodelcat( trimmed_noisemodelfile) fit.summary_table_memory(obsdata, noisemodel_vals, modelsedgrid, resume=args.resume, threshold=-10., save_every_npts=100, lnp_npts=60, stats_outname=statsfile, pdf1d_outname=pdf1dfile, grid_info_dict=grid_info_dict, lnp_outname=lnpfile, do_not_normalize=True) print('Done fitting on grid ' + trimmed_modelsedgridfile)
def trim_submodel(modelsedgridfile): modelsedgrid = FileSEDGrid(modelsedgridfile) noisefile = modelsedgridfile.replace("seds", "noisemodel") sed_trimname = modelsedgridfile.replace("seds", "seds_trim") noisemodel_trimname = sed_trimname.replace("seds", "noisemodel") # When working with density bins, we nees to work in a subfolder if args.dens_bin is not None: noisefile = os.path.join(bin_subfolder, noisefile) sed_trimname = os.path.join(bin_subfolder, sed_trimname) noisemodel_trimname = os.path.join(bin_subfolder, noisemodel_trimname) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(noisefile) # trim the model sedgrid trim_grid.trim_models( modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.0, )
def gen_obsmodel(modelsedgridfile, source_density=None, use_rate=True): """ Code to create filenames and run the toothpick noise model Parameters ---------- modelsedgridfile : string path+name of the physics model grid file source_density : string (default=None) set to None if there's no source density info, otherwise set to a string of the form "#-#" use_rate : boolean (default=True) Choose whether to use the rate or magnitude when creating the noise model. This should always be True, but is currently an option to be compatible with the phat_small example (which has no rate info). When that gets fixed, please remove this option! Returns ------- noisefile : string name of the created noise file """ print("") # noise and AST file names noisefile = modelsedgridfile.replace("seds", "noisemodel") astfile = datamodel.astfile # If we are treating regions with different # backgrounds/source densities separately, pick one of the # split ast files, and name noise file accordingly if source_density is not None: noisefile = noisefile.replace("noisemodel", "noisemodel_bin" + source_density) astfile = datamodel.astfile.replace( ".fits", "_bin" + source_density.replace("_", "-") + ".fits") # only create noise file if it doesn't already exist if not os.path.isfile(noisefile): print("creating " + noisefile) modelsedgrid = FileSEDGrid(modelsedgridfile) noisemodel.make_toothpick_noise_model( noisefile, astfile, modelsedgrid, absflux_a_matrix=datamodel.absflux_a_matrix, use_rate=use_rate, ) else: print(noisefile + " already exists") return noisefile # (same as noisefile)
def gen_subobsmodel(modelsedgridfile): modelsedgrid = FileSEDGrid(modelsedgridfile) # generate the AST noise model noisefile = modelsedgridfile.replace('seds', 'noisemodel') noisemodel.make_toothpick_noise_model( noisefile, datamodel.astfile, modelsedgrid, absflux_a_matrix=datamodel.absflux_a_matrix)
def fit_submodel(modelsedgridfile): # input files trimmed_modelsedgridfile = modelsedgridfile.replace( "seds", "seds_trim") trimmed_noisemodelfile = trimmed_modelsedgridfile.replace( "seds", "noisemodel") # output files lnpfile = modelsedgridfile.replace("seds", "lnp") statsfile = modelsedgridfile.replace("seds", "stats") statsfile = statsfile.replace(".hd5", ".fits") pdf1dfile = statsfile.replace("stats", "pdf1d") if args.dens_bin is not None: # Put everything in the right subfolder ( trimmed_modelsedgridfile, trimmed_noisemodelfile, lnpfile, statsfile, pdf1dfile, ) = [ os.path.join(bin_subfolder, f) for f in [ trimmed_modelsedgridfile, trimmed_noisemodelfile, lnpfile, statsfile, pdf1dfile, ] ] # load the subgrid seds and subgrid noisemodel modelsedgrid = FileSEDGrid(trimmed_modelsedgridfile) noisemodel_vals = noisemodel.get_noisemodelcat( trimmed_noisemodelfile) try: fit.summary_table_memory( obsdata, noisemodel_vals, modelsedgrid, resume=args.resume, threshold=-10.0, save_every_npts=100, lnp_npts=60, stats_outname=statsfile, pdf1d_outname=pdf1dfile, grid_info_dict=grid_info_dict, lnp_outname=lnpfile, do_not_normalize=True, ) print("Done fitting on grid " + trimmed_modelsedgridfile) except Exception as e: if not args.ignore_missing_subresults: raise e
def test_trim_grid(): # download the needed files vega_fname = download_rename("vega.hd5") seds_fname = download_rename("beast_example_phat_seds.grid.hd5") noise_fname = download_rename("beast_example_phat_noisemodel.grid.hd5") obs_fname = download_rename("b15_4band_det_27_A.fits") # download cached version of noisemodel on the sed grid noise_trim_fname_cache = download_rename( "beast_example_phat_noisemodel_trim.grid.hd5" ) seds_trim_fname_cache = download_rename("beast_example_phat_seds_trim.grid.hd5") ################ # read in the observed data filters = [ "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_ACS_WFC_F475W", "HST_ACS_WFC_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", ] basefilters = ["F275W", "F336W", "F475W", "F814W", "F110W", "F160W"] obs_colnames = [f.lower() + "_rate" for f in basefilters] obsdata = get_obscat(obs_fname, filters, obs_colnames, vega_fname=vega_fname) # get the modesedgrid modelsedgrid = FileSEDGrid(seds_fname) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(noise_fname) # trim the model sedgrid seds_trim_fname = "beast_example_phat_seds_trim.grid.hd5" noise_trim_fname = seds_trim_fname.replace("_seds", "_noisemodel") trim_models( modelsedgrid, noisemodel_vals, obsdata, seds_trim_fname, noise_trim_fname, sigma_fac=3.0, ) # compare the new to the cached version compare_hdf5(seds_trim_fname_cache, seds_trim_fname, ctype="seds") compare_hdf5(noise_trim_fname_cache, noise_trim_fname, ctype="noise")
def trim_submodel(modelsedgridfile): modelsedgrid = FileSEDGrid(modelsedgridfile) # read in the noise model just created noisefile = modelsedgridfile.replace('seds', 'noisemodel') noisemodel_vals = noisemodel.get_noisemodelcat(noisefile) # trim the model sedgrid sed_trimname = modelsedgridfile.replace('seds', 'seds_trim') noisemodel_trimname = sed_trimname.replace('seds', 'noisemodel') trim_grid.trim_models(modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.)
def remove_filters_from_files(catfile, physgrid, obsgrid, outbase, rm_filters): # remove the requested filters from the catalog file cat = Table.read(catfile) for cfilter in rm_filters: colname = "{}_rate".format(cfilter) if colname in cat.colnames: cat.remove_column(colname) else: print("{} not in catalog file".format(colname)) cat.write("{}_cat.fits".format(outbase), overwrite=True) # get the sed grid and process g0 = FileSEDGrid(physgrid, backend="cache") filters = g0.header["filters"].split(" ") shortfilters = [(cfilter.split("_"))[-1].lower() for cfilter in filters] nlamb = [] nfilters = [] rindxs = [] for csfilter, clamb, cfilter in zip(shortfilters, g0.lamb, filters): if csfilter not in rm_filters: nlamb.append(clamb) nfilters.append(cfilter) else: rindxs.append(shortfilters.index(csfilter)) nseds = np.delete(g0.seds, rindxs, 1) print("orig filters: {}".format(" ".join(filters))) print(" new filters: {}".format(" ".join(nfilters))) g = SpectralGrid(np.array(nlamb), seds=nseds, grid=g0.grid, backend="memory") g.grid.header["filters"] = " ".join(nfilters) g.writeHDF("{}_sed.grid.hd5".format(outbase)) # get and process the observation model obsgrid = noisemodel.get_noisemodelcat(obsgrid) with tables.open_file("{}_noisemodel.grid.hd5".format(outbase), "w") as outfile: outfile.create_array(outfile.root, "bias", np.delete(obsgrid.root.bias, rindxs, 1)) outfile.create_array(outfile.root, "error", np.delete(obsgrid.root.error, rindxs, 1)) outfile.create_array( outfile.root, "completeness", np.delete(obsgrid.root.completeness, rindxs, 1), )
def remove_filters_from_files(catfile, physgrid, obsgrid, outbase, rm_filters): # remove the requested filters from the catalog file cat = Table.read(catfile) for cfilter in rm_filters: colname = '{}_rate'.format(cfilter) if colname in cat.colnames: cat.remove_column(colname) else: print('{} not in catalog file'.format(colname)) cat.write('{}_cat.fits'.format(outbase), overwrite=True) # get the sed grid and process g0 = FileSEDGrid(physgrid, backend='cache') filters = g0.header['filters'].split(' ') shortfilters = [(cfilter.split('_'))[-1].lower() for cfilter in filters] nlamb = [] nfilters = [] rindxs = [] for csfilter, clamb, cfilter in zip(shortfilters, g0.lamb, filters): if csfilter not in rm_filters: nlamb.append(clamb) nfilters.append(cfilter) else: rindxs.append(shortfilters.index(csfilter)) nseds = np.delete(g0.seds, rindxs, 1) print('orig filters: {}'.format(' '.join(filters))) print(' new filters: {}'.format(' '.join(nfilters))) g = SpectralGrid(np.array(nlamb), seds=nseds, grid=g0.grid, backend='memory') g.grid.header['filters'] = ' '.join(nfilters) g.writeHDF('{}_sed.grid.hd5'.format(outbase)) # get and process the observation model obsgrid = noisemodel.get_noisemodelcat(obsgrid) with tables.open_file('{}_noisemodel.grid.hd5'.format(outbase), 'w') \ as outfile: outfile.create_array(outfile.root, 'bias', np.delete(obsgrid.root.bias, rindxs, 1)) outfile.create_array(outfile.root, 'error', np.delete(obsgrid.root.error, rindxs, 1)) outfile.create_array(outfile.root, 'completeness', np.delete(obsgrid.root.completeness, rindxs, 1))
def gen_subgrid(i, sub_name): sub_g_pspec = FileSEDGrid(sub_name) sub_seds_fname = '{}seds.gridsub{}.hd5'.format(file_prefix, i) (sub_seds_fname, sub_g_seds) = make_extinguished_sed_grid( datamodel.project, sub_g_pspec, datamodel.filters, extLaw=datamodel.extLaw, av=datamodel.avs, rv=datamodel.rvs, fA=datamodel.fAs, rv_prior_model=datamodel.rv_prior_model, av_prior_model=datamodel.av_prior_model, fA_prior_model=datamodel.fA_prior_model, add_spectral_properties_kwargs=extra_kwargs, seds_fname=sub_seds_fname) return sub_seds_fname
def test_toothpick_noisemodel(): # download the needed files asts_fname = download_rename("fake_stars_b15_27_all.hd5") filter_fname = download_rename("filters.hd5") vega_fname = download_rename("vega.hd5") hst_fname = download_rename("hst_whitedwarf_frac_covar.fits") seds_fname = download_rename("beast_example_phat_seds.grid.hd5") # download cached version of noisemodel on the sed grid noise_fname_cache = download_rename("beast_example_phat_noisemodel.grid.hd5") ################ # get the modesedgrid on which to generate the noisemodel modelsedgrid = FileSEDGrid(seds_fname) # absflux calibration covariance matrix for HST specific filters (AC) filters = [ "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_ACS_WFC_F475W", "HST_ACS_WFC_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", ] absflux_a_matrix = hst_frac_matrix( filters, hst_fname=hst_fname, filterLib=filter_fname ) # generate the AST noise model noise_fname = "/tmp/beast_example_phat_noisemodel.grid.hd5" noisemodel.make_toothpick_noise_model( noise_fname, asts_fname, modelsedgrid, absflux_a_matrix=absflux_a_matrix, vega_fname=vega_fname, use_rate=False, ) # compare the new to the cached version compare_hdf5(noise_fname_cache, noise_fname)
def test_trim_grid(): # download the needed files vega_fname = download_rename("vega.hd5") seds_fname = download_rename("beast_example_phat_seds.grid.hd5") noise_fname = download_rename("beast_example_phat_noisemodel.grid.hd5") # download cached version of noisemodel on the sed grid simobs_fname_cache = download_rename("beast_example_phat_simobs.fits") ################ # get the physics model grid - includes priors modelsedgrid = FileSEDGrid(seds_fname) # read in the noise model - includes bias, unc, and completeness noisegrid = noisemodel.get_noisemodelcat(noise_fname) table_new = gen_SimObs_from_sedgrid( modelsedgrid, noisegrid, nsim=100, compl_filter="f475w", ranseed=1234, vega_fname=vega_fname, ) # check that the simobs files are exactly the same table_cache = Table.read(simobs_fname_cache) # to avoid issues with uppercase vs lowercase column names, make them all # the same before comparing for col in table_new.colnames: table_new[col].name = col.upper() for col in table_cache.colnames: table_cache[col].name = col.upper() compare_tables(table_cache, table_new)
def gen_subgrid(i, sub_name): sub_g_pspec = FileSEDGrid(sub_name) sub_seds_fname = "{}seds.gridsub{}.hd5".format(file_prefix, i) # generate the SED grid by integrating the filter response functions # effect of dust extinction applied before filter integration # also computes the dust priors as weights (sub_seds_fname, sub_g_seds) = make_extinguished_sed_grid( datamodel.project, sub_g_pspec, datamodel.filters, extLaw=datamodel.extLaw, av=datamodel.avs, rv=datamodel.rvs, fA=datamodel.fAs, rv_prior_model=datamodel.rv_prior_model, av_prior_model=datamodel.av_prior_model, fA_prior_model=datamodel.fA_prior_model, add_spectral_properties_kwargs=extra_kwargs, seds_fname=sub_seds_fname, ) return sub_seds_fname
def gen_subobsmodel(modelsedgridfile): modelsedgrid = FileSEDGrid(modelsedgridfile) # generate the AST noise model noisefile = modelsedgridfile.replace("seds", "noisemodel") astfile = settings.astfile # If we are treating regions with different # backgrounds/source densities separately, pick one of the # split ast files, and put the results in a subfolder. if args.dens_bin is not None: noisefile = os.path.join(bin_subfolder, noisefile) create_project_dir(os.path.dirname(noisefile)) astfile = subcatalog_fname(astfile, args.dens_bin) outname = noisemodel.make_toothpick_noise_model( noisefile, astfile, modelsedgrid, absflux_a_matrix=settings.absflux_a_matrix, ) return outname
def pick_models_toothpick_style( sedgrid_fname, filters, Nfilter, N_fluxes, min_N_per_flux, outfile=None, outfile_params=None, bins_outfile=None, bright_cut=None, ): """ Creates a fake star catalog from a BEAST model grid. The chosen seds are optimized for the toothpick model, by working with a given number of flux bins, and making sure that every flux bin is covered by at least a given number of models (for each filter individually, which is how the toothpick model works). Parameters ---------- sedgrid_fname: string BEAST model grid from which the models are picked (hdf5 file) filters: list of string Names of the filters, to be used as columns of the output table Nfilter: integer In how many filters a fake star needs to be brighter than the mag_cut value N_fluxes: integer The number of flux bins into which the dynamic range of the model grid in each filter is divided min_N_per_flux: integer Minimum number of model seds that need to fall into each bin outfile: string Output path for the models (optional). If this file already exists, the chosen seds are loaded from this file instead. outfile_params: string (default=None) If a file name is given, the physical parameters associated with each model will be written to disk bins_outfile: string Output path for a file containing the flux bin limits for each filter, and the number of samples for each (optional) bright_cut: list of float List of magnitude limits for each filter (won't sample model SEDs that are too bright) Returns ------- sedsMags: astropy Table A table containing the selected model seds (columns are named after the filters) """ if outfile is not None and os.path.isfile(outfile): print( "{} already exists. Will attempt to load SEDs for ASTs from there." .format(outfile)) t = Table.read(outfile, format="ascii") return t with Vega() as v: vega_f, vega_flux, lambd = v.getFlux(filters) modelsedgrid = FileSEDGrid(sedgrid_fname) sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux) Nf = sedsMags.shape[1] # Check if logL=-9.999 model points sliently sneak through if min(modelsedgrid.grid["logL"]) < -9: warnings.warn('There are logL=-9.999 model points in the SED grid!') print('Excluding those SED models from selecting input ASTs') idxs = np.where(modelsedgrid.grid["logL"] > -9)[0] sedsMags = sedsMags[idxs] # Set up a number of flux bins for each filter maxes = np.amax(sedsMags, axis=0) mins = np.amin(sedsMags, axis=0) bin_edges = np.zeros((N_fluxes + 1, Nf)) # indexed on [fluxbin, nfilters] for f in range(Nf): bin_edges[:, f] = np.linspace(mins[f], maxes[f], N_fluxes + 1) bin_mins = bin_edges[:-1, :] bin_maxs = bin_edges[1:, :] if not len(bin_mins) == len(bin_maxs) == N_fluxes: raise AssertionError() bin_count = np.zeros((N_fluxes, Nf)) chosen_idxs = [] counter = 0 successes = 0 include_mask = np.full(idxs.shape, True, dtype=bool) chunksize = 100000 while True: counter += 1 # pick some random models rand_idx = np.random.choice(idxs[include_mask], size=chunksize) randomseds = sedsMags[rand_idx, :] # Find in which bin each model belongs, for each filter fluxbins = np.zeros(randomseds.shape, dtype=int) for fltr in range(Nf): fluxbins[:, fltr] = np.digitize(randomseds[:, fltr], bin_maxs[:, fltr]) # Clip in place (models of which the flux is equal to the max # are assigned bin nr N_fluxes. Move these down to bin nr # N_fluxes - 1) np.clip(fluxbins, a_min=0, a_max=N_fluxes - 1, out=fluxbins) add_these = np.full((len(rand_idx)), False, dtype=bool) for r in range(len(rand_idx)): # If any of the flux bins that this model falls into does # not have enough samples yet, add it to the list of model # spectra to be output if (bin_count[fluxbins[r, :], range(Nf)] < min_N_per_flux).any(): bin_count[fluxbins[r, :], range(Nf)] += 1 successes += 1 add_these[r] = True # If all these bins are full... else: # ... do not include this model again, since we will reject it # anyway. include_mask[idxs == rand_idx] = False # Add the approved models chosen_idxs.extend(rand_idx[add_these]) # If some of the randomly picked models were not added if not add_these.any(): # ... check if we have enough samples everywhere, or if all # the models have been exhausted (and hence the bins are # impossible to fill). enough_samples = (bin_count.flatten() >= min_N_per_flux).all() still_models_left = include_mask.any() if enough_samples or not still_models_left: break if not counter % 10: print("Sampled {} models. {} successfull seds. Ratio = {}".format( counter * chunksize, successes, successes / counter / chunksize)) print("Bin array:") print(bin_count) # Gather the selected model seds in a table sedsMags = Table(sedsMags[chosen_idxs, :], names=filters) if outfile is not None: ascii.write( sedsMags, outfile, overwrite=True, formats={k: "%.5f" for k in sedsMags.colnames}, ) # if chosen, save the corresponding model parameters if outfile_params is not None: grid_dict = {} for key in list(modelsedgrid.grid.keys()): grid_dict[key] = modelsedgrid.grid[key][chosen_idxs] grid_dict['sedgrid_indx'] = chosen_idxs ast_params = Table(grid_dict) ast_params.write(outfile_params, overwrite=True) if bins_outfile is not None: bin_info_table = Table() col_bigarrays = [bin_mins, bin_maxs, bin_count] col_basenames = ["bin_mins_", "bin_maxs_", "bin_count_"] for fltr, filter_name in enumerate(filters): for bigarray, basename in zip(col_bigarrays, col_basenames): bin_info_table.add_column( Column(bigarray[:, fltr], name=basename + filter_name)) ascii.write(bin_info_table, bins_outfile, overwrite=True) return sedsMags
def plot_noisemodel( sed_file, noise_file_list, plot_file, samp=100, color=["black", "red", "gold", "lime", "xkcd:azure"], label=None, ): """ Make a plot of the noise model: for each of the bandsm make plots of bias and uncertainty as a function of flux If there are multiple files in noise_file_list, each of them will be overplotted in each panel. Parameters ---------- sed_file : string path+name of the SED grid file noise_file_list : list of strings path+name of the noise model file(s) plot_file : string name of the file to save the plot samp : int (default=100) plotting all of the SED points takes a long time for a viewer to load, so set this to plot every Nth point color : list of strings (default=['black','red','gold','lime','xkcd:azure']) colors to cycle through when making plots label : list of strings (default=None) if set, use these labels in a legend for each item in noise_file_list """ # read in the SED grid print("* reading SED grid file") sed_object = FileSEDGrid(sed_file) if hasattr(sed_object.seds, "read"): sed_grid = sed_object.seds.read() else: sed_grid = sed_object.seds filter_list = sed_object.filters n_filter = len(filter_list) # figure fig = plt.figure(figsize=(4 * n_filter, 10)) # go through noise files for n, nfile in enumerate(noise_file_list): print("* reading " + nfile) # read in the values noisemodel_vals = noisemodel.get_noisemodelcat(nfile) # extract error and bias noise_err = noisemodel_vals.root.error[:] noise_bias = noisemodel_vals.root.bias[:] # plot things for f, filt in enumerate(filter_list): # error is negative where it's been extrapolated -> trim those good_err = np.where(noise_err[:, f] > 0)[0] plot_sed = sed_grid[good_err, f][::samp] plot_err = noise_err[good_err, f][::samp] plot_bias = noise_bias[good_err, f][::samp] # subplot region: bias ax1 = plt.subplot(2, n_filter, f + 1) (plot1,) = ax1.plot( np.log10(plot_sed), plot_bias / plot_sed, marker="o", linestyle="none", mew=0, ms=2, color=color[n % len(color)], alpha=0.1, ) if label is not None: plot1.set_label(label[n]) ax1.tick_params(axis="both", which="major", labelsize=13) # ax.set_xlim(ax.get_xlim()[::-1]) plt.xlabel("log " + filt, fontsize=12) plt.ylabel(r"Bias ($\mu$/F)", fontsize=12) # subplot region: error ax2 = plt.subplot(2, n_filter, n_filter + f + 1) (plot2,) = ax2.plot( np.log10(plot_sed), plot_err / plot_sed, marker="o", linestyle="none", mew=0, ms=2, color=color[n % len(color)], alpha=0.1, ) if label is not None: plot2.set_label(label[n]) ax2.tick_params(axis="both", which="major", labelsize=13) # ax.set_xlim(ax.get_xlim()[::-1]) plt.xlabel("log " + filt, fontsize=12) plt.ylabel(r"Error ($\sigma$/F)", fontsize=12) # do a legend if this is # (a) the leftmost panel # (b) the last line to be added # (c) there are labels set if (f == 0) and (n == len(noise_file_list) - 1) and (label is not None): leg = ax1.legend(fontsize=12) for lh in leg.legendHandles: lh._legmarker.set_alpha(1) leg = ax2.legend(fontsize=12) for lh in leg.legendHandles: lh._legmarker.set_alpha(1) plt.tight_layout() fig.savefig(plot_file) plt.close(fig)
parser.add_argument("obsgrid", help="filename of observation/nosie grid") parser.add_argument("outfile", help="filename for simulated observations") parser.add_argument("--nsim", default=100, type=int, help="number of simulated objects") parser.add_argument("--compl_filter", default="F475W", help="filter name to use for completeness") parser.add_argument("--ranseed", default=None, type=int, help="seed for random number generator") args = parser.parse_args() # get the physics model grid - includes priors modelsedgrid = FileSEDGrid(args.physgrid) # read in the noise model - includes bias, unc, and completeness noisegrid = noisemodel.get_noisemodelcat(args.obsgrid) simtable = gen_SimObs_from_sedgrid( modelsedgrid, noisegrid, nsim=args.nsim, compl_filter=args.compl_filter, ranseed=args.ranseed, ) simtable.write(args.outfile, overwrite=True)
def simulate_obs( physgrid_list, noise_model_list, output_catalog, nsim=100, compl_filter="F475W", ranseed=None, ): """ Wrapper for creating a simulated photometry. Parameters ---------- physgrid_list : list of strings Name of the physics model file. If there are multiple physics model grids (i.e., if there are subgrids), list them all here, and they will each be sampled nsim/len(physgrid_list) times. noise_model_list : list of strings Name of the noise model file. If there are multiple files for physgrid_list (because of subgrids), list the noise model file associated with each physics model file. output_catalog : string Name of the output simulated photometry catalog n_sim : int (default=100) Number of simulated objects to create. If nsim/len(physgrid_list) isn't an integer, this will be increased so that each grid has the same number of samples. compl_filter : string (default='F475W') filter name to use for completeness ranseed : int seed for random number generator """ # numbers of samples to do # (ensure there are enough for even sampling of multiple model grids) n_phys = len(physgrid_list) samples_per_grid = int(np.ceil(nsim / n_phys)) # list to hold all simulation tables simtable_list = [] # make a table for each physics model + noise model for physgrid, noise_model in zip( np.atleast_1d(physgrid_list), np.atleast_1d(noise_model_list) ): # get the physics model grid - includes priors modelsedgrid = FileSEDGrid(str(physgrid)) # read in the noise model - includes bias, unc, and completeness noisegrid = noisemodel.get_noisemodelcat(str(noise_model)) # generate the table simtable = gen_SimObs_from_sedgrid( modelsedgrid, noisegrid, nsim=samples_per_grid, compl_filter=compl_filter, ranseed=ranseed, ) # append to the list simtable_list.append(simtable) # stack all the tables into one and write it out vstack(simtable_list).write(output_catalog, overwrite=True)
def run_beast_production(basename, physicsmodel=False, ast=False, observationmodel=False, trim=False, fitting=False, resume=False, source_density='', sub_source_density=''): """ Turns the original command-line version of run_beast_production.py into something callable from within a function Parameters ---------- basename : string name of the gst file (assuming it's located in ./data/) For the info related to the other inputs, see the argparse info at the bottom """ # before doing ANYTHING, force datamodel to re-import (otherwise, any # changes within this python session will not be loaded!) importlib.reload(datamodel) # check input parameters, print what is the problem, stop run_beast verify_params.verify_input_format(datamodel) # update the filenames as needed for production # - photometry sub-file datamodel.obsfile = basename.replace( '.fits', '_with_sourceden' + '_SD_' + source_density.replace('_', '-') + '_sub' + sub_source_density + '.fits') # - stats files stats_filebase = "%s/%s"%(datamodel.project,datamodel.project) \ + '_sd' + source_density.replace('_','-') \ + '_sub' + sub_source_density sed_trimname = stats_filebase + '_sed_trim.grid.hd5' # - trimmed noise model noisemodel_trimname = stats_filebase + '_noisemodel_trim.hd5' # - SED grid #modelsedgrid_filename = "%s/%s_seds.grid.hd5"%(datamodel.project, # datamodel.project) modelsedgrid_filename = "METAL_seds.grid.hd5" print("***run information***") print(" project = " + datamodel.project) print(" obsfile = " + datamodel.obsfile) print(" astfile = " + datamodel.astfile) print(" noisefile = " + datamodel.noisefile) print(" trimmed sedfile = " + sed_trimname) print("trimmed noisefiles = " + noisemodel_trimname) print(" stats filebase = " + stats_filebase) # make sure the project directory exists pdir = create_project_dir(datamodel.project) if physicsmodel: # download and load the isochrones (iso_fname, oiso) = make_iso_table(datamodel.project, oiso=datamodel.oiso, logtmin=datamodel.logt[0], logtmax=datamodel.logt[1], dlogt=datamodel.logt[2], z=datamodel.z) if hasattr(datamodel, 'add_spectral_properties_kwargs'): extra_kwargs = datamodel.add_spectral_properties_kwargs else: extra_kwargs = None if hasattr(datamodel, 'velocity'): redshift = (datamodel.velocity / const.c).decompose().value else: redshift = 0 # generate the spectral library (no dust extinction) (spec_fname, g_spec) = make_spectral_grid( datamodel.project, oiso, osl=datamodel.osl, redshift=redshift, distance=datamodel.distances, distance_unit=datamodel.distance_unit, add_spectral_properties_kwargs=extra_kwargs) # add the stellar priors as weights # also computes the grid weights for the stellar part (pspec_fname, g_pspec) = add_stellar_priors(datamodel.project, g_spec) # generate the SED grid by integrating the filter response functions # effect of dust extinction applied before filter integration # also computes the dust priors as weights (seds_fname, g_seds) = make_extinguished_sed_grid( datamodel.project, g_pspec, datamodel.filters, extLaw=datamodel.extLaw, av=datamodel.avs, rv=datamodel.rvs, fA=datamodel.fAs, rv_prior_model=datamodel.rv_prior_model, av_prior_model=datamodel.av_prior_model, fA_prior_model=datamodel.fA_prior_model, spec_fname=modelsedgrid_filename, add_spectral_properties_kwargs=extra_kwargs) if ast: N_models = datamodel.ast_models_selected_per_age Nfilters = datamodel.ast_bands_above_maglimit Nrealize = datamodel.ast_realization_per_model mag_cuts = datamodel.ast_maglimit obsdata = datamodel.get_obscat(basename, datamodel.filters) if len(mag_cuts) == 1: tmp_cuts = mag_cuts min_mags = np.zeros(len(datamodel.filters)) for k, filtername in enumerate(obsdata.filters): sfiltername = obsdata.data.resolve_alias(filtername) sfiltername = sfiltername.replace('rate', 'vega') sfiltername = sfiltername.replace('RATE', 'VEGA') keep, = np.where(obsdata[sfiltername] < 99.) min_mags[k] = np.percentile(obsdata[keep][sfiltername], 90.) # max. mags from the gst observation cat. mag_cuts = min_mags + tmp_cuts outfile = './' + datamodel.project + '/' + datamodel.project + '_inputAST.txt' outfile_params = './' + datamodel.project + '/' + datamodel.project + '_ASTparams.fits' chosen_seds = pick_models(modelsedgrid_filename, datamodel.filters, mag_cuts, Nfilter=Nfilters, N_stars=N_models, Nrealize=Nrealize, outfile=outfile, outfile_params=outfile_params) if datamodel.ast_with_positions == True: separation = datamodel.ast_pixel_distribution filename = datamodel.project + '/' + datamodel.project + '_inputAST.txt' if datamodel.ast_reference_image is not None: # With reference image, use the background or source density map if available if datamodel.ast_density_table is not None: pick_positions_from_map( obsdata, chosen_seds, datamodel.ast_density_table, datamodel.ast_N_bins, datamodel.ast_realization_per_model, outfile=filename, refimage=datamodel.ast_reference_image, refimage_hdu=0, Nrealize=1, set_coord_boundary=datamodel.ast_coord_boundary) else: pick_positions(obsdata, filename, separation, refimage=datamodel.ast_reference_image) else: # Without reference image, we can only use this function if datamodel.ast_density_table is None: pick_positions(obsdata, filename, separation) else: print( "To use ast_density_table, ast_reference_image must be specified." ) if observationmodel: print('Generating noise model from ASTs and absflux A matrix') # get the modesedgrid on which to generate the noisemodel modelsedgrid = FileSEDGrid(modelsedgrid_filename) # generate the AST noise model noisemodel.make_toothpick_noise_model( \ datamodel.noisefile, datamodel.astfile, modelsedgrid, use_rate=True, absflux_a_matrix=datamodel.absflux_a_matrix) if trim: print('Trimming the model and noise grids') # read in the observed data obsdata = datamodel.get_obscat(basename, datamodel.filters) # get the modesedgrid on which to generate the noisemodel modelsedgrid = FileSEDGrid(modelsedgrid_filename) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat(datamodel.noisefile) # trim the model sedgrid trim_grid.trim_models(modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.) if fitting: start_time = time.clock() # read in the the AST noise model noisemodel_vals = noisemodel.get_noisemodelcat(noisemodel_trimname) # read in the observed data obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters) # output files statsfile = stats_filebase + '_stats.fits' pdf1dfile = statsfile.replace('stats.fits', 'pdf1d.fits') lnpfile = statsfile.replace('stats.fits', 'lnp.hd5') fit.summary_table_memory(obsdata, noisemodel_vals, sed_trimname, resume=resume, threshold=-10., save_every_npts=100, lnp_npts=500, stats_outname=statsfile, pdf1d_outname=pdf1dfile, lnp_outname=lnpfile, surveyname=datamodel.surveyname) new_time = time.clock() print('time to fit: ', (new_time - start_time) / 60., ' min')
def remove_filters_from_files( catfile, physgrid=None, obsgrid=None, outbase=None, physgrid_outfile=None, rm_filters=None, beast_filt=None, ): """ Remove filters from catalog, physics grid, and/or obsmodel grid. This has two primary use cases: 1. When making simulated observations, you want to test how your fit quality changes with different combinations of filters. In that case, put in files for both `physgrid` and `obsgrid`. Set `rm_filters` to the filter(s) you wish to remove, and they will be removed both from those and from the catalog file. The three new files will be output with the name prefix set in `outbase`. 2. When running the BEAST, you have a master physics model grid with all filters present in the survey, but some fields don't have observations in all of those filters. In that case, put the master grid in `physgrid` and set `rm_filters` to None. The catalog will be used to determine the filters to remove (if any). `obsgrid` should be left as None, because in this use case, the obsmodel grid has not yet been generated. The output physics model grid will be named using the filename in `physgrid_outfile` (if given) or with the prefix in `outbase`. Parameters ---------- catfile : string file name of photometry catalog physgrid : string (default=None) If set, remove filters from this physics model grid obsgrid : string (default=None) If set, remove filters from this obsmodel grid outbase : string (default=None) Path+file to prepend to all output file names. Useful for case 1 above. physgrid_outfile : string (default=None) Path+name of the output physics model grid. Useful for case 2 above. rm_filters : string or list of strings (default=None) If set, these are the filters to remove from all of the files. If not set, only the filters present in catfile will be retained in physgrid and/or obsgrid. beast_filt : list of strings Sometimes there is ambiguity in the filter name (e.g., the grid has both HST_ACS_WFC_F475W and HST_WFC3_F475W, and the filter name is F475W). Set this to the BEAST filter name to resolve any ambiguities. For example, ['HST_WFC3_F475W', 'HST_WFC3_F814W'] ensures that these are the names used for F475W and F814W. """ # read in the photometry catalog cat = Table.read(catfile) # if rm_filters set, remove the requested filters from the catalog if rm_filters is not None: for cfilter in np.atleast_1d(rm_filters): colname = "{}_rate".format(cfilter) if colname.upper() in cat.colnames: cat.remove_column(colname.upper()) elif colname.lower() in cat.colnames: cat.remove_column(colname.lower()) else: print("{} not in catalog file".format(colname)) cat.write("{}_cat.fits".format(outbase), overwrite=True) # if rm_filters not set, extract the filter names that are present if rm_filters is None: cat_filters = [f[:-5].upper() for f in cat.colnames if f[-4:].lower() == "rate"] # if beast_filt is set, make a list of the short versions if beast_filt is not None: beast_filt_short = [(f.split("_"))[-1].upper() for f in beast_filt] # if physgrid set, process the SED grid if physgrid is not None: # read in the sed grid g0 = FileSEDGrid(physgrid, backend="cache") # extract info filters = g0.header["filters"].split(" ") shortfilters = [(cfilter.split("_"))[-1].upper() for cfilter in filters] rindxs = [] rgridcols = [] # loop through filters and determine what needs deleting for csfilter, cfilter in zip(shortfilters, filters): # -------------------------- # if the user chose the filters to remove if rm_filters is not None: # if the current filter is in the list of filters to remove if csfilter in np.atleast_1d(rm_filters): # if there's a list of BEAST instrument+filter references if beast_filt is not None: # if the current filter is in the list of BEAST references if csfilter in beast_filt_short: # if it's the same instrument, delete it # (if it's not the same instrument, keep it) if beast_filt[beast_filt_short.index(csfilter)] == cfilter: rindxs.append(filters.index(cfilter)) for grid_col in g0.grid.colnames: if cfilter in grid_col: rgridcols.append(grid_col) # if the current filter isn't in the BEAST ref list, delete it else: rindxs.append(filters.index(cfilter)) for grid_col in g0.grid.colnames: if cfilter in grid_col: rgridcols.append(grid_col) # if there isn't a list of BEAST refs, delete it else: rindxs.append(filters.index(cfilter)) for grid_col in g0.grid.colnames: if cfilter in grid_col: rgridcols.append(grid_col) # -------------------------- # if the removed filters are determined from the catalog file if rm_filters is None: # if the current filter is present in the catalog filters if csfilter in cat_filters: # if there's a list of BEAST instrument+filter references # (if there isn't a list of BEAST refs, keep it) if beast_filt is not None: # if the current filter is in the list of BEAST references # (if the current filter isn't in the BEAST ref list, keep it) if csfilter in beast_filt_short: # if it's not the same instrument, delete it # (if it's the same instrument, keep it) if beast_filt[beast_filt_short.index(csfilter)] != cfilter: rindxs.append(filters.index(cfilter)) for grid_col in g0.grid.colnames: if cfilter in grid_col: rgridcols.append(grid_col) # if the current filter isn't in the catalog filters, delete it else: rindxs.append(filters.index(cfilter)) for grid_col in g0.grid.colnames: if cfilter in grid_col: rgridcols.append(grid_col) # delete column(s) nseds = np.delete(g0.seds, rindxs, 1) nlamb = np.delete(g0.lamb, rindxs, 0) nfilters = np.delete(filters, rindxs, 0) for rcol in rgridcols: g0.grid.delCol(rcol) print("orig filters: {}".format(" ".join(filters))) print(" new filters: {}".format(" ".join(nfilters))) # save the modified grid g = SpectralGrid(np.array(nlamb), seds=nseds, grid=g0.grid, backend="memory") g.grid.header["filters"] = " ".join(nfilters) if physgrid_outfile is not None: g.writeHDF(physgrid_outfile) elif outbase is not None: g.writeHDF("{}_seds.grid.hd5".format(outbase)) else: raise ValueError("Need to set either outbase or physgrid_outfile") # if obsgrid set, process the observation model if obsgrid is not None: obsgrid = noisemodel.get_noisemodelcat(obsgrid) with tables.open_file("{}_noisemodel.grid.hd5".format(outbase), "w") as outfile: outfile.create_array( outfile.root, "bias", np.delete(obsgrid["bias"], rindxs, 1) ) outfile.create_array( outfile.root, "error", np.delete(obsgrid["error"], rindxs, 1) ) outfile.create_array( outfile.root, "completeness", np.delete(obsgrid["completeness"], rindxs, 1), )
def pick_models( sedgrid_fname, filters, mag_cuts, Nfilter=3, N_stars=70, Nrealize=20, outfile=None, outfile_params=None, bright_cut=None, vega_fname=None, ranseed=None, ): """Creates a fake star catalog from a BEAST model grid Parameters ---------- sedgrid_fname: string BEAST model grid from which the models are picked (hdf5 file) filters: list of string Names of the filters mag_cuts: list List of magnitude limits for each filter Nfilter: Integer In how many filters, you want a fake star to be brighter than the limit (mag_cut) (default = 3) N_stars: Integer Number of stellar models picked per a single log(age) (default=70) Nrealize: Integer Number of realization of each models (default = 20) outfile: str If a file name is given, the selected models will be written to disk outfile_params: str If a file name is given, the physical parameters associated with each model will be written to disk bright_cut: list of float Same as mag_cuts, but for the bright end vega_fname: str filename of vega file ranseed : int used to set the seed to make the results reproducable useful for testing Returns ------- astropy Table of selected models - and optionally - ascii file: A list of selected models, written to 'outfile' fits file: the corresponding physical parameters, written to 'outfile_params' """ with Vega(source=vega_fname) as v: # Get the vega fluxes vega_f, vega_flux, lamb = v.getFlux(filters) # gridf = h5py.File(sedgrid_fname) modelsedgrid = FileSEDGrid(sedgrid_fname) # Convert to Vega mags # sedsMags = -2.5 * np.log10(gridf['seds'][:] / vega_flux) sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux) # make sure Nfilters isn't larger than the total number of filters if Nfilter > len(filters): Nfilter = len(filters) # Select the models above the magnitude limits in N filters idxs = mag_limits(sedsMags, mag_cuts, Nfilter=Nfilter, bright_cut=bright_cut) # grid_cut = gridf['grid'][list(idxs)] cols = {} for key in list(modelsedgrid.grid.keys()): cols[key] = modelsedgrid.grid[key][idxs] grid_cut = Table(cols) # Sample the model grid uniformly prime_params = np.column_stack( (grid_cut["logA"], grid_cut["M_ini"], grid_cut["Av"])) search_age = np.unique(prime_params[:, 0]) N_sample = N_stars model_ind = [] # indices for the model grid ast_params = grid_cut[[]] # the corresponding model parameters # set the random seed - mainly for testing if not None: np.random.seed(ranseed) for iage in search_age: tmp, = np.where(prime_params[:, 0] == iage) new_ind = np.random.choice(tmp, N_sample) model_ind.append(new_ind) [ast_params.add_row(grid_cut[new_ind[i]]) for i in range(len(new_ind))] index = np.repeat(idxs[np.array(model_ind).reshape((-1))], Nrealize) sedsMags = Table(sedsMags[index, :], names=filters) if outfile is not None: ascii.write( sedsMags, outfile, overwrite=True, formats={k: "%.5f" for k in sedsMags.colnames}, ) if outfile_params is not None: ast_params.write(outfile_params, overwrite=True) return sedsMags
def plot_completeness( physgrid_list, noise_model_list, output_plot_filename, param_list=['Av', 'Rv', 'logA', 'f_A', 'M_ini', 'Z', 'distance'], compl_filter='F475W', ): """ Make visualization of the completeness Parameters ---------- physgrid_list : string or list of strings Name of the physics model file. If there are multiple physics model grids (i.e., if there are subgrids), list them all here. noise_model_list : string or list of strings Name of the noise model file. If there are multiple files for physgrid_list (because of subgrids), list the noise model file associated with each physics model file. param_list : list of strings names of the parameters to plot compl_filter : str filter to use for completeness (required for toothpick model) output_plot_filename : string name of the file in which to save the output plot """ n_params = len(param_list) # If there are subgrids, we can't read them all into memory. Therefore, # we'll go through each one and just grab the relevant parts. compl_table_list = [] # make a table for each physics model + noise model for physgrid, noise_model in zip( np.atleast_1d(physgrid_list), np.atleast_1d(noise_model_list) ): # get the physics model grid - includes priors modelsedgrid = FileSEDGrid(str(physgrid)) # get list of filters short_filters = [filter.split(sep="_")[-1].upper() for filter in modelsedgrid.filters] if compl_filter.upper() not in short_filters: raise ValueError("requested completeness filter not present") filter_k = short_filters.index(compl_filter.upper()) print("Completeness from {0}".format(modelsedgrid.filters[filter_k])) # read in the noise model noisegrid = noisemodel.get_noisemodelcat(str(noise_model)) # get the completeness model_compl = noisegrid["completeness"] # close the file to save memory noisegrid.close() # put it all into a table table_dict = {x:modelsedgrid[x] for x in param_list} table_dict['compl'] = model_compl[:, filter_k] # append to the list compl_table_list.append(Table(table_dict)) # stack all the tables into one compl_table = vstack(compl_table_list) #import pdb; pdb.set_trace() # figure fig = plt.figure(figsize=(4*n_params, 4*n_params)) # label font sizes label_font = 25 tick_font = 22 # load in color map cmap = matplotlib.cm.get_cmap('magma') # iterate through the panels for i,pi in enumerate(param_list): for j,pj in enumerate(param_list[i:], i): print('plotting {0} and {1}'.format(pi, pj)) # not along diagonal if i != j: # set up subplot plt.subplot(n_params, n_params, i + j*(n_params) + 1) ax = plt.gca() # create image and labels x_col, x_bins, x_label = setup_axis(compl_table, pi) y_col, y_bins, y_label = setup_axis(compl_table, pj) compl_image, _, _, _ = binned_statistic_2d( x_col, y_col, compl_table['compl'], statistic='mean', bins=(x_bins, y_bins), ) # plot points im = plt.imshow( compl_image.T, #np.random.random((4,4)), extent=( np.min(x_bins), np.max(x_bins), np.min(y_bins), np.max(y_bins) ), cmap='magma', vmin=0, vmax=1, aspect='auto', origin='lower', ) ax.tick_params(axis='both', which='both', direction='in', labelsize=tick_font, bottom=True, top=True, left=True, right=True) # axis labels and ticks if i == 0: ax.set_ylabel(y_label, fontsize=label_font) #ax.get_yaxis().set_label_coords(-0.35,0.5) else: ax.set_yticklabels([]) if j == n_params-1: ax.set_xlabel(x_label, fontsize=label_font) plt.xticks(rotation=-45) else: ax.set_xticklabels([]) # along diagonal if i == j: # set up subplot plt.subplot(n_params, n_params, i + j*(n_params) + 1) ax = plt.gca() # create histogram and labels x_col, x_bins, x_label = setup_axis(compl_table, pi) compl_hist, _, _ = binned_statistic( x_col, compl_table['compl'], statistic='mean', bins=x_bins, ) # make histogram _, _, patches = plt.hist(x_bins[:-1], x_bins, weights=compl_hist) # color each bar by its completeness for c,comp in enumerate(compl_hist): patches[c].set_color(cmap(comp)) patches[c].set_linewidth=0.1 # make a black outline so it stands out as a histogram plt.hist(x_bins[:-1], x_bins, weights=compl_hist, histtype='step', color='k') # axis ranges plt.xlim(np.min(x_bins), np.max(x_bins)) plt.ylim(0, 1.05) ax.tick_params(axis='y',which='both',length=0, labelsize=tick_font) ax.tick_params(axis='x',which='both',direction='in', labelsize=tick_font) # axis labels and ticks ax.set_yticklabels([]) if i < n_params-1: ax.set_xticklabels([]) if i == n_params-1: ax.set_xlabel(x_label, fontsize=label_font) plt.xticks(rotation=-45) #plt.subplots_adjust(wspace=0.05, hspace=0.05) plt.tight_layout() # add a colorbar gs = GridSpec(nrows=20, ncols=n_params) cax = fig.add_subplot(gs[0, 2:]) cbar = plt.colorbar(im, cax=cax, orientation='horizontal') cbar.set_label('Completeness', fontsize=label_font) cbar.ax.tick_params(labelsize=tick_font) gs.tight_layout(fig) fig.savefig(output_plot_filename) plt.close(fig)
def fit_submodel( photometry_file, modelsedgrid_file, noise_file, stats_file, pdf_file, pdf2d_file, pdf2d_param_list, lnp_file, grid_info_file=None, resume=False, ): """ Code to run the SED fitting Parameters ---------- photometry_file : string path+name of the photometry file modelsedgrid_file : string path+name of the physics model grid file noise_file : string path+name of the noise model file stats_file : string path+name of the file to contain stats output pdf_file : string path+name of the file to contain 1D PDF output pdf2d_file : string path+name of the file to contain 2D PDF output pdf2d_param_list: list of strings or None parameters for which to make 2D PDFs (or None) lnp_file : string path+name of the file to contain log likelihood output grid_info_file : string (default=None) path+name for pickle file that contains dictionary with subgrid min/max/n_unique (required for a run with subgrids) resume : boolean (default=False) choose whether to resume existing run or start over Returns ------- noisefile : string name of the created noise file """ # read in the photometry catalog obsdata = datamodel.get_obscat(photometry_file, datamodel.filters) # check if it's a subgrid run by looking in the file name if "gridsub" in modelsedgrid_file: subgrid_run = True print("loading grid_info_dict from " + grid_info_file) with open(grid_info_file, "rb") as p: grid_info_dict = pickle.loads(p.read()) else: subgrid_run = False # load the SED grid and noise model modelsedgrid = FileSEDGrid(modelsedgrid_file) noisemodel_vals = noisemodel.get_noisemodelcat(noise_file) if subgrid_run: fit.summary_table_memory( obsdata, noisemodel_vals, modelsedgrid, resume=resume, threshold=-10.0, save_every_npts=100, lnp_npts=500, stats_outname=stats_file, pdf1d_outname=pdf_file, pdf2d_outname=pdf2d_file, pdf2d_param_list=pdf2d_param_list, grid_info_dict=grid_info_dict, lnp_outname=lnp_file, do_not_normalize=True, surveyname=datamodel.surveyname, ) print("Done fitting on grid " + modelsedgrid_file) else: fit.summary_table_memory( obsdata, noisemodel_vals, modelsedgrid, resume=resume, threshold=-10.0, save_every_npts=100, lnp_npts=500, stats_outname=stats_file, pdf1d_outname=pdf_file, pdf2d_outname=pdf2d_file, pdf2d_param_list=pdf2d_param_list, lnp_outname=lnp_file, surveyname=datamodel.surveyname, ) print("Done fitting on grid " + modelsedgrid_file)
g_pspec, datamodel.filters, extLaw=datamodel.extLaw, av=datamodel.avs, rv=datamodel.rvs, fA=datamodel.fAs, rv_prior_model=datamodel.rv_prior_model, av_prior_model=datamodel.av_prior_model, fA_prior_model=datamodel.fA_prior_model, add_spectral_properties_kwargs=extra_kwargs) if args.ast: # get the modesedgrid on which to grab input AST modelsedgridfile = datamodel.project + '/' + datamodel.project + \ '_seds.grid.hd5' modelsedgrid = FileSEDGrid(modelsedgridfile) N_models = datamodel.ast_models_selected_per_age Nfilters = datamodel.ast_bands_above_maglimit Nrealize = datamodel.ast_realization_per_model mag_cuts = datamodel.ast_maglimit if len(mag_cuts) == 1: tmp_cuts = mag_cuts obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters) min_mags = np.zeros(len(datamodel.filters)) for k, filtername in enumerate(obsdata.filters): sfiltername = obsdata.data.resolve_alias(filtername) sfiltername = sfiltername.replace('rate', 'vega')
separation = datamodel.ast_pixel_distribution filename = datamodel.project + '/' + datamodel.project + '_inputAST.txt' if datamodel.ast_reference_image is not None: pick_positions(obsdata, filename, separation, refimage=datamodel.ast_reference_image) else: pick_positions(obsdata, filename, separation) if args.observationmodel: print('Generating noise model from ASTs and absflux A matrix') # get the modesedgrid on which to generate the noisemodel modelsedgrid = FileSEDGrid(modelsedgrid_filename) # generate the AST noise model noisemodel.make_toothpick_noise_model( \ datamodel.noisefile, datamodel.astfile, modelsedgrid, use_rate=True, absflux_a_matrix=datamodel.absflux_a_matrix) if args.trim: print('Trimming the model and noise grids') # read in the observed data obsdata = datamodel.get_obscat(datamodel.obsfile, datamodel.filters)
parser.add_argument("trimfile", help="file with modelgrid, obsfiles, filebase to use") args = parser.parse_args() start_time = time.clock() # read in trim file f = open(args.trimfile, "r") file_lines = list(f) # physics model grid name modelfile = file_lines[0].rstrip() # get the modesedgrid on which to generate the noisemodel print("Reading the model grid files = ", modelfile) modelsedgrid = FileSEDGrid(modelfile) new_time = time.clock() print("time to read: ", (new_time - start_time) / 60.0, " min") old_noisefile = "" for k in range(1, len(file_lines)): print("\n\n") # file names noisefile, obsfile, filebase = file_lines[k].split() # make sure the proper directories exist if not os.path.isdir(os.path.dirname(filebase)): os.makedirs(os.path.dirname(filebase))
def plot_ast(ast_file, sed_grid_file=None): """ Make a histogram of the AST fluxes. If an SED grid is given, also plot a comparison histogram of the SED fluxes. The histogram bins are set by the bins originally used to create the ASTs (using the flux bin method), which are saved in ast_file.replace('inputAST','ASTfluxbins') and are automatically read in. Output plot is saved in the same location/name as ast_file, but with a .png instead of .txt. Parameters ---------- ast_file : string name of AST input file sed_grid_file : string (default=None) name of SED grid file """ # read in AST info ast_table = Table.read(ast_file, format='ascii') ast_fluxbins = Table.read(ast_file.replace('inputAST', 'ASTfluxbins'), format='ascii') # get filter names (and it's even in order!) filter_cols = [col for col in ast_table.colnames if '_' in col] filter_list = [col[-5:] for col in filter_cols] n_filter = len(filter_list) # if chosen, read in model grid if sed_grid_file is not None: modelsedgrid = FileSEDGrid(sed_grid_file) with Vega() as v: _, vega_flux, _ = v.getFlux(filter_cols) sedsMags = -2.5 * np.log10(modelsedgrid.seds[:] / vega_flux) good_seds = np.where(modelsedgrid.grid['logL'] > -9)[0] # make a histogram for each filter fig = plt.figure(figsize=(7, 4 * n_filter)) for f, filt in enumerate(filter_list): # names of table columns with bin values min_bin_col = [ b for b in ast_fluxbins.colnames if ('mins' in b and filt in b) ][0] max_bin_col = [ b for b in ast_fluxbins.colnames if ('maxs' in b and filt in b) ][0] # use those to make a bin list bin_list = np.append(ast_fluxbins[min_bin_col], ast_fluxbins[max_bin_col][-1]) # make histograms ax = plt.subplot(n_filter, 1, f + 1) ast_col = [b for b in ast_table.colnames if filt in b][0] plt.hist(ast_table[ast_col], bins=bin_list, normed=True, facecolor='black', edgecolor='none', alpha=0.3, label='ASTs') if sed_grid_file is not None: plt.hist(sedsMags[good_seds, f], bins=bin_list, normed=True, histtype='step', facecolor='none', edgecolor='black', label='Model grid') # labels ax.tick_params(axis='both', which='major', labelsize=13) ax.set_xlim(ax.get_xlim()[::-1]) plt.xlabel(filt + ' (Vega mag)', fontsize=14) plt.ylabel('Normalized Histogram', fontsize=14) # add legend in first plot if f == 0: ax.legend(fontsize=14) plt.tight_layout() fig.savefig(ast_file.replace('.txt', '.png')) plt.close(fig)