def beast_production_wrapper(): """ This does all of the steps for a full production run, and can be used as a wrapper to automatically do most steps for multiple fields. * make beast settings file * make source density map * make background density map * make physics model (SED grid) * make input list for ASTs Places for user to manually do things: * editing code before use - beast_settings_template.py: setting up the file with desired parameters - here: list the catalog filter names with the corresponding BEAST names - here: choose settings (pixel size, filter, mag range) for the source density map - here: choose settings (pixel size, reference image) for the background map """ # get the list of fields field_names = ["M31-B17-WEST"] # reference image headers ast_ref_im = ["./data/M31-B17-WEST_F475W_drz_head.fits"] # filter for sorting ref_filter = ["F475W", "F475W"] # filter for checking flags flag_filter = ["F475W", "F475W"] # coordinates for the boundaries of the field boundary_coord = [[ [ 11.3569155882981, 11.3332629372269, 11.1996295883163, 11.2230444144334 ], [ 42.0402119924196, 41.9280756279533, 41.9439995726193, 42.0552016945299 ], ]] # number of fields # n_field = len(field_names) # Need to know what the correspondence is between filter names in the # catalog and the BEAST filter names. # # These will be used to automatically determine the filters present in # each GST file and fill in the beast settings file. The order doesn't # matter, as long as the order in one list matches the order in the other # list. # gst_filter_names = ["F275W", "F336W", "F475W", "F814W", "F110W", "F160W"] beast_filter_names = [ "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_ACS_WFC_F475W", "HST_ACS_WFC_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", ] # for b in range(n_field): for b in [0]: print("********") print("field {0} (b={1})".format(field_names[b], b)) print("********") # only create an AST input list if the ASTs don't already exist ast_input_file = ("./" + field_names[b] + "_beast/" + field_names[b] + "_beast_inputAST.txt") if os.path.isfile(ast_input_file): print("AST input file already exists... skipping \n") continue # ----------------- # data file names # ----------------- # paths for the data/AST files gst_file = "./data/" + field_names[b] + ".st.fits" ast_file = "./data/" + field_names[b] + ".gst.fake.fits" # region file with catalog stars # make_ds9_region_file.region_file_fits(gst_file) # make_ds9_region_file.region_file_fits(ast_file) # ----------------- # 1a. make magnitude histograms # ----------------- print("") print("making magnitude histograms") print("") # if not os.path.isfile('./data/'+field_names[b]+'.gst_maghist.pdf'): peak_mags = plot_mag_hist.plot_mag_hist(gst_file, stars_per_bin=70, max_bins=75) # test = plot_mag_hist.plot_mag_hist(ast_file, stars_per_bin=200, max_bins=30) # ----------------- # 1b. make a source density map # ----------------- print("") print("making source density map") print("") # source density map sd_map = gst_file.replace(".fits", "_source_den_image.fits") if not os.path.isfile(sd_map): # if True: # - pixel size of 5 arcsec # - use ref_filter[b] between vega mags of 15 and peak_mags[ref_filter[b]]-0.5 sourceden_args = types.SimpleNamespace( subcommand="sourceden", catfile=gst_file, pixsize=5, npix=None, mag_name=ref_filter[b] + "_VEGA", mag_cut=[15, peak_mags[ref_filter[b]] - 0.5], flag_name=flag_filter[b] + "_FLAG", ) create_background_density_map.main_make_map(sourceden_args) # new file name with the source density column # gst_file_sd = gst_file.replace(".fits", "_with_sourceden.fits") with fits.open(sd_map) as hdu_sd: sd_data = hdu_sd[0].data[hdu_sd[0].data != 0] ast_n_bins = np.ceil((np.max(sd_data) - np.min(sd_data)) / 1.0) # ----------------- # 4/5. edit photometry/AST catalogs # ----------------- # remove sources that are # - in regions without full imaging coverage, # - flagged in flag_filter # print('') # print('editing photometry/AST catalogs') # print('') # gst_file_cut = gst_file.replace('.fits', '_with_sourceden_cut.fits') # ast_file_cut = ast_file.replace('.fits', '_cut.fits') # cut_catalogs.cut_catalogs( # gst_file_sd, gst_file_cut, # #ast_file, ast_file_cut, # partial_overlap=True, flagged=True, flag_filter=flag_filter[b], # region_file=True) # ----------------- # 0. make beast settings file # ----------------- print("") print("creating beast settings file") print("") # get the boundaries of the image boundary_ra = boundary_coord[b][0] boundary_dec = boundary_coord[b][1] # make an eroded version for ASTs (10 pix = 0.5") boundary_polygon = geometry.Polygon( [[float(boundary_ra[i]), float(boundary_dec[i])] for i in range(len(boundary_ra))]) erode_polygon = boundary_polygon.buffer(-0.5 / 3600) boundary_ra_erode = [ str(x) for x in erode_polygon.exterior.coords.xy[0] ] boundary_dec_erode = [ str(x) for x in erode_polygon.exterior.coords.xy[1] ] create_beast_settings( gst_file, ast_file, gst_filter_names, beast_filter_names, ref_image=ast_ref_im[b], ast_n_bins=ast_n_bins, boundary_ra=boundary_ra_erode, boundary_dec=boundary_dec_erode, ) # load in beast settings settings = beast_settings.beast_settings("beast_settings_" + field_names[i] + ".txt") # ----------------- # 2. make physics model # ----------------- print("") print("making physics model") print("") model_grid_file = "./{0}_beast/{0}_beast_seds.grid.hd5".format( field_names[b]) # only make the physics model if it doesn't already exist if not os.path.isfile(model_grid_file): create_physicsmodel.create_physicsmodel(settings, nprocs=1, nsubs=settings.n_subgrid) # ----------------- # 3. make ASTs # ----------------- if not os.path.isfile(ast_file): # if True: if not os.path.isfile(ast_input_file): # if True: print("") print("creating artificial stars") print("") make_ast_inputs.make_ast_inputs(settings, flux_bin_method=True) # make a region file of the ASTs make_ds9_region_file.region_file_txt(ast_input_file) # make histograms of the fluxes plot_ast_histogram.plot_ast(ast_input_file, sed_grid_file=model_grid_files) print("\n**** go run ASTs for " + field_names[b] + "! ****\n") continue
def beast_production_wrapper(): """ This does all of the steps for a full production run, and can be used as a wrapper to automatically do most steps for multiple fields. * make datamodel.py file * make source density map * make background density map * split catalog by source density * make physics model (SED grid) * make input list for ASTs * make noise model * generate batch script to trim models * generate batch script to fit models * merge stats files back together * spatially reorder the results Places for user to manually do things: * editing code before use - datamodel_template.py: setting up the file with desired parameters - here: list the catalog filter names with the corresponding BEAST names - here: choose settings (pixel size, filter, mag range) for the source density map - here: choose settings (pixel size, reference image) for the background map - here: choose settings (filter, number per file) for dividing catalog by source density - here: choose settings (# files, nice level) for the trimming/fitting batch scripts * process the ASTs, as described in BEAST documentation * run the trimming scripts * run the fitting scripts BEWARE: When running the trimming/fitting scripts, ensure that the correct datamodel.py file is in use. Since it gets updated every time this code is run, you may be unexpectedly be using one from another field. """ # the list of fields field_names = ["15275_IC1613"] # distance moduli and velocities # http://adsabs.harvard.edu/abs/2013AJ....146...86T dist_mod = [24.36] velocity = [-236] # the path+file for a reference image im_path = ["../beast_dwarfs/images/15275_IC1613_F555W_drz.fits.gz"] ref_filter = ["F555W"] # choose a filter to use for removing artifacts # (remove catalog sources with filter_FLAG > 99) flag_filter = ["F555W"] # number of fields n_field = len(field_names) # Need to know what the correspondence is between filter names in the # catalog and the BEAST filter names. # # These will be used to automatically determine the filters present in # each GST file and fill in the datamodel.py file. The order doesn't # matter, as long as the order in one list matches the order in the other # list. # gst_filter_names = [ "F275W", "F336W", "F390M", "F555W", "F814W", "F110W", "F160W" ] beast_filter_names = [ "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_WFC3_F390M", "HST_WFC3_F555W", "HST_WFC3_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", ] for b in range(n_field): # for b in [0]: print("********") print("field " + field_names[b]) print("********") # ----------------- # data file names # ----------------- # paths for the data/AST files gst_file = "./data/" + field_names[b] + ".gst.fits" ast_file = "./data/" + field_names[b] + ".gst.fake.fits" # path for the reference image (if using for the background map) im_file = im_path[b] # region file with catalog stars # make_region_file(gst_file, ref_filter[b]) # ----------------- # 0. make datamodel file # ----------------- # need to do this first, because otherwise any old version that exists # will be imported, and changes made here won't get imported again print("") print("creating datamodel file") print("") create_datamodel( gst_file, ast_file, gst_filter_names, beast_filter_names, dist_mod[b], velocity[b], ref_image=im_file, ) # load in datamodel to get number of subgrids import datamodel importlib.reload(datamodel) # ----------------- # 1a. make magnitude histograms # ----------------- print("") print("making magnitude histograms") print("") # if not os.path.isfile('./data/'+field_names[b]+'.gst_maghist.pdf'): peak_mags = plot_mag_hist.plot_mag_hist(gst_file, stars_per_bin=70, max_bins=75) # test = plot_mag_hist.plot_mag_hist(ast_file, stars_per_bin=200, max_bins=30) # ----------------- # 1b. make a source density map # ----------------- print("") print("making source density map") print("") # not currently doing background density bins # use_bg_info = True use_bg_info = False if use_bg_info: background_args = types.SimpleNamespace( subcommand="background", catfile=gst_file, pixsize=5, npix=None, reference=im_file, mask_radius=10, ann_width=20, cat_filter=[ref_filter, "90"], ) create_background_density_map.main_make_map(background_args) # but we are doing source density bins! if not os.path.isfile( gst_file.replace(".fits", "_source_den_image.fits")): # - pixel size of 10 arcsec # - use ref_filter[b] between vega mags of 17 and peak_mags[ref_filter[b]]-0.5 sourceden_args = types.SimpleNamespace( subcommand="sourceden", catfile=gst_file, pixsize=5, npix=None, mag_name=ref_filter + "_VEGA", mag_cut=[15, peak_mags[ref_filter - 0.5]], flag_name=flag_filter[b] + '_FLAG', ) create_background_density_map.main_make_map(sourceden_args) # new file name with the source density column gst_file_sd = gst_file.replace(".fits", "_with_sourceden.fits") # ----------------- # 2. make physics model # ----------------- print("") print("making physics model") print("") # see which subgrid files already exist gs_str = "" if datamodel.n_subgrid > 1: gs_str = "sub*" sed_files = glob.glob("./{0}_beast/{0}_beast_seds.grid{1}.hd5".format( field_names[b], gs_str)) # only make the physics model they don't already exist if len(sed_files) < datamodel.n_subgrid: # directly create physics model grids #create_physicsmodel.create_physicsmodel(nprocs=1, nsubs=datamodel.n_subgrid) # create grids with script create_physicsmodel.split_create_physicsmodel( nprocs=1, nsubs=datamodel.n_subgrid) print('\n**** go run physics model code for ' + field_names[b] + '! ****') continue # list of SED files model_grid_files = sorted( glob.glob("./{0}_beast/{0}_beast_seds.grid{1}.hd5".format( field_names[b], gs_str))) # ----------------- # 3. make ASTs # ----------------- # only create an AST input list if the ASTs don't already exist ast_input_file = ("./" + field_names[b] + "_beast/" + field_names[b] + "_beast_inputAST.txt") if not os.path.isfile(ast_file): if not os.path.isfile(ast_input_file): print("") print("creating artificial stars") print("") make_ast_inputs.make_ast_inputs(flux_bin_method=True) split_ast_input_file.split_asts(field_names[b] + "_beast", ast_input_file, 2000) print("\n**** go run ASTs for " + field_names[b] + "! ****\n") continue # ----------------- # 4/5. edit photometry/AST catalogs # ----------------- # remove sources that are # - in regions without full imaging coverage, # - flagged in flag_filter print("") print("editing photometry/AST catalogs") print("") # - photometry gst_file_cut = gst_file.replace(".fits", "_with_sourceden_cut.fits") cut_catalogs.cut_catalogs( gst_file_sd, gst_file_cut, partial_overlap=True, flagged=True, flag_filter=flag_filter[b], region_file=True, ) # - ASTs ast_file_cut = ast_file.replace(".fits", "_cut.fits") cut_catalogs.cut_catalogs( ast_file, ast_file_cut, partial_overlap=True, flagged=True, flag_filter=flag_filter[b], region_file=True, ) # test = plot_mag_hist.plot_mag_hist(ast_file_cut, stars_per_bin=200, max_bins=30) # edit the datamodel.py file to have the correct photometry file name # (AST file name is already automatically the cut version) create_datamodel( gst_file_cut, ast_file_cut, gst_filter_names, beast_filter_names, dist_mod[b], velocity[b], ref_image=im_file, ) # ----------------- # 6. split observations by source density # ----------------- print("") print("splitting observations by source density") print("") # - photometry if len(glob.glob(gst_file_cut.replace('.fits', '*sub*fits'))) == 0: # a smaller value for n_per_file will mean more individual files/runs, # but each run will take a shorter amount of time split_catalog_using_map.split_main( gst_file_cut, ast_file_cut, gst_file.replace('.fits', '_sourceden_map.hd5'), bin_width=1, n_per_file=6250, ) # -- at this point, we can run the code to create lists of filenames file_dict = create_filenames.create_filenames( use_sd=True, nsubs=datamodel.n_subgrid) # figure out how many files there are sd_sub_info = file_dict["sd_sub_info"] # - number of SD bins temp = set([i[0] for i in sd_sub_info]) print("** total SD bins: " + str(len(temp))) # - the unique sets of SD+sub unique_sd_sub = [ x for i, x in enumerate(sd_sub_info) if i == sd_sub_info.index(x) ] print("** total SD subfiles: " + str(len(unique_sd_sub))) # ----------------- # 7. make noise models # ----------------- print("") print("making noise models") print("") # create the noise model (this code will check if it exists) create_obsmodel.create_obsmodel(use_sd=True, nsubs=datamodel.n_subgrid, nprocs=1) # ----------------- # 8. make script to trim models # ----------------- print("") print("setting up script to trim models") print("") # save any at-queue commands at_list = [] # iterate through each model grid for i in range(datamodel.n_subgrid): # gst list temp = file_dict["photometry_files"] gst_input_list = [ x for i, x in enumerate(temp) if i == temp.index(x) ] # create corresponding files for each of those ast_input_list = [] noise_files = [] trim_prefix = [] for j in range(len(gst_input_list)): # get the sd/sub number curr_sd = unique_sd_sub[j][0] curr_sub = unique_sd_sub[j][1] subfolder = "bin{0}_sub{1}".format(curr_sd, curr_sub) # create file names ast_input_list.append( ast_file_cut.replace(".fits", "_bin" + curr_sd + ".fits")) if datamodel.n_subgrid > 1: noise_files.append( "./{0}_beast/{0}_beast_noisemodel_bin{1}.gridsub{2}.hd5" .format(field_names[b], curr_sd, i)) trim_prefix.append( "./{0}_beast/{1}/{0}_beast_{1}_gridsub{2}".format( field_names[b], subfolder, i)) if datamodel.n_subgrid == 1: noise_files.append(file_dict['noise_files'][j]) trim_prefix.append("./{0}_beast/{0}_beast_{1}".format( field_names[b], subfolder)) # check if the trimmed grids exist before moving on if datamodel.n_subgrid > 1: trim_files = sorted( glob.glob( "./{0}_beast/bin*_sub*/{0}_beast_*_gridsub{1}_sed_trim.grid.hd5" .format(field_names[b], i))) if datamodel.n_subgrid == 1: trim_files = sorted( glob.glob("./{0}_beast/{0}_beast_*_sub*_sed_trim.grid.hd5". format(field_names[b]))) if len(trim_files) < len(gst_input_list): job_path = "./" + field_names[b] + "_beast/trim_batch_jobs/" if datamodel.n_subgrid > 1: file_prefix = "BEAST_gridsub" + str(i) if datamodel.n_subgrid == 1: file_prefix = "BEAST" # generate trimming at-queue commands setup_batch_beast_trim.generic_batch_trim( model_grid_files[i], noise_files, gst_input_list, ast_input_list, trim_prefix, job_path=job_path, file_prefix=file_prefix, num_subtrim=1, nice=19, prefix="source activate b13", ) at_list.append("at -f " + job_path + file_prefix + "_batch_trim.joblist now") if len(at_list) > 0: print("\n**** go run trimming code for " + field_names[b] + "! ****") print("Here are the command(s) to run:") for cmd in at_list: print(cmd) return else: print("all files are trimmed for " + field_names[b]) # ----------------- # 9. make script to fit models # ----------------- print("") print("setting up script to fit models") print("") fit_run_info = setup_batch_beast_fit.setup_batch_beast_fit( num_percore=1, nice=19, overwrite_logfile=False, prefix="source activate b13", use_sd=True, nsubs=datamodel.n_subgrid, nprocs=1, ) # check if the fits exist before moving on tot_remaining = len(fit_run_info["done"]) - np.sum( fit_run_info["done"]) if tot_remaining > 0: print("\n**** go run fitting code for " + field_names[b] + "! ****") print("Here are the " + str(len(fit_run_info["files_to_run"])) + " commands to run:") for job_file in fit_run_info["files_to_run"]: print("at -f ./" + job_file + " now") continue else: print("all fits are complete for " + field_names[b]) # ----------------- # 10. merge stats files from each fit # ----------------- print("") print("merging stats files") print("") merge_files.merge_files(use_sd=True, nsubs=datamodel.n_subgrid)
def beast_ast_inputs(field_name=None, ref_image=None, filter_ids=None, galaxy=None, supp=0): """ This does all of the steps for generating AST inputs and can be used a wrapper to automatically do most steps for multiple fields. * make field's beast_settings file * make source density map * make background density map * split catalog by source density * make physics model (SED grid) * make input list for ASTs * prune input ASTs ---- Inputs: field_name (str): name of field ref_image (str): path to reference image filter_ids (list): list of indexes corresponding to the filters in the observation, referenced to the master list below. galaxy (str): name of target galaxy (e.g., 'SMC', 'LMC') ---- Places for user to manually do things: * editing code before use - here: list the catalog filter names with the corresponding BEAST names - here: choose settings (pixel size, filter, mag range) for the source density map - here: choose settings (pixel size, reference image) for the background map """ # the list of fields field_names = [field_name] # the path+file for a reference image im_path = [ref_image] ref_filter = ["F475W"] # choose a filter to use for removing artifacts # (remove catalog sources with filter_FLAG > 99) flag_filter = ["F475W"] # number of fields n_field = len(field_names) # Need to know what the correspondence is between filter names in the # catalog and the BEAST filter names. # # These will be used to automatically determine the filters present in # each GST file and fill in the beast settings file. The order doesn't # matter, as long as the order in one list matches the order in the other # list. # gst_filter_names = [ "F225W", "F275W", "F336W", "F475W", "F814W", "F110W", "F160W", "F657N", ] beast_filter_names = [ "HST_WFC3_F225W", "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_WFC3_F475W", "HST_WFC3_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", "HST_WFC3_F657N", ] for b in range(n_field): print("********") print("field " + field_names[b]) print("********") # ----------------- # data file names # ----------------- # paths for the data/AST files gst_file = "./data/{0}/{0}.vgst.fits".format(field_names[b]) ast_input_file = "./{0}/{0}_inputAST.txt".format(field_names[b]) # if no galaxy is manually indicated, try to fetch from gst_file name if galaxy == None: print("no galaxy specified") print("fetching galaxy from field name") galaxy_attempt = field_names[b].split("_")[1].split("-")[0] print("is this the correct galaxy? : %s" % galaxy_attempt) # raw_input returns the empty string for "enter" yes = {'yes', 'y', 'ye', ''} no = {'no', 'n'} response = 0 while response == 0: choice = input().lower() if choice in yes: galaxy = galaxy_attempt response = 1 elif choice in no: print("please rerun with --galaxy specified") break else: sys.stdout.write("Please respond with 'yes' or 'no'") # path for the reference image (if using for the background map) im_file = im_path[b] # fetch filter ids gst_data = Table.read(gst_file) filter_cols = [c for c in gst_data.colnames if "VEGA" in c] # extract every filter mentioned in the table filters = [f.split("_")[0] for f in filter_cols] # match with the gst filter list filter_ids = [gst_filter_names.index(i) for i in filters] filter_ids.sort() gst_filter_names = [gst_filter_names[i] for i in filter_ids] beast_filter_names = [beast_filter_names[i] for i in filter_ids] print(beast_filter_names) # region file with catalog stars # make_region_file(gst_file, ref_filter[b]) # ----------------- # 0. make beast settings file # ----------------- print("") print("creating beast settings file") print("") beast_settings_filename = create_beast_settings(gst_file, ast_input_file, gst_filter_names, beast_filter_names, galaxy, ref_image=im_file, supp=supp) # load in beast settings to get number of subgrids settings = beast_settings.beast_settings( beast_settings_filename #"beast_settings_" + galaxy + "_asts_" + field_names[b] + ".txt" ) # ----------------- # 1a. make magnitude histograms # ----------------- print("") print("making magnitude histograms") print("") # if not os.path.isfile('./data/'+field_names[b]+'.gst_maghist.pdf'): peak_mags = plot_mag_hist.plot_mag_hist(gst_file, stars_per_bin=70, max_bins=75) # ----------------- # 1b. make a source density map # ----------------- print("") print("making source density map") print("") # not currently doing background density bins # use_bg_info = True use_bg_info = False if use_bg_info: background_args = types.SimpleNamespace( subcommand="background", catfile=gst_file, erode_boundary=settings.ast_erode_selection_region, pixsize=5, npix=None, reference=im_file, mask_radius=10, ann_width=20, cat_filter=[ref_filter, "90"], ) create_background_density_map.main_make_map(background_args) # but we are doing source density bins! if not os.path.isfile( gst_file.replace(".fits", "_source_den_image.fits")): print("No sd image file found") # - pixel size of 10 arcsec # - use ref_filter[b] between vega mags of 17 and peak_mags[ref_filter[b]]-0.5 sourceden_args = types.SimpleNamespace( subcommand="sourceden", catfile=gst_file, erode_boundary=settings.ast_erode_selection_region, pixsize=5, npix=None, mag_name=ref_filter[0] + "_VEGA", mag_cut=[17, peak_mags[ref_filter[0]] - 0.5], flag_name=flag_filter[0] + "_FLAG", ) create_background_density_map.main_make_map(sourceden_args) # new file name with the source density column gst_file_sd = gst_file.replace(".fits", "_with_sourceden.fits") # ----------------- # 2. make physics model # ----------------- print("") print("making physics model") print("") # see which subgrid files already exist gs_str = "" if settings.n_subgrid > 1: gs_str = "sub*" # try to fetch the list of SED files (physics models) model_grid_files = sorted( glob.glob("./{0}/{0}_seds.grid*.hd5".format(field_names[b], ))) # only make the physics model they don't already exist if len(model_grid_files) < settings.n_subgrid: # directly create physics model grids create_physicsmodel.create_physicsmodel(settings, nprocs=1, nsubs=settings.n_subgrid) # fetch the list of SED files again (physics models) model_grid_files = sorted( glob.glob("./{0}/{0}_seds.grid*.hd5".format(field_names[b], ))) # ------------------- # 3. make AST inputs # ------------------- print("") print("making AST inputs") print("") # only create an AST input list if the ASTs don't already exist if not os.path.isfile(ast_input_file): make_ast_inputs.make_ast_inputs(settings, pick_method="flux_bin_method") # compare magnitude histograms of ASTs with catalog plot_ast_histogram.plot_ast_histogram( ast_file=ast_input_file, sed_grid_file=model_grid_files[0]) if supp != 0: print("") print("making supplemental AST inputs") print("") ast_input_supp_file = "./{0}/{0}_inputAST_suppl.txt".format( field_names[b]) if not os.path.isfile(ast_input_supp_file): make_ast_inputs.make_ast_inputs(settings, pick_method="suppl_seds") print("now go check the diagnostic plots!")
args = parser.parse_args() # read in BEAST settings settings = beast_settings.beast_settings("beast_settings.txt") if args.physicsmodel: create_physicsmodel.create_physicsmodel( settings, nsubs=settings.n_subgrid, nprocs=1, ) if args.ast: make_ast_inputs.make_ast_inputs(settings, flux_bin_method=False) if args.observationmodel: print("Generating noise model from ASTs and absflux A matrix") create_obsmodel.create_obsmodel( settings, use_sd=False, nsubs=settings.n_subgrid, nprocs=1, use_rate=False, ) # in the absence of ASTs, the splinter noise model can be used # instead of the toothpick model above # **warning** not very realistic
def generate_files_for_tests(run_beast=True, run_tools=True): """ Use the metal_small example to generate a full set of files for the BEAST regression tests. Parameters ---------- run_beast : boolean (default=True) if True, run the BEAST run_tools : boolean (default=True) if True, run the code to generate things for tools """ # read in BEAST settings settings_orig = beast_settings.beast_settings("beast_settings.txt") # also make a version with subgrids settings_subgrids = copy.deepcopy(settings_orig) settings_subgrids.n_subgrid = 2 settings_subgrids.project = f"{settings_orig.project}_subgrids" # ========================================== # run the beast for each set of settings # ========================================== if run_beast: for settings in [settings_orig, settings_subgrids]: # ----------------- # physics model # ----------------- create_physicsmodel.create_physicsmodel( settings, nsubs=settings.n_subgrid, nprocs=1, ) # ----------------- # ASTs # ----------------- # currently only works for no subgrids if settings.n_subgrid == 1: make_ast_inputs.make_ast_inputs(settings, pick_method="flux_bin_method") # ----------------- # obs model # ----------------- create_obsmodel.create_obsmodel( settings, use_sd=False, nsubs=settings.n_subgrid, nprocs=1, use_rate=True, ) # ----------------- # trimming # ----------------- # make file names file_dict = create_filenames.create_filenames( settings, use_sd=False, nsubs=settings.n_subgrid) # read in the observed data obsdata = Observations(settings.obsfile, settings.filters, settings.obs_colnames) for i in range(settings.n_subgrid): # get the modesedgrid on which to generate the noisemodel modelsedgridfile = file_dict["modelsedgrid_files"][i] modelsedgrid = SEDGrid(modelsedgridfile) # read in the noise model just created noisemodel_vals = noisemodel.get_noisemodelcat( file_dict["noise_files"][i]) # trim the model sedgrid sed_trimname = file_dict["modelsedgrid_trim_files"][i] noisemodel_trimname = file_dict["noise_trim_files"][i] trim_grid.trim_models( modelsedgrid, noisemodel_vals, obsdata, sed_trimname, noisemodel_trimname, sigma_fac=3.0, ) # ----------------- # fitting # ----------------- run_fitting.run_fitting( settings, use_sd=False, nsubs=settings.n_subgrid, nprocs=1, pdf2d_param_list=["Av", "M_ini", "logT"], pdf_max_nbins=200, ) # ----------------- # merging # ----------------- # it'll automatically skip for no subgrids merge_files.merge_files(settings, use_sd=False, nsubs=settings.n_subgrid) print("\n\n") # ========================================== # reference files for assorted tools # ========================================== if run_tools: # ----------------- # compare_spec_type # ----------------- # the input settings input = { "spec_ra": [72.67213351], "spec_dec": [-67.71720515], "spec_type": ["A"], "spec_subtype": [0], "lumin_class": ["IV"], "match_radius": 0.2, } # run it output = compare_spec_type.compare_spec_type( settings_orig.obsfile, "{0}/{0}_stats.fits".format(settings_orig.project), **input, ) # save the inputs and outputs asdf.AsdfFile({ "input": input, "output": output }).write_to("{0}/{0}_compare_spec_type.asdf".format( settings_orig.project)) # ----------------- # star_type_probability # ----------------- # input settings input = { "output_filebase": None, "ext_O_star_params": { "min_M_ini": 10, "min_Av": 0.5, "max_Av": 5 }, } # run it output = star_type_probability.star_type_probability( "{0}/{0}_pdf1d.fits".format(settings_orig.project), "{0}/{0}_pdf2d.fits".format(settings_orig.project), **input, ) # save the inputs and outputs asdf.AsdfFile({ "input": input, "output": output }).write_to("{0}/{0}_star_type_probability.asdf".format( settings_orig.project)) # ========================================== # asdf file permissions # ========================================== # for unknown reasons, asdf currently writes files with permissions set # to -rw-------. This changes it to -rw-r--r-- (like the rest of the # BEAST files) so Karl can easily copy them over to the cached file # website. # list of asdf files asdf_files = glob.glob("*/*.asdf") # go through each one to change permissions for fname in asdf_files: os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
parser.add_argument("-r", "--resume", help="Resume a fitting run", action="store_true") args = parser.parse_args() # check input parameters, print what is the problem, stop run_beast verify_params.verify_input_format(datamodel) if args.physicsmodel: create_physicsmodel.create_physicsmodel(nsubs=1, nprocs=1) if args.ast: make_ast_inputs.make_ast_inputs(flux_bin_method=False) if args.observationmodel: print("Generating noise model from ASTs and absflux A matrix") create_obsmodel.create_obsmodel(use_sd=False, nsubs=1, nprocs=1, use_rate=False) # in the absence of ASTs, the splinter noise model can be used # instead of the toothpick model above # **warning** not very realistic # import beast.observationmodel.noisemodel.splinter as noisemodel # # modelsedgridfile = datamodel.project + '/' + datamodel.project + \
def beast_ast_inputs(field_name=None, ref_image=None, filter_ids=None, galaxy=None): """ This does all of the steps for generating AST inputs and can be used a wrapper to automatically do most steps for multiple fields. * make field's beast_settings file * make source density map * make background density map * split catalog by source density * make physics model (SED grid) * make input list for ASTs * prune input ASTs ---- Inputs: field_name (str): name of field ref_image (str): path to reference image filter_ids (list): list of indexes corresponding to the filters in the observation, referenced to the master list below. galaxy (str): name of target galaxy (e.g., 'SMC', 'LMC') ---- Places for user to manually do things: * editing code before use - here: list the catalog filter names with the corresponding BEAST names - here: choose settings (pixel size, filter, mag range) for the source density map - here: choose settings (pixel size, reference image) for the background map """ # the list of fields field_names = [field_name] # the path+file for a reference image im_path = [ref_image] ref_filter = ["F475W"] # choose a filter to use for removing artifacts # (remove catalog sources with filter_FLAG > 99) flag_filter = ["F475W"] # number of fields n_field = len(field_names) # Need to know what the correspondence is between filter names in the # catalog and the BEAST filter names. # # These will be used to automatically determine the filters present in # each GST file and fill in the beast settings file. The order doesn't # matter, as long as the order in one list matches the order in the other # list. # gst_filter_names = [ "F225W", "F275W", "F336W", "F475W", "F814W", "F110W", "F160W", "F657N", ] beast_filter_names = [ "HST_WFC3_F225W", "HST_WFC3_F275W", "HST_WFC3_F336W", "HST_WFC3_F475W", "HST_WFC3_F814W", "HST_WFC3_F110W", "HST_WFC3_F160W", "HST_WFC3_F657N", ] filter_ids = [int(i) for i in filter_ids] gst_filter_names = [gst_filter_names[i] for i in filter_ids] beast_filter_names = [beast_filter_names[i] for i in filter_ids] for b in range(n_field): print("********") print("field " + field_names[b]) print("********") # ----------------- # data file names # ----------------- # paths for the data/AST files gst_file = "./data/" + field_names[b] + ".st.fits" ast_file = "./data/" + field_names[b] + ".st.fake.fits" # path for the reference image (if using for the background map) im_file = im_path[b] # region file with catalog stars # make_region_file(gst_file, ref_filter[b]) # ----------------- # 0. make beast settings file # ----------------- print("") print("creating beast settings file") print("") create_beast_settings( gst_file, ast_file, gst_filter_names, beast_filter_names, galaxy, ref_image=im_file, ) # load in beast settings to get number of subgrids settings = beast_settings.beast_settings( "beast_settings_" + galaxy + "_asts_" + field_names[b] + ".txt" ) # ----------------- # 1a. make magnitude histograms # ----------------- print("") print("making magnitude histograms") print("") # if not os.path.isfile('./data/'+field_names[b]+'.gst_maghist.pdf'): peak_mags = plot_mag_hist.plot_mag_hist(gst_file, stars_per_bin=70, max_bins=75) # ----------------- # 1b. make a source density map # ----------------- print("") print("making source density map") print("") # not currently doing background density bins # use_bg_info = True use_bg_info = False if use_bg_info: background_args = types.SimpleNamespace( subcommand="background", catfile=gst_file, pixsize=5, npix=None, reference=im_file, mask_radius=10, ann_width=20, cat_filter=[ref_filter, "90"], ) create_background_density_map.main_make_map(background_args) # but we are doing source density bins! if not os.path.isfile(gst_file.replace(".fits", "_source_den_image.fits")): # - pixel size of 10 arcsec # - use ref_filter[b] between vega mags of 17 and peak_mags[ref_filter[b]]-0.5 sourceden_args = types.SimpleNamespace( subcommand="sourceden", catfile=gst_file, pixsize=5, npix=None, mag_name=ref_filter[0] + "_VEGA", mag_cut=[17, peak_mags[ref_filter[0]] - 0.5], flag_name=flag_filter[0] + "_FLAG", ) create_background_density_map.main_make_map(sourceden_args) # new file name with the source density column gst_file_sd = gst_file.replace(".fits", "_with_sourceden.fits") # ----------------- # 2. make physics model # ----------------- print("") print("making physics model") print("") # see which subgrid files already exist gs_str = "" if settings.n_subgrid > 1: gs_str = "sub*" sed_files = glob.glob( "./{0}_beast/{0}_beast_seds.grid{1}.hd5".format(field_names[b], gs_str) ) # only make the physics model they don't already exist if len(sed_files) < settings.n_subgrid: # directly create physics model grids create_physicsmodel.create_physicsmodel( settings, nprocs=1, nsubs=settings.n_subgrid ) # ------------------- # 3. make AST inputs # ------------------- # only create an AST input list if the ASTs don't already exist ast_input_file = "./" + field_names[b] + "/" + field_names[b] + "_inputAST.txt" if not os.path.isfile(ast_input_file): make_ast_inputs.make_ast_inputs(settings, pick_method="flux_bin_method") # list of SED files (physics models) model_grid_files = sorted( glob.glob( "./{0}/{0}_seds.grid*.hd5".format( field_names[b], ) ) ) # -------------------- # 3.1 "prune" AST inputs # -------------------- # prune input AST by flux (empirically determined) ast_input_tab = Table.read(ast_input_file, format="ascii") ast_input_tab_pruned = ast_input_tab.copy() if "F336W" in gst_filter_names: prune_spots = ( (ast_input_tab_pruned["HST_WFC3_F336W"] > 30.5) & (ast_input_tab_pruned["HST_WFC3_F475W"] > 32.5) & (ast_input_tab_pruned["HST_WFC3_F814W"] > 29.0) ) else: prune_spots = (ast_input_tab_pruned["HST_WFC3_F475W"] > 32.5) & ( ast_input_tab_pruned["HST_WFC3_F814W"] > 29.0 ) ast_input_tab_pruned = ast_input_tab_pruned[~prune_spots] # write pruned ast input table to a txt file ast_input_file_pruned = ( "./" + field_name + "/" + field_name + "_inputAST_pruned.txt" ) ast_input_tab_pruned.write( ast_input_file_pruned, format="ascii", overwrite=True ) # print out number of pruned ASTs per source density bin as a sanity check print("pruned input AST statistics per bin") input_ast_bin_stats(settings, ast_input_file_pruned, field_names[b]) # compare magnitude histograms of pruned ASTs with catalog plot_ast_histogram.plot_ast_histogram( ast_file=ast_input_file_pruned, sed_grid_file=model_grid_files[0] ) print("now go check the diagnostic plots!")