def test_select_magnitude_column(): sim = catalog_seed_image.Catalog_seed() instrument_list = ['nircam', 'nircam', 'nircam', 'nircam', 'nircam'] cat_filter_list = [ 'f090w', 'F322W2/F323N', 'F115W/WLP8', 'WLM8/F090W', 'F405N' ] param_filter_list = ['f090w', 'F322W2', 'F115W', 'F090W', 'F444W', 'F444W'] pupil_list = ['clear', 'F323N', 'WLP8', 'WLM8', 'F405N', 'F470N'] truth_list = [ 'nircam_f090w_clear_magnitude', 'nircam_f322w2_f323n_magnitude', 'nircam_f115w_wlp8_magnitude', 'nircam_f090w_wlm8_magnitude', 'nircam_f444w_f405n_magnitude', 'nircam_f470n_magnitude' ] catalog = PointSourceCatalog(ra=[0.], dec=[0.]) for inst, filt in zip(instrument_list, cat_filter_list): catalog.add_magnitude_column([15.], instrument=inst, filter_name=filt) # Add an old-style magnitude column to check for backwards compatibility catalog.add_magnitude_column([15.], instrument='nircam', filter_name='F470N') for inst, filt, pup, truth in zip(instrument_list, param_filter_list, pupil_list, truth_list): sim.params = {} sim.params['Inst'] = {} sim.params['Readout'] = {} sim.params['Inst']['instrument'] = inst sim.params['Readout']['pupil'] = pup sim.params['Readout']['filter'] = filt col = sim.select_magnitude_column(catalog.table, 'junk.cat') assert col == truth
def test_overlap_coordinates_full_frame(): """Test the function that calculates the coordinates for the overlap between two stamp images """ # Create instance of Catalog_seed seed = catalog_seed_image.Catalog_seed(offline=True) # Create a point sources table tab = Table() tab['index'] = np.arange(10).astype(np.int) + 1 tab['pixelx'] = [ 1000, 50, 1000, 1900, -0.5, 1033.587, 622.63, 1523.75, 1013.413, 1124.371 ] tab['pixely'] = [ 1000, 400, 25, 2000, -0.5, 1023.425, 1024.223, 1013.25, 223.575, 1822.72 ] tab['RA_degrees'] = [ 12.008729, 11.999914, 12.003422, 11.995729, 12.000036, 11.99918, 12.008729, 11.999914, 12.003422, 11.995729 ] tab['Dec_degrees'] = [ -0.00885006, 1.7231344e-09, -1.5512744e-05, -4.9949034e-05, -0.006866415, 0.0068373946, -0.00885006, 1.7231344e-09, -1.5512744e-05, -4.9949034e-05 ] tab['magnitude'] = np.repeat(14, 10) tab['countrate_e/s'] = np.repeat(161630.72, 10) tab['counts_per_frame_e'] = np.repeat(1735391.9, 10) expected_results = [] expected_results.append((850, 1151, 850, 1151, 0, 301, 0, 301)) expected_results.append((0, 201, 250, 551, 100, 301, 0, 301)) expected_results.append((850, 1151, 0, 176, 0, 301, 125, 301)) expected_results.append((1750, 2048, 1850, 2048, 0, 298, 0, 198)) stamp_dims = (301, 301) stamp_x = 150 stamp_y = 150 # Check the basic function aperture_dims = (2048, 2048) for index in range(4): results = seed.cropped_coords(tab[index]['pixelx'], tab[index]['pixely'], aperture_dims, stamp_x, stamp_y, stamp_dims, ignore_detector=False) assert results == expected_results[index] # Check the function that wraps around the most basic function seed.ffsize = 2048 seed.subarray_bounds = [0, 0, 2047, 2047] for index in range(4): results = seed.create_psf_stamp_coords(tab[index]['pixelx'], tab[index]['pixely'], stamp_dims, stamp_x, stamp_y, coord_sys='full_frame', ignore_detector=False) ijkl = [] for ele in results[4:]: ijkl.extend(list(ele)) ijkl = tuple(ijkl) assert ijkl == expected_results[index] # Create dummy PSF library seed.psf_library = create_dummy_psf_grid() seed.psf_library_x_dim = seed.psf_library.data.shape[ 2] / seed.psf_library.oversampling seed.psf_library_y_dim = seed.psf_library.data.shape[ 1] / seed.psf_library.oversampling # Check the wrapped function, but call using 'aperture' rather than # 'full_frame' expected_k1l1 = [(850, 1151, 850, 1151, 0, 301, 0, 301), (0, 201, 250, 551, 0, 201, 0, 301), (850, 1151, 0, 176, 0, 301, 0, 176), (1750, 2048, 1850, 2048, 0, 298, 0, 198), (0, 150, 0, 150, 0, 150, 0, 150)] seed.coord_adjust['xoffset'] = 0 # Already defined, but let's be explicit seed.coord_adjust['yoffset'] = 0 # Already defined, but let's be explicit seed.output_dims = [2048, 2048] seed.psf_library_core_x_dim = 301 seed.psf_library_core_y_dim = 301 seed.params = {'simSignals': {}} seed.add_psf_wings = False seed.psf_library_oversamp = 1 for index in range(5): psf, min_x, min_y, add_wings = seed.create_psf_stamp( tab[index]['pixelx'], tab[index]['pixely'], stamp_dims[1], stamp_dims[0], ignore_detector=False) stamp_x_loc = 301 // 2 - min_x stamp_y_loc = 301 // 2 - min_y updated_psf_dimensions = psf.shape xap, yap, xpts, ypts, (i1, i2), (j1, j2), (k1, k2), \ (l1, l2) = seed.create_psf_stamp_coords(tab[index]['pixelx'], tab[index]['pixely'], updated_psf_dimensions, stamp_x_loc, stamp_y_loc, coord_sys='aperture') assert (i1, i2, j1, j2, k1, k2, l1, l2) == expected_k1l1[index]
# <br></br> # Dark current preparation # <br></br> # Observation creation (combining simulated sources and dark current) # <br></br><br></br> # This section shows how to call the three steps independently. The `imaging_simulator` function above is a wrapper around these three steps. Most users will want simply to call `imaging_simulator`. # ## First generate the "seed image" # # This is generally a 2D noiseless countrate image that contains only simulated astronomical sources. # # A seed image is generated based on a `.yaml` file that contains all the necessary parameters for simulating data. For this exercise, use the same yaml file that was used in the [Create Simulated Data](#run_steps_together) section as input. # In[ ]: cat = catalog_seed_image.Catalog_seed() cat.paramfile = yamlfile cat.make_seed() # ### Look at the seed image # In[ ]: show(cat.seedimage, 'Seed Image', max=20) # ## Prepare the dark current exposure # This will serve as the base of the simulated data. # This step will linearize the dark current (if it # is not already), and reorganize it into the # requested readout pattern and number of groups.
def test_overlap_coordinates_subarray(): """Test the function that calculates the coordinates for the overlap between two stamp images """ seed = catalog_seed_image.Catalog_seed(offline=True) # Create a point sources table tab = Table() tab['index'] = np.arange(5).astype(np.int) + 1 tab['pixelx'] = [200, 50, 200, 350, -0.5] tab['pixely'] = [200, 200, 25, 350, -0.5] tab['RA_degrees'] = [12.008729, 11.999914, 12.003422, 11.995729, 11.995729] tab['Dec_degrees'] = [ -0.00885006, 1.7231344e-09, -1.5512744e-05, -4.9949034e-05, -4.9949034e-05 ] tab['magnitude'] = np.repeat(14, 5) tab['countrate_e/s'] = np.repeat(161630.72, 5) tab['counts_per_frame_e'] = np.repeat(1735391.9, 5) subarray_expected_results = [] subarray_expected_results.append((50, 351, 50, 351, 0, 301, 0, 301)) subarray_expected_results.append((0, 201, 50, 351, 100, 301, 0, 301)) subarray_expected_results.append((50, 351, 0, 176, 0, 301, 125, 301)) subarray_expected_results.append((200, 400, 200, 400, 0, 200, 0, 200)) subarray_expected_results.append((0, 150, 0, 150, 151, 301, 151, 301)) # Test the basic function using a subarray stamp_dims = (301, 301) stamp_x = 150 stamp_y = 150 aperture_dims = (400, 400) # Assume 400x400 pixel subarray for index in range(5): results = seed.cropped_coords(tab[index]['pixelx'], tab[index]['pixely'], aperture_dims, stamp_x, stamp_y, stamp_dims, ignore_detector=False) assert results == subarray_expected_results[index] # Test the wrapper function seed.coord_adjust['xoffset'] = 0 # Already defined, but let's be explicit seed.coord_adjust['yoffset'] = 0 # Already defined, but let's be explicit seed.output_dims = [400, 400] # Assume 400x400 pixel subarray seed.subarray_bounds = [1648, 1648, 2047, 2047] seed.ffsize = 2048 seed.psf_library_core_x_dim = 301 seed.psf_library_core_y_dim = 301 seed.params = {'simSignals': {}} seed.add_psf_wings = False seed.psf_library_oversamp = 1 for index in range(5): results = seed.create_psf_stamp_coords(tab[index]['pixelx'], tab[index]['pixely'], stamp_dims, stamp_x, stamp_y, coord_sys='aperture', ignore_detector=False) ijkl = [] for ele in results[4:]: ijkl.extend(list(ele)) ijkl = tuple(ijkl) assert ijkl == subarray_expected_results[index] # Dummy PSF library seed.psf_library = create_dummy_psf_grid() seed.psf_library_x_dim = seed.psf_library.data.shape[ 2] / seed.psf_library.oversampling seed.psf_library_y_dim = seed.psf_library.data.shape[ 1] / seed.psf_library.oversampling # Test the wrapper function using a modified PSF stamp image shape expected_k1l1 = [(50, 351, 50, 351, 0, 301, 0, 301), (0, 201, 50, 351, 100, 301, 0, 301), (50, 351, 0, 176, 0, 301, 125, 301), (200, 400, 200, 400, 0, 200, 0, 200), (0, 150, 0, 150, 151, 301, 151, 301)] for index in range(5): psf, min_x, min_y, add_wings = seed.create_psf_stamp( tab[index]['pixelx'], tab[index]['pixely'], stamp_dims[1], stamp_dims[0], ignore_detector=False) stamp_x_loc = 301 // 2 - min_x stamp_y_loc = 301 // 2 - min_y updated_psf_dimensions = psf.shape xap, yap, xpts, ypts, (i1, i2), (j1, j2), (k1, k2), \ (l1, l2) = seed.create_psf_stamp_coords(tab[index]['pixelx'], tab[index]['pixely'], updated_psf_dimensions, stamp_x_loc, stamp_y_loc, coord_sys='aperture') assert (i1, i2, j1, j2, k1, k2, l1, l2) == expected_k1l1[index]
def create(self): """MAIN FUNCTION""" # Get parameters necessary to create the TSO data orig_parameters = self.get_param_info() subarray_table = utils.read_subarray_definition_file( orig_parameters['Reffiles']['subarray_defs']) orig_parameters = utils.get_subarray_info(orig_parameters, subarray_table) orig_parameters = utils.read_pattern_check(orig_parameters) self.basename = os.path.join( orig_parameters['Output']['directory'], orig_parameters['Output']['file'][0:-5].split('/')[-1]) # Determine file splitting information. First get some basic info # on the exposure self.numints = orig_parameters['Readout']['nint'] self.numgroups = orig_parameters['Readout']['ngroup'] self.numframes = orig_parameters['Readout']['nframe'] self.numskips = orig_parameters['Readout']['nskip'] self.namps = orig_parameters['Readout']['namp'] self.numresets = orig_parameters['Readout']['resets_bet_ints'] self.frames_per_group, self.frames_per_int, self.total_frames = utils.get_frame_count_info( self.numints, self.numgroups, self.numframes, self.numskips, self.numresets) # Get gain map for later unit conversion #gainfile = orig_parameters['Reffiles']['gain'] #gain, gainheader = file_io.read_gain_file(gainfile) # Make 2 copies of the input parameter file, separating the TSO # source from the other sources self.split_param_file(orig_parameters) print('Splitting background and TSO source into multiple yaml files') print('Running background sources through catalog_seed_image') print('background param file is:', self.background_paramfile) # Run the catalog_seed_generator on the non-TSO (background) sources background_direct = catalog_seed_image.Catalog_seed() background_direct.paramfile = self.background_paramfile background_direct.make_seed() background_segmentation_map = background_direct.seed_segmap # Stellar spectrum hdf5 file will be required, so no need to create one here. # Create hdf5 file with spectra of all sources if requested self.final_SED_file = spectra_from_catalog.make_all_spectra( self.catalog_files, input_spectra_file=self.SED_file, extrapolate_SED=self.extrapolate_SED, output_filename=self.final_SED_file, normalizing_mag_column=self.SED_normalizing_catalog_column) bkgd_waves, bkgd_fluxes = backgrounds.nircam_background_spectrum( orig_parameters, self.detector, self.module) # Run the disperser on the background sources. Add the background # signal here as well print('\n\nDispersing background sources\n\n') background_done = False background_seed_files = [ background_direct.ptsrc_seed_filename, background_direct.galaxy_seed_filename, background_direct.extended_seed_filename ] for seed_file in background_seed_files: if seed_file is not None: print("Dispersing seed image:", seed_file) disp = self.run_disperser(seed_file, orders=self.orders, add_background=not background_done, background_waves=bkgd_waves, background_fluxes=bkgd_fluxes, finalize=True) if not background_done: # Background is added at the first opportunity. At this # point, create an array to hold the final combined # dispersed background background_done = True background_dispersed = copy.deepcopy(disp.final) else: background_dispersed += disp.final # Run the catalog_seed_generator on the TSO source tso_direct = catalog_seed_image.Catalog_seed() tso_direct.paramfile = self.tso_paramfile tso_direct.make_seed() tso_segmentation_map = tso_direct.seed_segmap outside_tso_source = tso_segmentation_map == 0 tso_segmentation_map[outside_tso_source] = background_segmentation_map[ outside_tso_source] # Dimensions are (y, x) self.seed_dimensions = tso_direct.nominal_dims # Read in the transmission spectrum that goes with the TSO source tso_params = utils.read_yaml(self.tso_paramfile) tso_catalog_file = tso_params['simSignals']['tso_grism_catalog'] tso_catalog = ascii.read(tso_catalog_file) transmission_file = tso_catalog['Transmission_spectrum'].data transmission_spectrum = ascii.read(transmission_file[0]) # Calculate the total exposure time, including resets, to check # against the times provided in the catalog file. total_exposure_time = self.calculate_exposure_time() * u.second # Check to be sure the start and end times provided in the catalog # are enough to cover the length of the exposure. tso_catalog = self.tso_catalog_check(tso_catalog, total_exposure_time) # Use batman to create lightcurves from the transmission spectrum lightcurves, times = self.make_lightcurves(tso_catalog, self.frametime, transmission_spectrum) # Determine which frames of the exposure will take place with the unaltered stellar # spectrum. This will be all frames where the associated lightcurve is 1.0 everywhere. transit_frames, unaltered_frames = self.find_transit_frames( lightcurves) print('Frame numbers containing the transit: {} - {}'.format( np.min(transit_frames), np.max(transit_frames))) # Run the disperser using the original, unaltered stellar spectrum. Set 'cache=True' print('\n\nDispersing TSO source\n\n') grism_seed_object = self.run_disperser(tso_direct.seed_file, orders=self.orders, add_background=False, cache=True, finalize=True) # Crop dispersed seed images to correct final subarray size #no_transit_signal = grism_seed_object.final no_transit_signal = utils.crop_to_subarray(grism_seed_object.final, tso_direct.subarray_bounds) background_dispersed = utils.crop_to_subarray( background_dispersed, tso_direct.subarray_bounds) # Mulitp[ly the dispersed seed images by the flat field no_transit_signal *= tso_direct.flatfield background_dispersed *= tso_direct.flatfield # Save the dispersed seed images if requested if self.save_dispersed_seed: h_back = fits.PrimaryHDU(background_dispersed) h_back.header['EXTNAME'] = 'BACKGROUND_SOURCES' h_tso = fits.ImageHDU(grism_seed_object.final) h_tso.header['EXTNAME'] = 'TSO_SOURCE' hlist = fits.HDUList([h_back, h_tso]) disp_filename = '{}_dispersed_seed_images.fits'.format( self.basename) hlist.writeto(disp_filename, overwrite=True) print( '\nDispersed seed images (background sources and TSO source) saved to {}.\n\n' .format(disp_filename)) # Calculate file splitting info self.file_splitting() # Prepare for creating output files segment_file_dir = orig_parameters['Output']['directory'] if orig_parameters['Readout']['pupil'][0].upper() == 'F': usefilt = 'pupil' else: usefilt = 'filter' segment_file_base = orig_parameters['Output']['file'].replace( '.fits', '_') segment_file_base = '{}_{}_'.format( segment_file_base, orig_parameters['Readout'][usefilt]) segment_file_base = os.path.join(segment_file_dir, segment_file_base) # Loop over frames and integrations up to the size of the segment # file. ints_per_segment = self.int_segment_indexes[ 1:] - self.int_segment_indexes[:-1] groups_per_segment = self.grp_segment_indexes[ 1:] - self.grp_segment_indexes[:-1] total_frame_counter = 0 previous_segment = 1 segment_part_number = 0 segment_starting_int_number = 0 self.segment_part_number = 0 self.segment_ints = 0 self.segment_frames = 0 self.segment_frame_start_number = 0 self.segment_int_start_number = 0 self.part_int_start_number = 0 self.part_frame_start_number = 0 # Get split files' metadata split_meta = SplitFileMetaData(self.int_segment_indexes, self.grp_segment_indexes, self.file_segment_indexes, self.group_segment_indexes_g, self.frames_per_int, self.frames_per_group, self.frametime) # List of all output seed files self.seed_files = [] counter = 0 for i, int_dim in enumerate(ints_per_segment): int_start = self.int_segment_indexes[i] int_end = self.int_segment_indexes[i + 1] for j, grp_dim in enumerate(groups_per_segment): initial_frame = self.grp_segment_indexes[j] # int_dim and grp_dim are the number of integrations and # groups in the current segment PART print( "\n\nCurrent segment part contains: {} integrations and {} groups." .format(int_dim, grp_dim)) print("Creating frame by frame dispersed signal") segment_seed = np.zeros( (int_dim, grp_dim, self.seed_dimensions[0], self.seed_dimensions[1])) for integ in np.arange(int_dim): overall_integration_number = int_start + integ previous_frame = np.zeros(self.seed_dimensions) for frame in np.arange(grp_dim): #print('TOTAL FRAME COUNTER: ', total_frame_counter) #print('integ and frame: ', integ, frame) # If a frame is from the part of the lightcurve # with no transit, then the signal in the frame # comes from no_transit_signal if total_frame_counter in unaltered_frames: frame_only_signal = ( background_dispersed + no_transit_signal) * self.frametime # If the frame is from a part of the lightcurve # where the transit is happening, then call the # cached disperser with the appropriate lightcurve elif total_frame_counter in transit_frames: #print("{} is during the transit".format(total_frame_counter)) frame_transmission = lightcurves[ total_frame_counter, :] trans_interp = interp1d( transmission_spectrum['Wavelength'], frame_transmission) for order in self.orders: grism_seed_object.this_one[ order].disperse_all_from_cache( trans_interp) # Here is where we call finalize on the TSO object # This will update grism_seed_object.final to # contain the correct signal grism_seed_object.finalize(Back=None, BackLevel=None) cropped_grism_seed_object = utils.crop_to_subarray( grism_seed_object.final, tso_direct.subarray_bounds) frame_only_signal = ( background_dispersed + cropped_grism_seed_object) * self.frametime # Now add the signal from this frame to that in the # previous frame in order to arrive at the total # cumulative signal segment_seed[ integ, frame, :, :] = previous_frame + frame_only_signal previous_frame = copy.deepcopy( segment_seed[integ, frame, :, :]) total_frame_counter += 1 # At the end of each integration, increment the # total_frame_counter by the number of resets between # integrations total_frame_counter += self.numresets # Use the split files' metadata self.segment_number = split_meta.segment_number[counter] self.segment_ints = split_meta.segment_ints[counter] self.segment_frames = split_meta.segment_frames[counter] self.segment_part_number = split_meta.segment_part_number[ counter] self.segment_frame_start_number = split_meta.segment_frame_start_number[ counter] self.segment_int_start_number = split_meta.segment_int_start_number[ counter] self.part_int_start_number = split_meta.part_int_start_number[ counter] self.part_frame_start_number = split_meta.part_frame_start_number[ counter] counter += 1 print('Overall integration number: ', overall_integration_number) segment_file_name = '{}seg{}_part{}_seed_image.fits'.format( segment_file_base, str(self.segment_number).zfill(3), str(self.segment_part_number).zfill(3)) print('Segment int and frame start numbers: {} {}'.format( self.segment_int_start_number, self.segment_frame_start_number)) #print('Part int and frame start numbers (ints and frames within the segment): {} {}'.format(self.part_int_start_number, self.part_frame_start_number)) # Disperser output is always full frame. Crop to the # requested subarray if necessary if orig_parameters['Readout'][ 'array_name'] not in self.fullframe_apertures: print("Dispersed seed image size: {}".format( segment_seed.shape)) segment_seed = utils.crop_to_subarray( segment_seed, tso_direct.subarray_bounds) #gain = utils.crop_to_subarray(gain, tso_direct.subarray_bounds) # Segmentation map will be centered in a frame that is larger # than full frame by a factor of sqrt(2), so crop appropriately print('Cropping segmentation map to appropriate aperture') segy, segx = tso_segmentation_map.shape dx = int((segx - tso_direct.nominal_dims[1]) / 2) dy = int((segy - tso_direct.nominal_dims[0]) / 2) segbounds = [ tso_direct.subarray_bounds[0] + dx, tso_direct.subarray_bounds[1] + dy, tso_direct.subarray_bounds[2] + dx, tso_direct.subarray_bounds[3] + dy ] tso_segmentation_map = utils.crop_to_subarray( tso_segmentation_map, segbounds) # Convert seed image to ADU/sec to be consistent # with other simulator outputs gain = MEAN_GAIN_VALUES['nircam']['lw{}'.format( self.module.lower())] segment_seed /= gain # Update seed image header to reflect the # division by the gain tso_direct.seedinfo['units'] = 'ADU/sec' # Save the seed image. Save in units of ADU/sec print('Saving seed image') tso_seed_header = fits.getheader(tso_direct.seed_file) self.save_seed(segment_seed, tso_segmentation_map, tso_seed_header, orig_parameters) #, #segment_number, segment_part_number) # Prepare dark current exposure if # needed. print('Running dark prep') d = dark_prep.DarkPrep() d.paramfile = self.paramfile d.prepare() # Combine into final observation print('Running observation generator') obs = obs_generator.Observation() obs.linDark = d.dark_files obs.seed = self.seed_files obs.segmap = tso_segmentation_map obs.seedheader = tso_direct.seedinfo obs.paramfile = self.paramfile obs.create()