def test_select_magnitude_column():
sim = catalog_seed_image.Catalog_seed()
instrument_list = ['nircam', 'nircam', 'nircam', 'nircam', 'nircam']
cat_filter_list = [
'f090w', 'F322W2/F323N', 'F115W/WLP8', 'WLM8/F090W', 'F405N'
]
param_filter_list = ['f090w', 'F322W2', 'F115W', 'F090W', 'F444W', 'F444W']
pupil_list = ['clear', 'F323N', 'WLP8', 'WLM8', 'F405N', 'F470N']
truth_list = [
'nircam_f090w_clear_magnitude', 'nircam_f322w2_f323n_magnitude',
'nircam_f115w_wlp8_magnitude', 'nircam_f090w_wlm8_magnitude',
'nircam_f444w_f405n_magnitude', 'nircam_f470n_magnitude'
]
catalog = PointSourceCatalog(ra=[0.], dec=[0.])
for inst, filt in zip(instrument_list, cat_filter_list):
catalog.add_magnitude_column([15.], instrument=inst, filter_name=filt)
# Add an old-style magnitude column to check for backwards compatibility
catalog.add_magnitude_column([15.],
instrument='nircam',
filter_name='F470N')
for inst, filt, pup, truth in zip(instrument_list, param_filter_list,
pupil_list, truth_list):
sim.params = {}
sim.params['Inst'] = {}
sim.params['Readout'] = {}
sim.params['Inst']['instrument'] = inst
sim.params['Readout']['pupil'] = pup
sim.params['Readout']['filter'] = filt
col = sim.select_magnitude_column(catalog.table, 'junk.cat')
assert col == truth
def test_overlap_coordinates_full_frame():
"""Test the function that calculates the coordinates for the
overlap between two stamp images
"""
# Create instance of Catalog_seed
seed = catalog_seed_image.Catalog_seed(offline=True)
# Create a point sources table
tab = Table()
tab['index'] = np.arange(10).astype(np.int) + 1
tab['pixelx'] = [
1000, 50, 1000, 1900, -0.5, 1033.587, 622.63, 1523.75, 1013.413,
1124.371
]
tab['pixely'] = [
1000, 400, 25, 2000, -0.5, 1023.425, 1024.223, 1013.25, 223.575,
1822.72
]
tab['RA_degrees'] = [
12.008729, 11.999914, 12.003422, 11.995729, 12.000036, 11.99918,
12.008729, 11.999914, 12.003422, 11.995729
]
tab['Dec_degrees'] = [
-0.00885006, 1.7231344e-09, -1.5512744e-05, -4.9949034e-05,
-0.006866415, 0.0068373946, -0.00885006, 1.7231344e-09, -1.5512744e-05,
-4.9949034e-05
]
tab['magnitude'] = np.repeat(14, 10)
tab['countrate_e/s'] = np.repeat(161630.72, 10)
tab['counts_per_frame_e'] = np.repeat(1735391.9, 10)
expected_results = []
expected_results.append((850, 1151, 850, 1151, 0, 301, 0, 301))
expected_results.append((0, 201, 250, 551, 100, 301, 0, 301))
expected_results.append((850, 1151, 0, 176, 0, 301, 125, 301))
expected_results.append((1750, 2048, 1850, 2048, 0, 298, 0, 198))
stamp_dims = (301, 301)
stamp_x = 150
stamp_y = 150
# Check the basic function
aperture_dims = (2048, 2048)
for index in range(4):
results = seed.cropped_coords(tab[index]['pixelx'],
tab[index]['pixely'],
aperture_dims,
stamp_x,
stamp_y,
stamp_dims,
ignore_detector=False)
assert results == expected_results[index]
# Check the function that wraps around the most basic function
seed.ffsize = 2048
seed.subarray_bounds = [0, 0, 2047, 2047]
for index in range(4):
results = seed.create_psf_stamp_coords(tab[index]['pixelx'],
tab[index]['pixely'],
stamp_dims,
stamp_x,
stamp_y,
coord_sys='full_frame',
ignore_detector=False)
ijkl = []
for ele in results[4:]:
ijkl.extend(list(ele))
ijkl = tuple(ijkl)
assert ijkl == expected_results[index]
# Create dummy PSF library
seed.psf_library = create_dummy_psf_grid()
seed.psf_library_x_dim = seed.psf_library.data.shape[
2] / seed.psf_library.oversampling
seed.psf_library_y_dim = seed.psf_library.data.shape[
1] / seed.psf_library.oversampling
# Check the wrapped function, but call using 'aperture' rather than
# 'full_frame'
expected_k1l1 = [(850, 1151, 850, 1151, 0, 301, 0, 301),
(0, 201, 250, 551, 0, 201, 0, 301),
(850, 1151, 0, 176, 0, 301, 0, 176),
(1750, 2048, 1850, 2048, 0, 298, 0, 198),
(0, 150, 0, 150, 0, 150, 0, 150)]
seed.coord_adjust['xoffset'] = 0 # Already defined, but let's be explicit
seed.coord_adjust['yoffset'] = 0 # Already defined, but let's be explicit
seed.output_dims = [2048, 2048]
seed.psf_library_core_x_dim = 301
seed.psf_library_core_y_dim = 301
seed.params = {'simSignals': {}}
seed.add_psf_wings = False
seed.psf_library_oversamp = 1
for index in range(5):
psf, min_x, min_y, add_wings = seed.create_psf_stamp(
tab[index]['pixelx'],
tab[index]['pixely'],
stamp_dims[1],
stamp_dims[0],
ignore_detector=False)
stamp_x_loc = 301 // 2 - min_x
stamp_y_loc = 301 // 2 - min_y
updated_psf_dimensions = psf.shape
xap, yap, xpts, ypts, (i1, i2), (j1, j2), (k1, k2), \
(l1, l2) = seed.create_psf_stamp_coords(tab[index]['pixelx'], tab[index]['pixely'],
updated_psf_dimensions, stamp_x_loc, stamp_y_loc,
coord_sys='aperture')
assert (i1, i2, j1, j2, k1, k2, l1, l2) == expected_k1l1[index]
# <br></br> # Dark current preparation # <br></br> # Observation creation (combining simulated sources and dark current) # <br></br><br></br> # This section shows how to call the three steps independently. The `imaging_simulator` function above is a wrapper around these three steps. Most users will want simply to call `imaging_simulator`. # ## First generate the "seed image" # # This is generally a 2D noiseless countrate image that contains only simulated astronomical sources. # # A seed image is generated based on a `.yaml` file that contains all the necessary parameters for simulating data. For this exercise, use the same yaml file that was used in the [Create Simulated Data](#run_steps_together) section as input. # In[ ]: cat = catalog_seed_image.Catalog_seed() cat.paramfile = yamlfile cat.make_seed() # ### Look at the seed image # In[ ]: show(cat.seedimage, 'Seed Image', max=20) # ## Prepare the dark current exposure # This will serve as the base of the simulated data. # This step will linearize the dark current (if it # is not already), and reorganize it into the # requested readout pattern and number of groups.
def test_overlap_coordinates_subarray():
"""Test the function that calculates the coordinates for the
overlap between two stamp images
"""
seed = catalog_seed_image.Catalog_seed(offline=True)
# Create a point sources table
tab = Table()
tab['index'] = np.arange(5).astype(np.int) + 1
tab['pixelx'] = [200, 50, 200, 350, -0.5]
tab['pixely'] = [200, 200, 25, 350, -0.5]
tab['RA_degrees'] = [12.008729, 11.999914, 12.003422, 11.995729, 11.995729]
tab['Dec_degrees'] = [
-0.00885006, 1.7231344e-09, -1.5512744e-05, -4.9949034e-05,
-4.9949034e-05
]
tab['magnitude'] = np.repeat(14, 5)
tab['countrate_e/s'] = np.repeat(161630.72, 5)
tab['counts_per_frame_e'] = np.repeat(1735391.9, 5)
subarray_expected_results = []
subarray_expected_results.append((50, 351, 50, 351, 0, 301, 0, 301))
subarray_expected_results.append((0, 201, 50, 351, 100, 301, 0, 301))
subarray_expected_results.append((50, 351, 0, 176, 0, 301, 125, 301))
subarray_expected_results.append((200, 400, 200, 400, 0, 200, 0, 200))
subarray_expected_results.append((0, 150, 0, 150, 151, 301, 151, 301))
# Test the basic function using a subarray
stamp_dims = (301, 301)
stamp_x = 150
stamp_y = 150
aperture_dims = (400, 400) # Assume 400x400 pixel subarray
for index in range(5):
results = seed.cropped_coords(tab[index]['pixelx'],
tab[index]['pixely'],
aperture_dims,
stamp_x,
stamp_y,
stamp_dims,
ignore_detector=False)
assert results == subarray_expected_results[index]
# Test the wrapper function
seed.coord_adjust['xoffset'] = 0 # Already defined, but let's be explicit
seed.coord_adjust['yoffset'] = 0 # Already defined, but let's be explicit
seed.output_dims = [400, 400] # Assume 400x400 pixel subarray
seed.subarray_bounds = [1648, 1648, 2047, 2047]
seed.ffsize = 2048
seed.psf_library_core_x_dim = 301
seed.psf_library_core_y_dim = 301
seed.params = {'simSignals': {}}
seed.add_psf_wings = False
seed.psf_library_oversamp = 1
for index in range(5):
results = seed.create_psf_stamp_coords(tab[index]['pixelx'],
tab[index]['pixely'],
stamp_dims,
stamp_x,
stamp_y,
coord_sys='aperture',
ignore_detector=False)
ijkl = []
for ele in results[4:]:
ijkl.extend(list(ele))
ijkl = tuple(ijkl)
assert ijkl == subarray_expected_results[index]
# Dummy PSF library
seed.psf_library = create_dummy_psf_grid()
seed.psf_library_x_dim = seed.psf_library.data.shape[
2] / seed.psf_library.oversampling
seed.psf_library_y_dim = seed.psf_library.data.shape[
1] / seed.psf_library.oversampling
# Test the wrapper function using a modified PSF stamp image shape
expected_k1l1 = [(50, 351, 50, 351, 0, 301, 0, 301),
(0, 201, 50, 351, 100, 301, 0, 301),
(50, 351, 0, 176, 0, 301, 125, 301),
(200, 400, 200, 400, 0, 200, 0, 200),
(0, 150, 0, 150, 151, 301, 151, 301)]
for index in range(5):
psf, min_x, min_y, add_wings = seed.create_psf_stamp(
tab[index]['pixelx'],
tab[index]['pixely'],
stamp_dims[1],
stamp_dims[0],
ignore_detector=False)
stamp_x_loc = 301 // 2 - min_x
stamp_y_loc = 301 // 2 - min_y
updated_psf_dimensions = psf.shape
xap, yap, xpts, ypts, (i1, i2), (j1, j2), (k1, k2), \
(l1, l2) = seed.create_psf_stamp_coords(tab[index]['pixelx'], tab[index]['pixely'],
updated_psf_dimensions, stamp_x_loc, stamp_y_loc,
coord_sys='aperture')
assert (i1, i2, j1, j2, k1, k2, l1, l2) == expected_k1l1[index]
def create(self):
"""MAIN FUNCTION"""
# Get parameters necessary to create the TSO data
orig_parameters = self.get_param_info()
subarray_table = utils.read_subarray_definition_file(
orig_parameters['Reffiles']['subarray_defs'])
orig_parameters = utils.get_subarray_info(orig_parameters,
subarray_table)
orig_parameters = utils.read_pattern_check(orig_parameters)
self.basename = os.path.join(
orig_parameters['Output']['directory'],
orig_parameters['Output']['file'][0:-5].split('/')[-1])
# Determine file splitting information. First get some basic info
# on the exposure
self.numints = orig_parameters['Readout']['nint']
self.numgroups = orig_parameters['Readout']['ngroup']
self.numframes = orig_parameters['Readout']['nframe']
self.numskips = orig_parameters['Readout']['nskip']
self.namps = orig_parameters['Readout']['namp']
self.numresets = orig_parameters['Readout']['resets_bet_ints']
self.frames_per_group, self.frames_per_int, self.total_frames = utils.get_frame_count_info(
self.numints, self.numgroups, self.numframes, self.numskips,
self.numresets)
# Get gain map for later unit conversion
#gainfile = orig_parameters['Reffiles']['gain']
#gain, gainheader = file_io.read_gain_file(gainfile)
# Make 2 copies of the input parameter file, separating the TSO
# source from the other sources
self.split_param_file(orig_parameters)
print('Splitting background and TSO source into multiple yaml files')
print('Running background sources through catalog_seed_image')
print('background param file is:', self.background_paramfile)
# Run the catalog_seed_generator on the non-TSO (background) sources
background_direct = catalog_seed_image.Catalog_seed()
background_direct.paramfile = self.background_paramfile
background_direct.make_seed()
background_segmentation_map = background_direct.seed_segmap
# Stellar spectrum hdf5 file will be required, so no need to create one here.
# Create hdf5 file with spectra of all sources if requested
self.final_SED_file = spectra_from_catalog.make_all_spectra(
self.catalog_files,
input_spectra_file=self.SED_file,
extrapolate_SED=self.extrapolate_SED,
output_filename=self.final_SED_file,
normalizing_mag_column=self.SED_normalizing_catalog_column)
bkgd_waves, bkgd_fluxes = backgrounds.nircam_background_spectrum(
orig_parameters, self.detector, self.module)
# Run the disperser on the background sources. Add the background
# signal here as well
print('\n\nDispersing background sources\n\n')
background_done = False
background_seed_files = [
background_direct.ptsrc_seed_filename,
background_direct.galaxy_seed_filename,
background_direct.extended_seed_filename
]
for seed_file in background_seed_files:
if seed_file is not None:
print("Dispersing seed image:", seed_file)
disp = self.run_disperser(seed_file,
orders=self.orders,
add_background=not background_done,
background_waves=bkgd_waves,
background_fluxes=bkgd_fluxes,
finalize=True)
if not background_done:
# Background is added at the first opportunity. At this
# point, create an array to hold the final combined
# dispersed background
background_done = True
background_dispersed = copy.deepcopy(disp.final)
else:
background_dispersed += disp.final
# Run the catalog_seed_generator on the TSO source
tso_direct = catalog_seed_image.Catalog_seed()
tso_direct.paramfile = self.tso_paramfile
tso_direct.make_seed()
tso_segmentation_map = tso_direct.seed_segmap
outside_tso_source = tso_segmentation_map == 0
tso_segmentation_map[outside_tso_source] = background_segmentation_map[
outside_tso_source]
# Dimensions are (y, x)
self.seed_dimensions = tso_direct.nominal_dims
# Read in the transmission spectrum that goes with the TSO source
tso_params = utils.read_yaml(self.tso_paramfile)
tso_catalog_file = tso_params['simSignals']['tso_grism_catalog']
tso_catalog = ascii.read(tso_catalog_file)
transmission_file = tso_catalog['Transmission_spectrum'].data
transmission_spectrum = ascii.read(transmission_file[0])
# Calculate the total exposure time, including resets, to check
# against the times provided in the catalog file.
total_exposure_time = self.calculate_exposure_time() * u.second
# Check to be sure the start and end times provided in the catalog
# are enough to cover the length of the exposure.
tso_catalog = self.tso_catalog_check(tso_catalog, total_exposure_time)
# Use batman to create lightcurves from the transmission spectrum
lightcurves, times = self.make_lightcurves(tso_catalog, self.frametime,
transmission_spectrum)
# Determine which frames of the exposure will take place with the unaltered stellar
# spectrum. This will be all frames where the associated lightcurve is 1.0 everywhere.
transit_frames, unaltered_frames = self.find_transit_frames(
lightcurves)
print('Frame numbers containing the transit: {} - {}'.format(
np.min(transit_frames), np.max(transit_frames)))
# Run the disperser using the original, unaltered stellar spectrum. Set 'cache=True'
print('\n\nDispersing TSO source\n\n')
grism_seed_object = self.run_disperser(tso_direct.seed_file,
orders=self.orders,
add_background=False,
cache=True,
finalize=True)
# Crop dispersed seed images to correct final subarray size
#no_transit_signal = grism_seed_object.final
no_transit_signal = utils.crop_to_subarray(grism_seed_object.final,
tso_direct.subarray_bounds)
background_dispersed = utils.crop_to_subarray(
background_dispersed, tso_direct.subarray_bounds)
# Mulitp[ly the dispersed seed images by the flat field
no_transit_signal *= tso_direct.flatfield
background_dispersed *= tso_direct.flatfield
# Save the dispersed seed images if requested
if self.save_dispersed_seed:
h_back = fits.PrimaryHDU(background_dispersed)
h_back.header['EXTNAME'] = 'BACKGROUND_SOURCES'
h_tso = fits.ImageHDU(grism_seed_object.final)
h_tso.header['EXTNAME'] = 'TSO_SOURCE'
hlist = fits.HDUList([h_back, h_tso])
disp_filename = '{}_dispersed_seed_images.fits'.format(
self.basename)
hlist.writeto(disp_filename, overwrite=True)
print(
'\nDispersed seed images (background sources and TSO source) saved to {}.\n\n'
.format(disp_filename))
# Calculate file splitting info
self.file_splitting()
# Prepare for creating output files
segment_file_dir = orig_parameters['Output']['directory']
if orig_parameters['Readout']['pupil'][0].upper() == 'F':
usefilt = 'pupil'
else:
usefilt = 'filter'
segment_file_base = orig_parameters['Output']['file'].replace(
'.fits', '_')
segment_file_base = '{}_{}_'.format(
segment_file_base, orig_parameters['Readout'][usefilt])
segment_file_base = os.path.join(segment_file_dir, segment_file_base)
# Loop over frames and integrations up to the size of the segment
# file.
ints_per_segment = self.int_segment_indexes[
1:] - self.int_segment_indexes[:-1]
groups_per_segment = self.grp_segment_indexes[
1:] - self.grp_segment_indexes[:-1]
total_frame_counter = 0
previous_segment = 1
segment_part_number = 0
segment_starting_int_number = 0
self.segment_part_number = 0
self.segment_ints = 0
self.segment_frames = 0
self.segment_frame_start_number = 0
self.segment_int_start_number = 0
self.part_int_start_number = 0
self.part_frame_start_number = 0
# Get split files' metadata
split_meta = SplitFileMetaData(self.int_segment_indexes,
self.grp_segment_indexes,
self.file_segment_indexes,
self.group_segment_indexes_g,
self.frames_per_int,
self.frames_per_group, self.frametime)
# List of all output seed files
self.seed_files = []
counter = 0
for i, int_dim in enumerate(ints_per_segment):
int_start = self.int_segment_indexes[i]
int_end = self.int_segment_indexes[i + 1]
for j, grp_dim in enumerate(groups_per_segment):
initial_frame = self.grp_segment_indexes[j]
# int_dim and grp_dim are the number of integrations and
# groups in the current segment PART
print(
"\n\nCurrent segment part contains: {} integrations and {} groups."
.format(int_dim, grp_dim))
print("Creating frame by frame dispersed signal")
segment_seed = np.zeros(
(int_dim, grp_dim, self.seed_dimensions[0],
self.seed_dimensions[1]))
for integ in np.arange(int_dim):
overall_integration_number = int_start + integ
previous_frame = np.zeros(self.seed_dimensions)
for frame in np.arange(grp_dim):
#print('TOTAL FRAME COUNTER: ', total_frame_counter)
#print('integ and frame: ', integ, frame)
# If a frame is from the part of the lightcurve
# with no transit, then the signal in the frame
# comes from no_transit_signal
if total_frame_counter in unaltered_frames:
frame_only_signal = (
background_dispersed +
no_transit_signal) * self.frametime
# If the frame is from a part of the lightcurve
# where the transit is happening, then call the
# cached disperser with the appropriate lightcurve
elif total_frame_counter in transit_frames:
#print("{} is during the transit".format(total_frame_counter))
frame_transmission = lightcurves[
total_frame_counter, :]
trans_interp = interp1d(
transmission_spectrum['Wavelength'],
frame_transmission)
for order in self.orders:
grism_seed_object.this_one[
order].disperse_all_from_cache(
trans_interp)
# Here is where we call finalize on the TSO object
# This will update grism_seed_object.final to
# contain the correct signal
grism_seed_object.finalize(Back=None,
BackLevel=None)
cropped_grism_seed_object = utils.crop_to_subarray(
grism_seed_object.final,
tso_direct.subarray_bounds)
frame_only_signal = (
background_dispersed +
cropped_grism_seed_object) * self.frametime
# Now add the signal from this frame to that in the
# previous frame in order to arrive at the total
# cumulative signal
segment_seed[
integ,
frame, :, :] = previous_frame + frame_only_signal
previous_frame = copy.deepcopy(
segment_seed[integ, frame, :, :])
total_frame_counter += 1
# At the end of each integration, increment the
# total_frame_counter by the number of resets between
# integrations
total_frame_counter += self.numresets
# Use the split files' metadata
self.segment_number = split_meta.segment_number[counter]
self.segment_ints = split_meta.segment_ints[counter]
self.segment_frames = split_meta.segment_frames[counter]
self.segment_part_number = split_meta.segment_part_number[
counter]
self.segment_frame_start_number = split_meta.segment_frame_start_number[
counter]
self.segment_int_start_number = split_meta.segment_int_start_number[
counter]
self.part_int_start_number = split_meta.part_int_start_number[
counter]
self.part_frame_start_number = split_meta.part_frame_start_number[
counter]
counter += 1
print('Overall integration number: ',
overall_integration_number)
segment_file_name = '{}seg{}_part{}_seed_image.fits'.format(
segment_file_base,
str(self.segment_number).zfill(3),
str(self.segment_part_number).zfill(3))
print('Segment int and frame start numbers: {} {}'.format(
self.segment_int_start_number,
self.segment_frame_start_number))
#print('Part int and frame start numbers (ints and frames within the segment): {} {}'.format(self.part_int_start_number, self.part_frame_start_number))
# Disperser output is always full frame. Crop to the
# requested subarray if necessary
if orig_parameters['Readout'][
'array_name'] not in self.fullframe_apertures:
print("Dispersed seed image size: {}".format(
segment_seed.shape))
segment_seed = utils.crop_to_subarray(
segment_seed, tso_direct.subarray_bounds)
#gain = utils.crop_to_subarray(gain, tso_direct.subarray_bounds)
# Segmentation map will be centered in a frame that is larger
# than full frame by a factor of sqrt(2), so crop appropriately
print('Cropping segmentation map to appropriate aperture')
segy, segx = tso_segmentation_map.shape
dx = int((segx - tso_direct.nominal_dims[1]) / 2)
dy = int((segy - tso_direct.nominal_dims[0]) / 2)
segbounds = [
tso_direct.subarray_bounds[0] + dx,
tso_direct.subarray_bounds[1] + dy,
tso_direct.subarray_bounds[2] + dx,
tso_direct.subarray_bounds[3] + dy
]
tso_segmentation_map = utils.crop_to_subarray(
tso_segmentation_map, segbounds)
# Convert seed image to ADU/sec to be consistent
# with other simulator outputs
gain = MEAN_GAIN_VALUES['nircam']['lw{}'.format(
self.module.lower())]
segment_seed /= gain
# Update seed image header to reflect the
# division by the gain
tso_direct.seedinfo['units'] = 'ADU/sec'
# Save the seed image. Save in units of ADU/sec
print('Saving seed image')
tso_seed_header = fits.getheader(tso_direct.seed_file)
self.save_seed(segment_seed, tso_segmentation_map,
tso_seed_header, orig_parameters) #,
#segment_number, segment_part_number)
# Prepare dark current exposure if
# needed.
print('Running dark prep')
d = dark_prep.DarkPrep()
d.paramfile = self.paramfile
d.prepare()
# Combine into final observation
print('Running observation generator')
obs = obs_generator.Observation()
obs.linDark = d.dark_files
obs.seed = self.seed_files
obs.segmap = tso_segmentation_map
obs.seedheader = tso_direct.seedinfo
obs.paramfile = self.paramfile
obs.create()