def test_hdf5_file_input():
"""Case where an hdf5 file is input. One of these also includes a
normalized spectrum"""
catfile = os.path.join(TEST_DATA_DIR, 'point_sources.cat')
output_hdf5 = os.path.join(TEST_DATA_DIR, 'all_spectra.hdf5')
sed_file = os.path.join(TEST_DATA_DIR,
'sed_file_with_normalized_dataset.hdf5')
sed_catalog = spec.make_all_spectra(
catfile,
input_spectra_file=sed_file,
normalizing_mag_column='nircam_f444w_magnitude',
output_filename=output_hdf5)
comparison = hdf5.open(
os.path.join(TEST_DATA_DIR,
'output_spec_from_hdf5_input_including_normalized.hdf5'))
constructed = hdf5.open(sed_catalog)
for key in comparison:
assert key in constructed.keys()
assert all(comparison[key]["wavelengths"].value == constructed[key]
["wavelengths"].value)
assert all(comparison[key]["fluxes"].value == constructed[key]
["fluxes"].value)
assert comparison[key]["wavelengths"].unit == constructed[key][
"wavelengths"].unit
assert comparison[key]["fluxes"].unit == constructed[key][
"fluxes"].unit
cat_base = catfile.split('.')[0]
outbase = cat_base + '_with_flambda.cat'
flambda_output_catalog = os.path.join(TEST_DATA_DIR, outbase)
os.remove(flambda_output_catalog)
os.remove(sed_catalog)
def test_manual_inputs():
"""Case where spectra are input manually along side ascii catalog"""
# Test case where spectra are input manually
catfile = os.path.join(TEST_DATA_DIR, 'point_sources.cat')
output_hdf5 = os.path.join(TEST_DATA_DIR, 'all_spectra.hdf5')
hdf5file = os.path.join(TEST_DATA_DIR,
'sed_file_with_normalized_dataset.hdf5')
sed_dict = hdf5.open(hdf5file)
sed_catalog = spec.make_all_spectra(
catfile,
input_spectra=sed_dict,
normalizing_mag_column='nircam_f444w_magnitude',
output_filename=output_hdf5)
constructed = hdf5.open(sed_catalog)
comparison = hdf5.open(
os.path.join(TEST_DATA_DIR, 'output_spec_from_manual_input.hdf5'))
for key in comparison:
assert key in constructed.keys()
assert all(comparison[key]["wavelengths"].value == constructed[key]
["wavelengths"].value)
assert all(comparison[key]["fluxes"].value == constructed[key]
["fluxes"].value)
assert comparison[key]["wavelengths"].unit == constructed[key][
"wavelengths"].unit
assert comparison[key]["fluxes"].unit == constructed[key][
"fluxes"].unit
cat_base = catfile.split('.')[0]
outbase = cat_base + '_with_flambda.cat'
flambda_output_catalog = os.path.join(TEST_DATA_DIR, outbase)
os.remove(flambda_output_catalog)
os.remove(sed_catalog)
def test_multiple_ascii_catalogs():
"""Case where multiple ascii catalogs are input"""
catfile = os.path.join(TEST_DATA_DIR, 'point_sources.cat')
galfile = os.path.join(TEST_DATA_DIR, 'galaxies.cat')
catalogs = [catfile, galfile]
output_hdf5 = os.path.join(TEST_DATA_DIR, 'all_spectra.hdf5')
sed_catalog = spec.make_all_spectra(catalogs, output_filename=output_hdf5)
comparison = hdf5.open(
os.path.join(TEST_DATA_DIR, 'source_sed_file_from_point_sources.hdf5'))
constructed = hdf5.open(output_hdf5)
for key in comparison:
assert key in constructed.keys()
assert all(comparison[key]["wavelengths"].value == constructed[key]
["wavelengths"].value)
assert all(comparison[key]["fluxes"].value == constructed[key]
["fluxes"].value)
assert comparison[key]["wavelengths"].unit == constructed[key][
"wavelengths"].unit
assert comparison[key]["fluxes"].unit == constructed[key][
"fluxes"].unit
for catfile in catalogs:
cat_base = catfile.split('.')[0]
outbase = cat_base + '_with_flambda.cat'
flambda_output_catalog = os.path.join(TEST_DATA_DIR, outbase)
os.remove(flambda_output_catalog)
os.remove(sed_catalog)
def test_single_mag_column():
"""Case where input ascii catalog contains only one magntude column.
In this case extrapolation is necessary."""
catfile = os.path.join(TEST_DATA_DIR, 'point_sources_one_filter.cat')
output_hdf5 = os.path.join(TEST_DATA_DIR, 'all_spectra.hdf5')
sed_catalog = spec.make_all_spectra(catfile, output_filename=output_hdf5)
comparison = hdf5.open(
os.path.join(TEST_DATA_DIR, 'output_spec_from_one_filter.hdf5'))
constructed = hdf5.open(output_hdf5)
for key in comparison:
assert key in constructed.keys()
assert all(comparison[key]["wavelengths"].value == constructed[key]
["wavelengths"].value)
assert all(comparison[key]["fluxes"].value == constructed[key]
["fluxes"].value)
assert comparison[key]["wavelengths"].unit == constructed[key][
"wavelengths"].unit
assert comparison[key]["fluxes"].unit == constructed[key][
"fluxes"].unit
cat_base = catfile.split('.')[0]
outbase = cat_base + '_with_flambda.cat'
flambda_output_catalog = os.path.join(TEST_DATA_DIR, outbase)
os.remove(flambda_output_catalog)
os.remove(sed_catalog)
def test_manual_plus_file_inputs():
"""Case where spectra are input via hdf5 file as well as manually"""
catfile = os.path.join(TEST_DATA_DIR, 'point_sources.cat')
sed_file = os.path.join(TEST_DATA_DIR, 'sed_file_with_normalized_dataset.hdf5')
output_hdf5 = os.path.join(TEST_DATA_DIR, 'all_spectra.hdf5')
manual_sed = {}
manual_sed[7] = {"wavelengths": [0.9, 1.4, 1.9, 3.5, 5.1]*u.micron,
"fluxes": [1e-17, 1.1e-17, 1.5e-17, 1.4e-17, 1.1e-17] * FLAMBDA_CGS_UNITS}
sed_catalog = spec.make_all_spectra(catfile, input_spectra=manual_sed, input_spectra_file=sed_file,
normalizing_mag_column='nircam_f444w_magnitude',
output_filename=output_hdf5, module='A')
comparison = hdf5.open(os.path.join(TEST_DATA_DIR, 'output_spec_from_file_plus_manual_input.hdf5'))
constructed = hdf5.open(output_hdf5)
for key in comparison:
assert key in constructed.keys()
assert all(comparison[key]["wavelengths"].value == constructed[key]["wavelengths"].value)
assert all(comparison[key]["fluxes"].value == constructed[key]["fluxes"].value)
assert comparison[key]["wavelengths"].unit == constructed[key]["wavelengths"].unit
assert comparison[key]["fluxes"].unit == constructed[key]["fluxes"].unit
cat_base = catfile.split('.')[0]
outbase = cat_base + '_with_flambda.cat'
flambda_output_catalog = os.path.join(TEST_DATA_DIR, outbase)
os.remove(flambda_output_catalog)
os.remove(sed_catalog)
def test_multiple_mag_columns():
"""Case where ascii catalog with multiple magnitude columns is input"""
catfile = os.path.join(TEST_DATA_DIR, 'point_sources.cat')
output_hdf5 = os.path.join(TEST_DATA_DIR, 'all_spectra.hdf5')
sed_catalog = spec.make_all_spectra(catfile, output_filename=output_hdf5)
constructed = hdf5.open(sed_catalog)
comparison = hdf5.open(os.path.join(TEST_DATA_DIR, 'output_spec_from_multiple_filter.hdf5'))
for key in comparison:
assert key in constructed.keys()
assert all(comparison[key]["wavelengths"].value == constructed[key]["wavelengths"].value)
assert all(comparison[key]["fluxes"].value == constructed[key]["fluxes"].value)
assert comparison[key]["wavelengths"].unit == constructed[key]["wavelengths"].unit
assert comparison[key]["fluxes"].unit == constructed[key]["fluxes"].unit
cat_base = catfile.split('.')[0]
outbase = cat_base + '_with_flambda.cat'
flambda_output_catalog = os.path.join(TEST_DATA_DIR, outbase)
os.remove(flambda_output_catalog)
os.remove(sed_catalog)
def create(self):
"""MAIN FUNCTION"""
# Get parameters necessary to create the TSO data
orig_parameters = self.get_param_info()
subarray_table = utils.read_subarray_definition_file(
orig_parameters['Reffiles']['subarray_defs'])
orig_parameters = utils.get_subarray_info(orig_parameters,
subarray_table)
orig_parameters = utils.read_pattern_check(orig_parameters)
self.basename = os.path.join(
orig_parameters['Output']['directory'],
orig_parameters['Output']['file'][0:-5].split('/')[-1])
# Determine file splitting information. First get some basic info
# on the exposure
self.numints = orig_parameters['Readout']['nint']
self.numgroups = orig_parameters['Readout']['ngroup']
self.numframes = orig_parameters['Readout']['nframe']
self.numskips = orig_parameters['Readout']['nskip']
self.namps = orig_parameters['Readout']['namp']
self.numresets = orig_parameters['Readout']['resets_bet_ints']
self.frames_per_group, self.frames_per_int, self.total_frames = utils.get_frame_count_info(
self.numints, self.numgroups, self.numframes, self.numskips,
self.numresets)
# Get gain map for later unit conversion
#gainfile = orig_parameters['Reffiles']['gain']
#gain, gainheader = file_io.read_gain_file(gainfile)
# Make 2 copies of the input parameter file, separating the TSO
# source from the other sources
self.split_param_file(orig_parameters)
print('Splitting background and TSO source into multiple yaml files')
print('Running background sources through catalog_seed_image')
print('background param file is:', self.background_paramfile)
# Run the catalog_seed_generator on the non-TSO (background) sources
background_direct = catalog_seed_image.Catalog_seed()
background_direct.paramfile = self.background_paramfile
background_direct.make_seed()
background_segmentation_map = background_direct.seed_segmap
# Stellar spectrum hdf5 file will be required, so no need to create one here.
# Create hdf5 file with spectra of all sources if requested
self.final_SED_file = spectra_from_catalog.make_all_spectra(
self.catalog_files,
input_spectra_file=self.SED_file,
extrapolate_SED=self.extrapolate_SED,
output_filename=self.final_SED_file,
normalizing_mag_column=self.SED_normalizing_catalog_column)
bkgd_waves, bkgd_fluxes = backgrounds.nircam_background_spectrum(
orig_parameters, self.detector, self.module)
# Run the disperser on the background sources. Add the background
# signal here as well
print('\n\nDispersing background sources\n\n')
background_done = False
background_seed_files = [
background_direct.ptsrc_seed_filename,
background_direct.galaxy_seed_filename,
background_direct.extended_seed_filename
]
for seed_file in background_seed_files:
if seed_file is not None:
print("Dispersing seed image:", seed_file)
disp = self.run_disperser(seed_file,
orders=self.orders,
add_background=not background_done,
background_waves=bkgd_waves,
background_fluxes=bkgd_fluxes,
finalize=True)
if not background_done:
# Background is added at the first opportunity. At this
# point, create an array to hold the final combined
# dispersed background
background_done = True
background_dispersed = copy.deepcopy(disp.final)
else:
background_dispersed += disp.final
# Run the catalog_seed_generator on the TSO source
tso_direct = catalog_seed_image.Catalog_seed()
tso_direct.paramfile = self.tso_paramfile
tso_direct.make_seed()
tso_segmentation_map = tso_direct.seed_segmap
outside_tso_source = tso_segmentation_map == 0
tso_segmentation_map[outside_tso_source] = background_segmentation_map[
outside_tso_source]
# Dimensions are (y, x)
self.seed_dimensions = tso_direct.nominal_dims
# Read in the transmission spectrum that goes with the TSO source
tso_params = utils.read_yaml(self.tso_paramfile)
tso_catalog_file = tso_params['simSignals']['tso_grism_catalog']
tso_catalog = ascii.read(tso_catalog_file)
transmission_file = tso_catalog['Transmission_spectrum'].data
transmission_spectrum = ascii.read(transmission_file[0])
# Calculate the total exposure time, including resets, to check
# against the times provided in the catalog file.
total_exposure_time = self.calculate_exposure_time() * u.second
# Check to be sure the start and end times provided in the catalog
# are enough to cover the length of the exposure.
tso_catalog = self.tso_catalog_check(tso_catalog, total_exposure_time)
# Use batman to create lightcurves from the transmission spectrum
lightcurves, times = self.make_lightcurves(tso_catalog, self.frametime,
transmission_spectrum)
# Determine which frames of the exposure will take place with the unaltered stellar
# spectrum. This will be all frames where the associated lightcurve is 1.0 everywhere.
transit_frames, unaltered_frames = self.find_transit_frames(
lightcurves)
print('Frame numbers containing the transit: {} - {}'.format(
np.min(transit_frames), np.max(transit_frames)))
# Run the disperser using the original, unaltered stellar spectrum. Set 'cache=True'
print('\n\nDispersing TSO source\n\n')
grism_seed_object = self.run_disperser(tso_direct.seed_file,
orders=self.orders,
add_background=False,
cache=True,
finalize=True)
# Crop dispersed seed images to correct final subarray size
#no_transit_signal = grism_seed_object.final
no_transit_signal = utils.crop_to_subarray(grism_seed_object.final,
tso_direct.subarray_bounds)
background_dispersed = utils.crop_to_subarray(
background_dispersed, tso_direct.subarray_bounds)
# Mulitp[ly the dispersed seed images by the flat field
no_transit_signal *= tso_direct.flatfield
background_dispersed *= tso_direct.flatfield
# Save the dispersed seed images if requested
if self.save_dispersed_seed:
h_back = fits.PrimaryHDU(background_dispersed)
h_back.header['EXTNAME'] = 'BACKGROUND_SOURCES'
h_tso = fits.ImageHDU(grism_seed_object.final)
h_tso.header['EXTNAME'] = 'TSO_SOURCE'
hlist = fits.HDUList([h_back, h_tso])
disp_filename = '{}_dispersed_seed_images.fits'.format(
self.basename)
hlist.writeto(disp_filename, overwrite=True)
print(
'\nDispersed seed images (background sources and TSO source) saved to {}.\n\n'
.format(disp_filename))
# Calculate file splitting info
self.file_splitting()
# Prepare for creating output files
segment_file_dir = orig_parameters['Output']['directory']
if orig_parameters['Readout']['pupil'][0].upper() == 'F':
usefilt = 'pupil'
else:
usefilt = 'filter'
segment_file_base = orig_parameters['Output']['file'].replace(
'.fits', '_')
segment_file_base = '{}_{}_'.format(
segment_file_base, orig_parameters['Readout'][usefilt])
segment_file_base = os.path.join(segment_file_dir, segment_file_base)
# Loop over frames and integrations up to the size of the segment
# file.
ints_per_segment = self.int_segment_indexes[
1:] - self.int_segment_indexes[:-1]
groups_per_segment = self.grp_segment_indexes[
1:] - self.grp_segment_indexes[:-1]
total_frame_counter = 0
previous_segment = 1
segment_part_number = 0
segment_starting_int_number = 0
self.segment_part_number = 0
self.segment_ints = 0
self.segment_frames = 0
self.segment_frame_start_number = 0
self.segment_int_start_number = 0
self.part_int_start_number = 0
self.part_frame_start_number = 0
# Get split files' metadata
split_meta = SplitFileMetaData(self.int_segment_indexes,
self.grp_segment_indexes,
self.file_segment_indexes,
self.group_segment_indexes_g,
self.frames_per_int,
self.frames_per_group, self.frametime)
# List of all output seed files
self.seed_files = []
counter = 0
for i, int_dim in enumerate(ints_per_segment):
int_start = self.int_segment_indexes[i]
int_end = self.int_segment_indexes[i + 1]
for j, grp_dim in enumerate(groups_per_segment):
initial_frame = self.grp_segment_indexes[j]
# int_dim and grp_dim are the number of integrations and
# groups in the current segment PART
print(
"\n\nCurrent segment part contains: {} integrations and {} groups."
.format(int_dim, grp_dim))
print("Creating frame by frame dispersed signal")
segment_seed = np.zeros(
(int_dim, grp_dim, self.seed_dimensions[0],
self.seed_dimensions[1]))
for integ in np.arange(int_dim):
overall_integration_number = int_start + integ
previous_frame = np.zeros(self.seed_dimensions)
for frame in np.arange(grp_dim):
#print('TOTAL FRAME COUNTER: ', total_frame_counter)
#print('integ and frame: ', integ, frame)
# If a frame is from the part of the lightcurve
# with no transit, then the signal in the frame
# comes from no_transit_signal
if total_frame_counter in unaltered_frames:
frame_only_signal = (
background_dispersed +
no_transit_signal) * self.frametime
# If the frame is from a part of the lightcurve
# where the transit is happening, then call the
# cached disperser with the appropriate lightcurve
elif total_frame_counter in transit_frames:
#print("{} is during the transit".format(total_frame_counter))
frame_transmission = lightcurves[
total_frame_counter, :]
trans_interp = interp1d(
transmission_spectrum['Wavelength'],
frame_transmission)
for order in self.orders:
grism_seed_object.this_one[
order].disperse_all_from_cache(
trans_interp)
# Here is where we call finalize on the TSO object
# This will update grism_seed_object.final to
# contain the correct signal
grism_seed_object.finalize(Back=None,
BackLevel=None)
cropped_grism_seed_object = utils.crop_to_subarray(
grism_seed_object.final,
tso_direct.subarray_bounds)
frame_only_signal = (
background_dispersed +
cropped_grism_seed_object) * self.frametime
# Now add the signal from this frame to that in the
# previous frame in order to arrive at the total
# cumulative signal
segment_seed[
integ,
frame, :, :] = previous_frame + frame_only_signal
previous_frame = copy.deepcopy(
segment_seed[integ, frame, :, :])
total_frame_counter += 1
# At the end of each integration, increment the
# total_frame_counter by the number of resets between
# integrations
total_frame_counter += self.numresets
# Use the split files' metadata
self.segment_number = split_meta.segment_number[counter]
self.segment_ints = split_meta.segment_ints[counter]
self.segment_frames = split_meta.segment_frames[counter]
self.segment_part_number = split_meta.segment_part_number[
counter]
self.segment_frame_start_number = split_meta.segment_frame_start_number[
counter]
self.segment_int_start_number = split_meta.segment_int_start_number[
counter]
self.part_int_start_number = split_meta.part_int_start_number[
counter]
self.part_frame_start_number = split_meta.part_frame_start_number[
counter]
counter += 1
print('Overall integration number: ',
overall_integration_number)
segment_file_name = '{}seg{}_part{}_seed_image.fits'.format(
segment_file_base,
str(self.segment_number).zfill(3),
str(self.segment_part_number).zfill(3))
print('Segment int and frame start numbers: {} {}'.format(
self.segment_int_start_number,
self.segment_frame_start_number))
#print('Part int and frame start numbers (ints and frames within the segment): {} {}'.format(self.part_int_start_number, self.part_frame_start_number))
# Disperser output is always full frame. Crop to the
# requested subarray if necessary
if orig_parameters['Readout'][
'array_name'] not in self.fullframe_apertures:
print("Dispersed seed image size: {}".format(
segment_seed.shape))
segment_seed = utils.crop_to_subarray(
segment_seed, tso_direct.subarray_bounds)
#gain = utils.crop_to_subarray(gain, tso_direct.subarray_bounds)
# Segmentation map will be centered in a frame that is larger
# than full frame by a factor of sqrt(2), so crop appropriately
print('Cropping segmentation map to appropriate aperture')
segy, segx = tso_segmentation_map.shape
dx = int((segx - tso_direct.nominal_dims[1]) / 2)
dy = int((segy - tso_direct.nominal_dims[0]) / 2)
segbounds = [
tso_direct.subarray_bounds[0] + dx,
tso_direct.subarray_bounds[1] + dy,
tso_direct.subarray_bounds[2] + dx,
tso_direct.subarray_bounds[3] + dy
]
tso_segmentation_map = utils.crop_to_subarray(
tso_segmentation_map, segbounds)
# Convert seed image to ADU/sec to be consistent
# with other simulator outputs
gain = MEAN_GAIN_VALUES['nircam']['lw{}'.format(
self.module.lower())]
segment_seed /= gain
# Update seed image header to reflect the
# division by the gain
tso_direct.seedinfo['units'] = 'ADU/sec'
# Save the seed image. Save in units of ADU/sec
print('Saving seed image')
tso_seed_header = fits.getheader(tso_direct.seed_file)
self.save_seed(segment_seed, tso_segmentation_map,
tso_seed_header, orig_parameters) #,
#segment_number, segment_part_number)
# Prepare dark current exposure if
# needed.
print('Running dark prep')
d = dark_prep.DarkPrep()
d.paramfile = self.paramfile
d.prepare()
# Combine into final observation
print('Running observation generator')
obs = obs_generator.Observation()
obs.linDark = d.dark_files
obs.seed = self.seed_files
obs.segmap = tso_segmentation_map
obs.seedheader = tso_direct.seedinfo
obs.paramfile = self.paramfile
obs.create()
