def pathtest(step_input_filename, reffile, comparison_filename,
writefile=True, show_figs=False, save_figs=True, threshold_diff=1.0e-7, debug=False):
"""
This function calculates the difference between the pipeline and
calculated pathloss values. The functions use the output of sourcetype.
Args:
step_input_filename: str, name of sourcetype step output fits file
reffile: str, path to the pathloss MSA UNI reference fits files
comparison_filename: str, path to comparison pipeline pathloss file
writefile: boolean, if True writes fits files of calculated pathloss
and difference images
show_figs: boolean, whether to show plots or not
save_figs: boolean, save the plots
threshold_diff: float, threshold difference between pipeline output
& ESA file
debug: boolean, if true print statements will show on-screen
Returns:
- 1 plot, if told to save and/or show them.
- median_diff: Boolean, True if smaller or equal to threshold.
- log_msgs: list, all print statements are captured in this variable
"""
log_msgs = []
# start the timer
pathtest_start_time = time.time()
# get info from the rate file header
det = fits.getval(step_input_filename, "DETECTOR", 0)
msg = 'step_input_filename='+step_input_filename
print(msg)
log_msgs.append(msg)
exptype = fits.getval(step_input_filename, "EXP_TYPE", 0)
grat = fits.getval(step_input_filename, "GRATING", 0)
filt = fits.getval(step_input_filename, "FILTER", 0)
# aper = fits.getval(step_input_filename[1], "SLTNAME", 0)
msg = "path_loss_file: Grating:"+grat+" Filter:"+filt+" EXP_TYPE:"+exptype
print(msg)
log_msgs.append(msg)
# get the reference files
msg = "Using reference file: "+reffile
print(msg)
log_msgs.append(msg)
if writefile:
# create the fits list to hold the calculated flat values for each slit
hdu0 = fits.PrimaryHDU()
outfile = fits.HDUList()
outfile.append(hdu0)
# create fits list to hold image of pipeline-calculated diff values
hdu0 = fits.PrimaryHDU()
compfile = fits.HDUList()
compfile.append(hdu0)
# list to determine if pytest is passed or not
total_test_result = []
print("""Checking if files exist and obtaining datamodels.
This takes a few minutes...""")
if os.path.isfile(comparison_filename):
if debug:
print('Comparison file does exist.')
else:
result_msg = 'Comparison file does NOT exist. Skipping pathloss test.'
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the comparison data model
pathloss_pipe = datamodels.open(comparison_filename)
if debug:
print('got comparison datamodel!')
if os.path.isfile(step_input_filename):
if debug:
print('Input file does exist.')
else:
result_msg = 'Input file does NOT exist. Skipping pathloss test.'
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the input data model
pl = datamodels.open(step_input_filename)
if debug:
print('got input datamodel!')
# loop through the slits
msg = " Looping through the slits... "
print(msg)
log_msgs.append(msg)
slit_val = 0
for slit, pipe_slit in zip(pl.slits, pathloss_pipe.slits):
slit_val = slit_val+1
mode = "MOS"
is_point_source = False
print("Retrieving extensions")
ps_uni_ext_list = get_mos_ps_uni_extensions(reffile, is_point_source)
slit_id = slit.name
try:
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
if is_point_source:
if nshutters == 3:
shutter_key = "MOS1x3"
elif nshutters == 1:
shutter_key = "MOS1x1"
ext = ps_uni_ext_list[0][shutter_key]
print("Retrieved point source extension")
if is_point_source is False:
if nshutters == 1:
shutter_key = "MOS1x1"
elif nshutters == 3:
shutter_key = "MOS1x3"
ext = ps_uni_ext_list[1][shutter_key]
print("Retrieved extended source extension {}".format(ext))
except KeyError:
print("Unable to retrieve extension. Using ext 3, but may be 7")
ext = 3
wcs_obj = slit.meta.wcs
x, y = wcstools.grid_from_bounding_box(wcs_obj.bounding_box, step=(1, 1), center=True)
ra, dec, wave = slit.meta.wcs(x, y)
wave_sci = wave * 10**(-6) # microns --> meters
plcor_ref_ext = fits.getdata(reffile, ext)
print("plcor_ref_ext.shape", plcor_ref_ext.shape)
hdul = fits.open(reffile)
plcor_ref = hdul[1].data
w = wcs.WCS(hdul[1].header)
w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], : plcor_ref.shape[1],
:plcor_ref.shape[2]]
slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0)
comp_sci = pipe_slit.data
previous_sci = slit.data
pipe_correction = pipe_slit.pathloss
# set up generals for all the plots
font = {'weight': 'normal',
'size': 10}
matplotlib.rc('font', **font)
corr_vals = np.interp(wave_sci, wave_ref[:, 0, 0], plcor_ref_ext)
corrected_array = previous_sci/corr_vals
# plots:
step_input_filepath = step_input_filename.replace(".fits", "")
# my correction values
fig = plt.figure()
ax = plt.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
ax.get_xaxis().get_major_formatter().set_scientific(False)
# calculated correction values
plt.subplot(221)
norm = ImageNormalize(corr_vals)
plt.imshow(corr_vals, norm=norm, aspect=10.0, origin='lower',
cmap='viridis')
plt.xlabel('x in pixels')
plt.ylabel('y in pixels')
plt.title('Calculated Correction')
plt.colorbar()
# pipeline correction values
plt.subplot(222)
norm = ImageNormalize(pipe_correction)
plt.imshow(pipe_correction, norm=norm, aspect=10.0, origin='lower',
cmap='viridis')
plt.xlabel('x in pixels')
plt.ylabel('y in pixels')
plt.title('Pipeline Correction')
plt.colorbar()
# residuals (pipe correction - my correction)
corr_residuals = pipe_correction - corr_vals
plt.subplot(223)
norm = ImageNormalize(corr_residuals)
plt.ticklabel_format(useOffset=False)
plt.imshow(corr_residuals, norm=norm, aspect=10.0, origin='lower',
cmap='viridis')
plt.xlabel('x in pixels')
plt.ylabel('y in pixels')
plt.title('Correction residuals')
plt.colorbar()
# my science data after pathloss
plt.subplot(224)
norm = ImageNormalize(corrected_array)
plt.imshow(corrected_array, norm=norm, aspect=10.0, origin='lower',
cmap='viridis')
plt.title('Corrected Data After Pathloss')
plt.xlabel('x in pixels')
plt.ylabel('y in pixels')
plt.colorbar()
fig.suptitle("MOS UNI Pathloss Calibration Testing")
if show_figs:
plt.show()
if save_figs:
plt_name = step_input_filepath+"Pathloss_test_slitlet_" + str(mode) + "_UNI_" + str(slit_id) + ".png"
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
plt.close()
ax = plt.subplot(212)
plt.hist(corr_residuals[~np.isnan(corr_residuals)], bins=100,
range=(-0.00000013, 0.00000013))
plt.title('Residuals Histogram')
plt.xlabel("Correction Value")
plt.ylabel("Number of Occurences")
nanind = np.isnan(corr_residuals) # get all the nan indexes
notnan = ~nanind # get all the not-nan indexes
arr_mean = np.mean(corr_residuals[notnan])
arr_median = np.median(corr_residuals[notnan])
arr_stddev = np.std(corr_residuals[notnan])
plt.axvline(arr_mean, label="mean = %0.3e" % (arr_mean), color="g")
plt.axvline(arr_median, label="median = %0.3e" % (arr_median),
linestyle="-.", color="b")
str_arr_stddev = "stddev = {:0.3e}".format(arr_stddev)
ax.text(0.73, 0.67, str_arr_stddev, transform=ax.transAxes,
fontsize=16)
plt.legend()
plt.minorticks_on()
# Show and/or save figures
if save_figs:
plt_name = step_input_filepath + "Pathlosstest_MOS_UNI_slitlet_" + slit_id + ".png"
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
if show_figs:
plt.show()
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
if debug:
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
if debug:
print(msg)
log_msgs.append(msg)
plt.close()
# create fits file to hold the calculated pathloss for each slit
if writefile:
msg = "Saving the fits files with the calculated pathloss for each slit..."
print(msg)
log_msgs.append(msg)
# this is the file to hold the image of pipeline-calculated difference values
outfile_ext = fits.ImageHDU(corr_vals, name=slit_id)
outfile.append(outfile_ext)
# this is the file to hold the image of pipeline-calculated difference values
compfile_ext = fits.ImageHDU(corr_residuals, name=slit_id)
compfile.append(compfile_ext)
if corr_residuals[~np.isnan(corr_residuals)].size == 0:
msg1 = """Unable to calculate statistics because difference
array has all values as NaN.
Test will be set to FAILED."""
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
msg = "Calculating statistics... "
print(msg)
log_msgs.append(msg)
# ignore outliers:
corr_residuals = corr_residuals[np.where((corr_residuals != 999.0)
& (corr_residuals < 0.1)
& (corr_residuals > -0.1))]
if corr_residuals.size == 0:
msg1 = """ * Unable to calculate statistics because
difference array has all outlier values.
Test will be set to FAILED."""
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
stats_and_strings = auxfunc.print_stats(corr_residuals, "Difference", float(threshold_diff), absolute=True)
stats, stats_print_strings = stats_and_strings
corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats
for msg in stats_print_strings:
log_msgs.append(msg)
# This is the key argument for the assert pytest function
median_diff = False
if abs(corr_residuals_median) <= float(threshold_diff):
median_diff = True
if median_diff:
test_result = "PASSED"
else:
test_result = "FAILED"
msg = " *** Result of the test: "+test_result+"\n"
print(msg)
log_msgs.append(msg)
total_test_result.append(test_result)
if writefile:
outfile_name = step_input_filename.replace("srctype", det+"_calcuated_pathloss")
compfile_name = step_input_filename.replace("srctype", det+"_comparison_pathloss")
# create the fits list to hold the calculated pathloss values for each slit
outfile.writeto(outfile_name, overwrite=True)
# this is the file to hold the image of pipeline-calculated difference values
compfile.writeto(compfile_name, overwrite=True)
msg = "\nFits file with calculated pathloss values of each slit saved as: "
print(msg)
log_msgs.append(msg)
print(outfile_name)
log_msgs.append(outfile_name)
msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: "
print(msg)
log_msgs.append(msg)
print(compfile_name)
log_msgs.append(compfile_name)
# If all tests passed then pytest will be marked as PASSED, else it will be FAILED
FINAL_TEST_RESULT = False
for t in total_test_result:
if t == "FAILED":
FINAL_TEST_RESULT = False
break
else:
FINAL_TEST_RESULT = True
if FINAL_TEST_RESULT:
msg = "\n *** Final result for path_loss test will be reported as PASSED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "All slits PASSED path_loss test."
else:
msg = "\n *** Final result for path_loss test will be reported as FAILED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "One or more slits FAILED path_loss test."
# end the timer
pathloss_end_time = time.time() - pathtest_start_time
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time/60.0 # in minutes
pathloss_tot_time = "* Script msa_uni.py took ", repr(pathloss_end_time)+" minutes to finish."
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time/60. # in hours
pathloss_tot_time = "* Script msa_uni.py took ", repr(pathloss_end_time)+" hours to finish."
else:
pathloss_tot_time = "* Script msa_uni.py took ", repr(pathloss_end_time)+" seconds to finish."
print(pathloss_tot_time)
log_msgs.append(pathloss_tot_time)
return FINAL_TEST_RESULT, result_msg, log_msgs
def _corrections_for_mos(slit, pathloss, exp_type, source_type=None):
"""Calculate the correction arrasy for MOS slit
Parameters
----------
slit : jwst.datamodels.SlitModel
The slit being operated on.
pathloss : jwst.datamodels.DataModel
The pathloss reference data
exp_type : str
Exposure type
source_type : str or None
Force processing using the specified source type.
Returns
-------
correction : jwst.datamodels.SlitModel
The correction arrays
"""
correction = None
size = slit.data.size
# Only work on slits with data.size > 0
if size > 0:
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
aperture = get_aperture_from_model(pathloss, nshutters)
if aperture is not None:
(wavelength_pointsource, pathloss_pointsource_vector,
is_inside_slitlet) = calculate_pathloss_vector(
aperture.pointsource_data, aperture.pointsource_wcs, xcenter,
ycenter)
(wavelength_uniformsource, pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs, xcenter,
ycenter)
if is_inside_slitlet:
# Wavelengths in the reference file are in meters,
# need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the point source pathloss 2D correction
pathloss_2d_ps = interpolate_onto_grid(
wavelength_array, wavelength_pointsource,
pathloss_pointsource_vector)
# Compute the uniform source pathloss 2D correction
pathloss_2d_un = interpolate_onto_grid(
wavelength_array, wavelength_uniformsource,
pathloss_uniform_vector)
# Use the appropriate correction for this slit
if is_pointsource(source_type or slit.source_type):
pathloss_2d = pathloss_2d_ps
else:
pathloss_2d = pathloss_2d_un
# Save the corrections. The `data` portion is the correction used.
# The individual ones will be saved in the respective attributes.
correction = datamodels.SlitModel(data=pathloss_2d)
correction.pathloss_point = pathloss_2d_ps
correction.pathloss_uniform = pathloss_2d_un
else:
log.warning("Source is outside slit.")
else:
log.warning("Cannot find matching pathloss model for slit with"
f"{nshutters} shutters")
else:
log.warning(f"Slit has data size = {size}")
return correction
def do_correction(input_model, pathloss_model):
"""
Short Summary
-------------
Execute all tasks for Path Loss Correction
Parameters
----------
input_model : data model object
science data to be corrected
pathloss_model : pathloss model object
pathloss correction data
Returns
-------
output_model : data model object
Science data with pathloss extensions added
"""
exp_type = input_model.meta.exposure.type
log.info('Input exposure type is {}'.format(exp_type))
output_model = input_model.copy()
if exp_type == 'NRS_MSASPEC':
slit_number = 0
for slit in output_model.slits:
slit_number = slit_number + 1
log.info('Working on slit {}'.format(slit_number))
size = slit.data.size
# That has data.size > 0
if size > 0:
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
aperture = get_aperture_from_model(pathloss_model, nshutters)
if aperture is not None:
(wavelength_pointsource,
pathloss_pointsource_vector,
is_inside_slitlet) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slitlet:
# Wavelengths in the reference file are in meters,
#need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the pathloss 2D correction
if is_pointsource(slit.source_type):
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
else:
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Apply the pathloss 2D correction and attach to datamodel
slit.data /= pathloss_2d
slit.err /= pathloss_2d
slit.var_poisson /= pathloss_2d**2
slit.var_rnoise /= pathloss_2d**2
if slit.var_flat is not None and np.size(slit.var_flat) > 0:
slit.var_flat /= pathloss_2d**2
slit.pathloss = pathloss_2d
else:
log.warning("Source is outside slit. Skipping "
"pathloss correction for slit {}".format(slit_number))
else:
log.warning("Cannot find matching pathloss model for slit "
"with {} shutters, skipping pathloss correction for this "
"slit".format(nshutters))
continue
else:
log.warning("Slit has data size = {}, skipping "
"pathloss correction for this slitlet".format(size))
output_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']:
slit_number = 0
is_inside_slit = True
# For each slit
for slit in output_model.slits:
log.info('Working on slit {}'.format(slit.name))
slit_number = slit_number + 1
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
aperture = get_aperture_from_model(pathloss_model, slit.name)
if aperture is not None:
log.info("Using aperture {}".format(aperture.name))
(wavelength_pointsource,
pathloss_pointsource_vector,
is_inside_slit) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slit:
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the pathloss 2D correction
if is_pointsource(slit.source_type):
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
else:
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Apply the pathloss 2D correction and attach to datamodel
slit.data /= pathloss_2d
slit.err /= pathloss_2d
slit.var_poisson /= pathloss_2d**2
slit.var_rnoise /= pathloss_2d**2
if slit.var_flat is not None and np.size(slit.var_flat) > 0:
slit.var_flat /= pathloss_2d**2
slit.pathloss = pathloss_2d
else:
log.warning("Source is outside slit. Skipping "
"pathloss correction for slit {}".format(slit.name))
else:
log.warning("Cannot find matching pathloss model for aperture {} "
"skipping pathloss correction for this slit".format(slit.name))
continue
output_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type == 'NRS_IFU':
# IFU targets are always inside slit
# Get centering
xcenter, ycenter = get_center(exp_type, None)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
aperture = pathloss_model.apertures[0]
(wavelength_pointsource,
pathloss_pointsource_vector,
dummy) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
# Create the 2-d pathloss arrays, initialize with NaNs
wavelength_array = np.zeros(input_model.shape, dtype=np.float32)
wavelength_array.fill(np.nan)
for slice in NIRSPEC_IFU_SLICES:
slice_wcs = nirspec.nrs_wcs_set_input(input_model, slice)
x, y = wcstools.grid_from_bounding_box(slice_wcs.bounding_box)
xmin = int(x.min())
xmax = int(x.max())
ymin = int(y.min())
ymax = int(y.max())
ra, dec, wavelength = slice_wcs(x, y)
wavelength_array[ymin:ymax+1, xmin:xmax+1] = wavelength
# Compute the pathloss 2D correction
if is_pointsource(input_model.meta.target.source_type):
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
else:
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Apply the pathloss 2D correction and attach to datamodel
output_model.data /= pathloss_2d
output_model.err /= pathloss_2d
output_model.var_poisson /= pathloss_2d**2
output_model.var_rnoise /= pathloss_2d**2
if output_model.var_flat is not None and np.size(output_model.var_flat) > 0:
output_model.var_flat /= pathloss_2d**2
output_model.pathloss = pathloss_2d
# This might be useful to other steps
output_model.wavelength = wavelength_array
output_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type == 'NIS_SOSS':
"""NIRISS SOSS pathloss correction is basically a correction for the
flux from the 2nd and 3rd order dispersion that falls outside the
subarray aperture. The correction depends
on the pupil wheel position and column number (or wavelength). The
simple option is to do the correction by column number, then the only
interpolation needed is a 1-d interpolation into the pupil wheel position
dimension."""
# Omit correction if this is a TSO observation
if input_model.meta.visit.tsovisit:
log.warning("NIRISS SOSS TSO observations skip the pathloss step")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
pupil_wheel_position = input_model.meta.instrument.pupil_position
if pupil_wheel_position is None:
log.warning("Unable to get pupil wheel position from PWCPOS keyword "
"for {}".format(input_model.meta.filename))
log.warning("Pathloss correction skipped")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
subarray = input_model.meta.subarray.name
# Get the aperture from the reference file that matches the subarray
aperture = get_aperture_from_model(pathloss_model, subarray)
if aperture is None:
log.warning("Unable to get Aperture from reference file "
"for subarray {}".format(subarray))
log.warning("Pathloss correction skipped")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
else:
log.info("Aperture {} selected from reference file".format(aperture.name))
pathloss_array = aperture.pointsource_data[0]
nrows, ncols = pathloss_array.shape
_, data_ncols = input_model.data.shape
correction = np.ones(data_ncols, dtype=np.float32)
crpix1 = aperture.pointsource_wcs.crpix1
crval1 = aperture.pointsource_wcs.crval1
cdelt1 = aperture.pointsource_wcs.cdelt1
pupil_wheel_index = crpix1 + (pupil_wheel_position - crval1) / cdelt1 - 1
if pupil_wheel_index < 0 or pupil_wheel_index > (ncols - 2):
log.info("Pupil Wheel position outside reference file coverage")
log.info("Setting pathloss correction to 1.0")
else:
ix = int(pupil_wheel_index)
dx = pupil_wheel_index - ix
crpix2 = aperture.pointsource_wcs.crpix2
crval2 = aperture.pointsource_wcs.crval2
cdelt2 = aperture.pointsource_wcs.cdelt2
for row in range(data_ncols):
row_1indexed = row + 1
refrow_index = math.floor(crpix2 + (row_1indexed - crval2) / cdelt2 - 0.5)
if refrow_index < 0 or refrow_index > (nrows - 1):
correction[row] = 1.0
else:
correction[row] = (1.0 - dx) * pathloss_array[refrow_index, ix] + \
dx * pathloss_array[refrow_index, ix + 1]
pathloss_2d = np.broadcast_to(correction, input_model.data.shape)
output_model.data /= pathloss_2d
output_model.err /= pathloss_2d
output_model.var_poisson /= pathloss_2d**2
output_model.var_rnoise /= pathloss_2d**2
if output_model.var_flat is not None and np.size(output_model.var_flat) > 0:
output_model.var_flat /= pathloss_2d**2
output_model.pathloss = pathloss_2d
output_model.meta.cal_step.pathloss = 'COMPLETE'
return output_model
def do_correction(input_model, pathloss_model):
"""
Short Summary
-------------
Execute all tasks for Path Loss Correction
Parameters
----------
input_model : data model object
science data to be corrected
pathloss_model : pathloss model object
pathloss correction data
Returns
-------
output_model : data model object
Science data with pathloss extensions added
"""
exp_type = input_model.meta.exposure.type
log.info(exp_type)
output_model = input_model.copy()
if exp_type == 'NRS_MSASPEC':
slit_number = 0
for slit in output_model.slits:
slit_number = slit_number + 1
log.info('Working on slit {}'.format(slit_number))
size = slit.data.size
# That has data.size > 0
if size > 0:
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
aperture = get_aperture_from_model(pathloss_model, nshutters)
if aperture is not None:
(wavelength_pointsource,
pathloss_pointsource_vector,
is_inside_slitlet) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slitlet:
# Wavelengths in the reference file are in meters,
#need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the pathloss 2D correction
if is_pointsource(slit.source_type):
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
else:
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Apply the pathloss 2D correction and attach to datamodel
slit.data /= pathloss_2d
slit.err /= pathloss_2d
slit.var_poisson /= pathloss_2d**2
slit.var_rnoise /= pathloss_2d**2
slit.pathloss = pathloss_2d
else:
log.warning("Source is outside slit. Skipping "
"pathloss correction for slit {}".format(slit_number))
else:
log.warning("Cannot find matching pathloss model for slit "
"with {} shutters, skipping pathloss correction for this "
"slit".format(nshutters))
continue
else:
log.warning("Slit has data size = {}, skipping "
"pathloss correction for this slitlet".format(size))
output_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']:
slit_number = 0
is_inside_slit = True
# For each slit
for slit in output_model.slits:
log.info(slit.name)
slit_number = slit_number + 1
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
aperture = get_aperture_from_model(pathloss_model, slit.name)
if aperture is not None:
log.info("Using aperture {}".format(aperture.name))
(wavelength_pointsource,
pathloss_pointsource_vector,
is_inside_slit) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slit:
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the pathloss 2D correction
if is_pointsource(slit.source_type):
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
else:
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Apply the pathloss 2D correction and attach to datamodel
slit.data /= pathloss_2d
slit.err /= pathloss_2d
slit.var_poisson /= pathloss_2d**2
slit.var_rnoise /= pathloss_2d**2
slit.pathloss = pathloss_2d
else:
log.warning("Source is outside slit. Skipping "
"pathloss correction for slit {}".format(slit.name))
else:
log.warning("Cannot find matching pathloss model for aperture {} "
"skipping pathloss correction for this slit".format(slit.name))
continue
output_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type == 'NRS_IFU':
# IFU targets are always inside slit
# Get centering
xcenter, ycenter = get_center(exp_type, None)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
aperture = pathloss_model.apertures[0]
(wavelength_pointsource,
pathloss_pointsource_vector,
dummy) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
# Create the 2-d pathloss arrays, initialize with NaNs
wavelength_array = np.zeros(input_model.shape, dtype=np.float32)
wavelength_array.fill(np.nan)
for slice in NIRSPEC_IFU_SLICES:
slice_wcs = nirspec.nrs_wcs_set_input(input_model, slice)
x, y = wcstools.grid_from_bounding_box(slice_wcs.bounding_box)
xmin = int(x.min())
xmax = int(x.max())
ymin = int(y.min())
ymax = int(y.max())
ra, dec, wavelength = slice_wcs(x, y)
wavelength_array[ymin:ymax+1, xmin:xmax+1] = wavelength
# Compute the pathloss 2D correction
if is_pointsource(input_model.meta.target.source_type):
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
else:
pathloss_2d = interpolate_onto_grid(
wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Apply the pathloss 2D correction and attach to datamodel
output_model.data /= pathloss_2d
output_model.err /= pathloss_2d
output_model.var_poisson /= pathloss_2d**2
output_model.var_rnoise /= pathloss_2d**2
output_model.pathloss = pathloss_2d
# This might be useful to other steps
output_model.wavelength = wavelength_array
output_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type == 'NIS_SOSS':
"""NIRISS SOSS pathloss correction is basically a correction for the
flux from the 2nd and 3rd order dispersion that falls outside the
subarray aperture. The correction depends
on the pupil wheel position and column number (or wavelength). The
simple option is to do the correction by column number, then the only
interpolation needed is a 1-d interpolation into the pupil wheel position
dimension."""
# Omit correction if this is a TSO observation
if input_model.meta.visit.tsovisit:
log.warning("NIRISS SOSS TSO observations skip the pathloss step")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
pupil_wheel_position = input_model.meta.instrument.pupil_position
if pupil_wheel_position is None:
log.warning("Unable to get pupil wheel position from PWCPOS keyword "
"for {}".format(input_model.meta.filename))
log.warning("Pathloss correction skipped")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
subarray = input_model.meta.subarray.name
# Get the aperture from the reference file that matches the subarray
aperture = get_aperture_from_model(pathloss_model, subarray)
if aperture is None:
log.warning("Unable to get Aperture from reference file "
"for subarray {}".format(subarray))
log.warning("Pathloss correction skipped")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
else:
log.info("Aperture {} selected from reference file".format(aperture.name))
pathloss_array = aperture.pointsource_data[0]
nrows, ncols = pathloss_array.shape
_, data_ncols = input_model.data.shape
correction = np.ones(data_ncols, dtype=np.float32)
crpix1 = aperture.pointsource_wcs.crpix1
crval1 = aperture.pointsource_wcs.crval1
cdelt1 = aperture.pointsource_wcs.cdelt1
pupil_wheel_index = crpix1 + (pupil_wheel_position - crval1) / cdelt1 - 1
if pupil_wheel_index < 0 or pupil_wheel_index > (ncols - 2):
log.info("Pupil Wheel position outside reference file coverage")
log.info("Setting pathloss correction to 1.0")
else:
ix = int(pupil_wheel_index)
dx = pupil_wheel_index - ix
crpix2 = aperture.pointsource_wcs.crpix2
crval2 = aperture.pointsource_wcs.crval2
cdelt2 = aperture.pointsource_wcs.cdelt2
for row in range(data_ncols):
row_1indexed = row + 1
refrow_index = math.floor(crpix2 + (row_1indexed - crval2) / cdelt2 - 0.5)
if refrow_index < 0 or refrow_index > (nrows - 1):
correction[row] = 1.0
else:
correction[row] = (1.0 - dx) * pathloss_array[refrow_index, ix] + \
dx * pathloss_array[refrow_index, ix + 1]
pathloss_2d = np.broadcast_to(correction, input_model.data.shape)
output_model.data /= pathloss_2d
output_model.err /= pathloss_2d
output_model.var_poisson /= pathloss_2d**2
output_model.var_rnoise /= pathloss_2d**2
output_model.pathloss = pathloss_2d
output_model.meta.cal_step.pathloss = 'COMPLETE'
return output_model
def do_correction(input_model, pathloss_model):
"""
Short Summary
-------------
Execute all tasks for Path Loss Correction
Parameters
----------
input_model : data model object
science data to be corrected
pathloss_model : pathloss model object
pathloss correction data
Returns
-------
output_model : data model object
Corrected science data with pathloss extensions added
"""
exp_type = input_model.meta.exposure.type
log.info(f'Input exposure type is {exp_type}')
output_model = input_model.copy()
# NIRSpec MOS data
if exp_type == 'NRS_MSASPEC':
slit_number = 0
# Loop over all MOS slitlets
for slit in output_model.slits:
slit_number = slit_number + 1
log.info(f'Working on slit {slit_number}')
size = slit.data.size
# Only work on slits with data.size > 0
if size > 0:
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
aperture = get_aperture_from_model(pathloss_model, nshutters)
if aperture is not None:
(wavelength_pointsource, pathloss_pointsource_vector,
is_inside_slitlet) = calculate_pathloss_vector(
aperture.pointsource_data, aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource, pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(
aperture.uniform_data, aperture.uniform_wcs, xcenter,
ycenter)
if is_inside_slitlet:
# Wavelengths in the reference file are in meters,
# need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the point source pathloss 2D correction
pathloss_2d_ps = interpolate_onto_grid(
wavelength_array, wavelength_pointsource,
pathloss_pointsource_vector)
# Compute the uniform source pathloss 2D correction
pathloss_2d_un = interpolate_onto_grid(
wavelength_array, wavelength_uniformsource,
pathloss_uniform_vector)
# Use the appropriate correction for this slit
if is_pointsource(slit.source_type):
pathloss_2d = pathloss_2d_ps
else:
pathloss_2d = pathloss_2d_un
# Apply the pathloss 2D correction and attach to datamodel
slit.data /= pathloss_2d
slit.err /= pathloss_2d
slit.var_poisson /= pathloss_2d**2
slit.var_rnoise /= pathloss_2d**2
if slit.var_flat is not None and np.size(
slit.var_flat) > 0:
slit.var_flat /= pathloss_2d**2
slit.pathloss_point = pathloss_2d_ps
slit.pathloss_uniform = pathloss_2d_un
else:
log.warning(
"Source is outside slit. Skipping "
f"pathloss correction for slit {slit_number}")
else:
log.warning(
"Cannot find matching pathloss model for slit with"
f"{nshutters} shutters")
log.warning("Skipping pathloss correction for this slit")
continue
else:
log.warning(f"Slit has data size = {size}")
log.warning("Skipping pathloss correction for this slitlet")
# Set step status to complete
output_model.meta.cal_step.pathloss = 'COMPLETE'
# NIRSpec fixed-slit data
elif exp_type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']:
slit_number = 0
is_inside_slit = True
# Loop over all slits contained in the input
for slit in output_model.slits:
log.info(f'Working on slit {slit.name}')
slit_number = slit_number + 1
# Get centering
xcenter, ycenter = get_center(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors for the source position
# Get the aperture from the reference file that matches the slit
aperture = get_aperture_from_model(pathloss_model, slit.name)
if aperture is not None:
log.info(f'Using aperture {aperture.name}')
(wavelength_pointsource, pathloss_pointsource_vector,
is_inside_slit) = calculate_pathloss_vector(
aperture.pointsource_data, aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource, pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slit:
# Wavelengths in the reference file are in meters,
# need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
wavelength_array = slit.wavelength
# Compute the point source pathloss 2D correction
pathloss_2d_ps = interpolate_onto_grid(
wavelength_array, wavelength_pointsource,
pathloss_pointsource_vector)
# Compute the uniform source pathloss 2D correction
pathloss_2d_un = interpolate_onto_grid(
wavelength_array, wavelength_uniformsource,
pathloss_uniform_vector)
# Use the appropriate correction for this slit
if is_pointsource(slit.source_type):
pathloss_2d = pathloss_2d_ps
else:
pathloss_2d = pathloss_2d_un
# Apply the pathloss 2D correction and attach to datamodel
slit.data /= pathloss_2d
slit.err /= pathloss_2d
slit.var_poisson /= pathloss_2d**2
slit.var_rnoise /= pathloss_2d**2
if slit.var_flat is not None and np.size(
slit.var_flat) > 0:
slit.var_flat /= pathloss_2d**2
slit.pathloss_point = pathloss_2d_ps
slit.pathloss_uniform = pathloss_2d_un
else:
log.warning('Source is outside slit. Skipping '
f'pathloss correction for slit {slit.name}')
else:
log.warning(
f'Cannot find matching pathloss model for {slit.name}')
log.warning('Skipping pathloss correction for this slit')
continue
# Set step status to complete
output_model.meta.cal_step.pathloss = 'COMPLETE'
# NIRSpec IFU
elif exp_type == 'NRS_IFU':
# IFU targets are always inside slit
# Get centering
xcenter, ycenter = get_center(exp_type, None)
# Calculate the 1-d wavelength and pathloss vectors for the source position
aperture = pathloss_model.apertures[0]
(wavelength_pointsource, pathloss_pointsource_vector,
dummy) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs, xcenter,
ycenter)
(wavelength_uniformsource, pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs, xcenter,
ycenter)
# Wavelengths in the reference file are in meters;
# need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
# Create the 2-d wavelength arrays, initialize with NaNs
wavelength_array = np.zeros(input_model.shape, dtype=np.float32)
wavelength_array.fill(np.nan)
for slice in NIRSPEC_IFU_SLICES:
slice_wcs = nirspec.nrs_wcs_set_input(input_model, slice)
x, y = wcstools.grid_from_bounding_box(slice_wcs.bounding_box)
ra, dec, wavelength = slice_wcs(x, y)
valid = ~np.isnan(wavelength)
x = x[valid]
y = y[valid]
wavelength_array[y.astype(int), x.astype(int)] = wavelength[valid]
# Compute the point source pathloss 2D correction
pathloss_2d_ps = interpolate_onto_grid(wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
# Compute the uniform source pathloss 2D correction
pathloss_2d_un = interpolate_onto_grid(wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
# Use the appropriate correction for the source type
if is_pointsource(input_model.meta.target.source_type):
pathloss_2d = pathloss_2d_ps
else:
pathloss_2d = pathloss_2d_un
# Apply the pathloss 2D correction and attach to datamodel
output_model.data /= pathloss_2d
output_model.err /= pathloss_2d
output_model.var_poisson /= pathloss_2d**2
output_model.var_rnoise /= pathloss_2d**2
if output_model.var_flat is not None and np.size(
output_model.var_flat) > 0:
output_model.var_flat /= pathloss_2d**2
output_model.pathloss_point = pathloss_2d_ps
output_model.pathloss_uniform = pathloss_2d_un
# This might be useful to other steps
output_model.wavelength = wavelength_array
# Set the step status to complete
output_model.meta.cal_step.pathloss = 'COMPLETE'
# NIRISS SOSS
elif exp_type == 'NIS_SOSS':
"""NIRISS SOSS pathloss correction is basically a correction for the
flux from the 2nd and 3rd order dispersion that falls outside the
subarray aperture. The correction depends on the pupil wheel position
and column number (or wavelength). The simple option is to do the
correction by column number, then the only interpolation needed is a
1-d interpolation into the pupil wheel position dimension.
"""
# Omit correction if this is a TSO observation
if input_model.meta.visit.tsovisit:
log.warning("NIRISS SOSS TSO observations skip the pathloss step")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
# Get the pupil wheel position
pupil_wheel_position = input_model.meta.instrument.pupil_position
if pupil_wheel_position is None:
log.warning(
'Unable to get pupil wheel position from PWCPOS keyword '
f'for {input_model.meta.filename}')
log.warning("Pathloss correction skipped")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
# Get the aperture from the reference file that matches the subarray
subarray = input_model.meta.subarray.name
aperture = get_aperture_from_model(pathloss_model, subarray)
if aperture is None:
log.warning('Unable to get Aperture from reference file '
f'for subarray {subarray}')
log.warning("Pathloss correction skipped")
output_model.meta.cal_step.pathloss = 'SKIPPED'
return output_model
else:
log.info(f'Aperture {aperture.name} selected from reference file')
# Set up pathloss correction array
pathloss_array = aperture.pointsource_data[0]
nrows, ncols = pathloss_array.shape
_, data_ncols = input_model.data.shape
correction = np.ones(data_ncols, dtype=np.float32)
crpix1 = aperture.pointsource_wcs.crpix1
crval1 = aperture.pointsource_wcs.crval1
cdelt1 = aperture.pointsource_wcs.cdelt1
pupil_wheel_index = crpix1 + (pupil_wheel_position -
crval1) / cdelt1 - 1
if pupil_wheel_index < 0 or pupil_wheel_index > (ncols - 2):
log.warning("Pupil Wheel position outside reference file coverage")
log.warning("Setting pathloss correction to 1.0")
else:
ix = int(pupil_wheel_index)
dx = pupil_wheel_index - ix
crpix2 = aperture.pointsource_wcs.crpix2
crval2 = aperture.pointsource_wcs.crval2
cdelt2 = aperture.pointsource_wcs.cdelt2
for row in range(data_ncols):
row_1indexed = row + 1
refrow_index = math.floor(crpix2 +
(row_1indexed - crval2) / cdelt2 -
0.5)
if refrow_index < 0 or refrow_index > (nrows - 1):
correction[row] = 1.0
else:
correction[row] = (1.0 - dx) * pathloss_array[refrow_index, ix] + \
dx * pathloss_array[refrow_index, ix + 1]
# Create and apply the 2D correction
pathloss_2d = np.broadcast_to(correction, input_model.data.shape)
output_model.data /= pathloss_2d
output_model.err /= pathloss_2d
output_model.var_poisson /= pathloss_2d**2
output_model.var_rnoise /= pathloss_2d**2
if output_model.var_flat is not None and np.size(
output_model.var_flat) > 0:
output_model.var_flat /= pathloss_2d**2
output_model.pathloss_point = pathloss_2d
# Set step status to complete
output_model.meta.cal_step.pathloss = 'COMPLETE'
return output_model
def pathtest(step_input_filename,
reffile,
comparison_filename,
writefile=True,
show_figs=True,
save_figs=False,
threshold_diff=1.0e-7,
debug=False):
"""
This function calculates the difference between the pipeline and
the calculated pathloss values. The functions use the output of
the compute_world_coordinates.py script.
Args:
step_input_filename: str, name of the output fits file from
the sourcetype step (with full path)
reffile: str, path to the pathloss MOS reference fits file
comparison_filename: str, path to comparison pipeline pathloss file
writefile: boolean, if True writes the fits files of the calculated
pathloss and difference images
show_figs: boolean, whether to show plots or not
save_figs: boolean, save the plots (the 3 plots can be saved or
not independently with the function call)
plot_name: string, desired name (if name is not given,
the plot function will name the plot by default)
threshold_diff: float, threshold difference between pipeline output
and ESA file
debug: boolean, if true, print statements will show on-screen
Returns:
- 1 plot, if told to save and/or show them.
- median_diff: Boolean, True if smaller or equal to threshold.
- log_msgs: list, all print statements are captured in this variable
"""
log_msgs = []
# start the timer
pathtest_start_time = time.time()
# get info from the input sourcetype file header
msg = 'step_input_filename=' + step_input_filename
print(msg)
log_msgs.append(msg)
exptype = fits.getval(step_input_filename, "EXP_TYPE", 0)
grat = fits.getval(step_input_filename, "GRATING", 0)
filt = fits.getval(step_input_filename, "FILTER", 0)
msg = "pathloss file: Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype
print(msg)
log_msgs.append(msg)
msg = "Using reference file: " + reffile
print(msg)
log_msgs.append(msg)
if writefile:
# create fits list to hold the calculated pathloss values for each slit
hdu0 = fits.PrimaryHDU()
outfile = fits.HDUList()
outfile.append(hdu0)
# create fits list to hold pipeline-calculated difference values
hdu0 = fits.PrimaryHDU()
compfile = fits.HDUList()
compfile.append(hdu0)
# list to determine if pytest is passed or not
total_test_result = []
# get all the science extensions
is_point_source = True
print("Retrieving exensions")
ps_uni_ext_list = get_mos_ps_uni_extensions(reffile, is_point_source)
# get files
print("""Checking if files exist & obtaining datamodels.
This takes a few minutes...""")
if os.path.isfile(comparison_filename):
if debug:
msg = 'Comparison file does exist.'
print(msg)
else:
result_msg = """Comparison file does NOT exist.
Pathloss test will be skipped."""
print(result_msg)
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the comparison data model
pathloss_pipe = datamodels.open(comparison_filename)
if debug:
print('Retrieved comparison datamodel.')
if os.path.isfile(step_input_filename):
if debug:
print('Input file does exist.')
else:
result_msg = 'Input file does NOT exist. Skipping pathloss test.'
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the input data model
pl = datamodels.open(step_input_filename)
if debug:
print('got input datamodel!')
# loop through the slits
msg = "Looping through the slits... "
print(msg)
log_msgs.append(msg)
slit_val = 1
for slit, pipe_slit in zip(pl.slits, pathloss_pipe.slits):
try:
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
if is_point_source:
if nshutters == 3:
shutter_key = "MOS1x3"
elif nshutters == 1:
shutter_key = "MOS1x1"
ext = ps_uni_ext_list[0][shutter_key]
print("Retrieved point source extension {}".format(ext))
if is_point_source is False:
if nshutters == 1:
shutter_key = "MOS1x1"
elif nshutters == 3:
shutter_key = "MOS1x3"
ext = ps_uni_ext_list[1][shutter_key]
print("Retrieved extended source extension {}".format(ext))
except KeyError:
print("Unable to retrieve extension. Using 1, but may be 5")
ext = 1 # or 5 (1: one slit open. 5: three adjacent slits open)
mode = "MOS"
slit_id = pipe_slit.name
print('Working with slitlet ', slit_id)
if slit.name == slit_id:
msg = """Slitlet name in fits file previous to pathloss
and in pathloss output file are the same."""
log_msgs.append(msg)
print(msg)
else:
msg = """* Missmatch of slitlet names in fits file previous
to pathloss and in pathloss output file. Skipping test."""
result = 'skip'
log_msgs.append(msg)
return result, msg, log_msgs
wcs_obj = slit.meta.wcs
# get the wavelength
x, y = wcstools.grid_from_bounding_box(wcs_obj.bounding_box,
step=(1, 1),
center=True)
ra, dec, wave = wcs_obj(x, y) # wave is in microns
# get positions of source in file:
slit_x = slit.source_xpos
slit_y = slit.source_ypos * (
-1
) # Scaling introduced for time being because error in assign_wcs transformation
if debug:
print("slit_x, slit_y (" + str(slit_x) + ", " + str(slit_y) + ")")
ref_ext = fits.getdata(reffile, ext)
hdul = fits.open(reffile)
# plcor_ref = hdul[1].data
if debug:
print("ref_ext.shape", ref_ext.shape)
w = wcs.WCS(hdul[1].header)
# make cube
w1, y1, x1 = np.mgrid[:ref_ext.shape[0], :ref_ext.shape[1], :ref_ext.
shape[2]]
slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0)
comp_sci = pipe_slit.data
previous_sci = slit.data
pipe_correction = pipe_slit.pathloss
# Set up source position manually to test correction at nonzero point:
# pipe_x = -0.2
# pipe_y = -0.2
# slit_x = pipe_x
# slit_y = pipe_y
# print("""WARNING: Using manually set slit_x and slit_y: ({}, {})!
# The pipeline correction will not use manually set values
# and thus the residuals will change""".format(slit_x, slit_y))
wave_sci = wave * 10**(-6) # microns --> meters
wave_sci_flat = wave_sci.reshape(wave_sci.size)
wave_ref_flat = wave_ref.reshape(wave_ref.size)
ref_xy = np.column_stack((slitx_ref.reshape(slitx_ref.size),
slity_ref.reshape(slitx_ref.size)))
correction_list = [(get_corr_val(lambda_val, wave_ref, ref_ext, ref_xy,
slit_x, slit_y))
for lambda_val in wave_ref_flat]
correction_array = np.asarray(correction_list)
# Option to test alternative interpolation method:
# first_interp_method='linear'
# second_interp_method = 'linear'
# if first_interp_method == 'linear':
# correction_array_cubic = correction_array
# else:
# correction_list_cubic = [(get_corr_val_cubic(lambda_val, wave_ref, ref_ext,
# ref_xy, slit_x, slit_y, first_interp_method)) for lambda_val in wave_ref_flat]
# correction_array_cubic = np.asarray(correction_list_cubic)
lambda_array = wave_ref_flat
# get correction value for each pixel
corr_vals = np.interp(wave_sci_flat, lambda_array, correction_array)
# corr_vals_cubic = np.interp(wave_sci_flat, lambda_array, correction_array_cubic)
corr_vals = corr_vals.reshape(wave_sci.shape)
# corr_vals_cubic = corr_vals_cubic.reshape(wave_sci.shape)
corrected_array = previous_sci / corr_vals
# Plots:
step_input_filepath = step_input_filename.replace(".fits", "")
# my correction values
fig = plt.figure(figsize=(12, 10))
plt.subplot(221)
norm = ImageNormalize(corr_vals)
plt.imshow(corr_vals,
norm=norm,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Calculated Correction')
plt.colorbar()
# pipe correction
plt.subplot(222)
norm = ImageNormalize(pipe_correction)
plt.imshow(pipe_correction,
norm=norm,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.title("Pipeline Correction")
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.colorbar()
# residuals (pipeline correction-my correction)
corr_residuals = pipe_correction - corr_vals
plt.subplot(223)
norm = ImageNormalize(corr_residuals)
plt.imshow(corr_residuals,
norm=norm,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Correction residuals')
plt.colorbar()
# Calculated Corrected Array
plt.subplot(224)
norm = ImageNormalize(corrected_array)
plt.imshow(corrected_array,
norm=norm,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Calculated Corrected Array')
plt.colorbar()
# #cubic v linear
# plt.subplot(324)
# norm = ImageNormalize(corr_vals_cubic-corr_vals)
# plt.imshow(wave_sci, vmin = -0.00025, vmax=0.00025, aspect=10.0, origin='lower',
# cmap='viridis')
# plt.xlabel('dispersion in pixels')
# plt.ylabel('y in pixels')
# plt.title('corr_vals_1+'+first_interp_method+'2'+second_interp_method+'-corr_vals_linear')
# plt.colorbar()
fig.suptitle("MOS PS at ({}, {}) Pathloss Correction Testing Slit ".
format(slit_x, slit_y) + str(slit_id))
if save_figs:
plt_name = step_input_filepath + "_Pathloss_test_slitlet_" + str(
mode) + "_" + str(slit_val) + ".png"
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
if show_figs:
plt.show()
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
if debug:
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
if debug:
print(msg)
log_msgs.append(msg)
plt.close()
# do not overwrite plots
slit_val = slit_val + 1
# create fits file to hold the calculated pathloss for each slit
if writefile:
msg = "Saving the fits files with the calculated pathloss for each slit..."
print(msg)
log_msgs.append(msg)
# this is the file to hold the image of pipeline-calculated difference values
outfile_ext = fits.ImageHDU(corr_vals, name=slit_id)
outfile.append(outfile_ext)
# this is the file to hold the image of pipeline-calculated difference values
compfile_ext = fits.ImageHDU(corr_residuals, name=slit_id)
compfile.append(compfile_ext)
if corr_residuals[~np.isnan(corr_residuals)].size == 0:
msg1 = """Unable to calculate statistics because difference
array has all values as NaN. Test will be set to FAILED.
"""
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
# delfg_mean, delfg_median, delfg_std = np.nan, np.nan, np.nan
# stats = [delfg_mean, delfg_median, delfg_std]
else:
msg = "Calculating statistics... "
print(msg)
log_msgs.append(msg)
# ignore outliers:
corr_residuals = corr_residuals[np.where(
(corr_residuals != 999.0)
& (corr_residuals < 0.1)
& (corr_residuals > -0.1)
& ~np.isnan(corr_residuals))]
if corr_residuals.size == 0:
msg1 = """Unable to calculate statistics because
difference array has all outlier values.
Test will be set to FAILED."""
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
stats_and_strings = auxfunc.print_stats(corr_residuals,
"Difference",
float(threshold_diff),
abs=True)
stats, stats_print_strings = stats_and_strings
corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats
for msg in stats_print_strings:
log_msgs.append(msg)
# This is the key argument for the assert pytest function
median_diff = False
if abs(corr_residuals_median) <= float(threshold_diff):
median_diff = True
if median_diff:
test_result = "PASSED"
else:
test_result = "FAILED"
msg = " *** Result of the test: " + test_result + "\n"
print(msg)
log_msgs.append(msg)
total_test_result.append(test_result)
if writefile:
outfile_name = step_input_filename.replace(
"srctype", "MOS_PS_calcuated_FS_UNI_pathloss")
compfile_name = step_input_filename.replace(
"srctype", "MOS_PS_comparison_FS_UNI_pathloss")
# create the fits list to hold the calculated flat values for each slit
outfile.writeto(outfile_name, overwrite=True)
# this is the file to hold the image of pipeline-calculated difference values
compfile.writeto(compfile_name, overwrite=True)
msg = "\nFits file with calculated pathloss values of each slit saved as: "
print(msg)
log_msgs.append(msg)
print(outfile_name)
log_msgs.append(outfile_name)
msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: "
print(msg)
log_msgs.append(msg)
print(compfile_name)
log_msgs.append(compfile_name)
# If all tests passed pytest will be marked PASSED, else FAILED
FINAL_TEST_RESULT = False
for t in total_test_result:
if t == "FAILED":
FINAL_TEST_RESULT = False
break
else:
FINAL_TEST_RESULT = True
if FINAL_TEST_RESULT:
msg = "\n *** Final pathloss test result reported as PASSED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "All slits PASSED path_loss test."
else:
msg = "\n *** Final pathloss test result reported as FAILED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "One or more slits FAILED path_loss test."
# end the timer
pathloss_end_time = time.time() - pathtest_start_time
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time / 60.0 # in minutes
pathloss_tot_time = "* Script MSA.py took ", repr(
pathloss_end_time) + " minutes to finish."
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time / 60. # in hours
pathloss_tot_time = "* Script MSA.py took ", repr(
pathloss_end_time) + " hours to finish."
else:
pathloss_tot_time = "* Script MSA.py took ", repr(
pathloss_end_time) + " seconds to finish."
print(pathloss_tot_time)
log_msgs.append(pathloss_tot_time)
return FINAL_TEST_RESULT, result_msg, log_msgs
def do_correction(input_model, pathloss_model):
"""
Short Summary
-------------
Execute all tasks for Path Loss Correction
Parameters
----------
input_model: data model object
science data to be corrected
pathloss_model: pathloss model object
pathloss correction data
Returns
-------
output_model: data model object
Science data with pathloss extensions added
"""
exp_type = input_model.meta.exposure.type
log.info(exp_type)
if exp_type == 'NRS_MSASPEC':
slit_number = 0
# For each slit
for slit in input_model.slits:
slit_number = slit_number + 1
log.info('Working on slit %d' % slit_number)
size = slit.data.size
# That has data.size > 0
if size > 0:
# Get centering
xcenter, ycenter = getCenter(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
aperture = getApertureFromModel(pathloss_model, nshutters)
if aperture is not None:
wavelength_pointsource, pathloss_pointsource_vector = \
calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
wavelength_uniformsource, pathloss_uniform_vector = \
calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
#
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
slit.pathloss_pointsource = pathloss_pointsource_vector
slit.wavelength_pointsource = wavelength_pointsource
slit.pathloss_uniformsource = pathloss_uniform_vector
slit.wavelength_uniformsource = wavelength_uniformsource
else:
log.warning(
"Cannot find matching pathloss model for slit with size %d"
% nshutters)
continue
input_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']:
slit_number = 0
# For each slit
for slit in input_model.slits:
log.info(slit.name)
slit_number = slit_number + 1
# Get centering
xcenter, ycenter = getCenter(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
aperture = getApertureFromModel(pathloss_model, slit.name)
if aperture is not None:
log.info("Using aperture {0}".format(aperture.name))
wavelength_pointsource, pathloss_pointsource_vector = \
calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
wavelength_uniformsource, pathloss_uniform_vector = \
calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
#
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
slit.pathloss_pointsource = pathloss_pointsource_vector
slit.wavelength_pointsource = wavelength_pointsource
slit.pathloss_uniformsource = pathloss_uniform_vector
slit.wavelength_uniformsource = wavelength_uniformsource
else:
log.warning(
"Cannot find matching pathloss model for aperture %s" %
slit.name)
continue
input_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type == 'NRS_IFU':
# Get centering
xcenter, ycenter = getCenter(exp_type, None)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
aperture = pathloss_model.apertures[0]
wavelength_pointsource, pathloss_pointsource_vector = \
calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
wavelength_uniformsource, pathloss_uniform_vector = \
calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
input_model.wavelength_pointsource = wavelength_pointsource
input_model.pathloss_pointsource = pathloss_pointsource_vector
input_model.wavelength_uniformsource = wavelength_uniformsource
input_model.pathloss_uniformsource = pathloss_uniform_vector
input_model.meta.cal_step.pathloss = 'COMPLETE'
return input_model.copy()
def do_correction(input_model, pathloss_model):
"""
Short Summary
-------------
Execute all tasks for Path Loss Correction
Parameters
----------
input_model: data model object
science data to be corrected
pathloss_model: pathloss model object
pathloss correction data
Returns
-------
output_model: data model object
Science data with pathloss extensions added
"""
exp_type = input_model.meta.exposure.type
log.info(exp_type)
if exp_type == 'NRS_MSASPEC':
slit_number = 0
# For each slit
for slit in input_model.slits:
slit_number = slit_number + 1
log.info('Working on slit %d' % slit_number)
size = slit.data.size
# That has data.size > 0
if size > 0:
# Get centering
xcenter, ycenter = getCenter(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
nshutters = util.get_num_msa_open_shutters(slit.shutter_state)
aperture = getApertureFromModel(pathloss_model, nshutters)
if aperture is not None:
(wavelength_pointsource,
pathloss_pointsource_vector,
is_inside_slitlet) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slitlet:
#
# Wavelengths in the reference file are in meters,
#need them to be in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
slit.pathloss_pointsource = pathloss_pointsource_vector
slit.wavelength_pointsource = wavelength_pointsource
slit.pathloss_uniformsource = pathloss_uniform_vector
slit.wavelength_uniformsource = wavelength_uniformsource
#
# Create the 2-d pathloss arrays
wavelength_array = slit.wavelength
pathloss_pointsource_2d = interpolate_onto_grid(wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
pathloss_uniformsource_2d = interpolate_onto_grid(wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
slit.pathloss_pointsource2d = pathloss_pointsource_2d
slit.pathloss_uniformsource2d = pathloss_uniformsource_2d
else:
log.warning("Source is outside slitlet, skipping pathloss correction for this slitlet")
else:
log.warning("Cannot find matching pathloss model for slit with size %d, skipping pathloss correction for this slitlet" % nshutters)
continue
else:
log.warning("Slit has data size = {}, skipping pathloss correction for this slitlet".format(size))
input_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']:
slit_number = 0
is_inside_slit = True
# For each slit
for slit in input_model.slits:
log.info(slit.name)
slit_number = slit_number + 1
# Get centering
xcenter, ycenter = getCenter(exp_type, slit)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
# Get the aperture from the reference file that matches the slit
aperture = getApertureFromModel(pathloss_model, slit.name)
if aperture is not None:
log.info("Using aperture {0}".format(aperture.name))
(wavelength_pointsource,
pathloss_pointsource_vector,
is_inside_slit) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
if is_inside_slit:
#
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
slit.pathloss_pointsource = pathloss_pointsource_vector
slit.wavelength_pointsource = wavelength_pointsource
slit.pathloss_uniformsource = pathloss_uniform_vector
slit.wavelength_uniformsource = wavelength_uniformsource
#
# Create the 2-d pathloss arrays
wavelength_array = slit.wavelength
pathloss_pointsource_2d = interpolate_onto_grid(wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
pathloss_uniformsource_2d = interpolate_onto_grid(wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
slit.pathloss_pointsource2d = pathloss_pointsource_2d
slit.pathloss_uniformsource2d = pathloss_uniformsource_2d
else:
log.warning("Source is outside slit, skipping pathloss correction for this slit")
else:
log.warning("Cannot find matching pathloss model for aperture %s, skipping pathloss correction for this slit" % slit.name)
continue
input_model.meta.cal_step.pathloss = 'COMPLETE'
elif exp_type == 'NRS_IFU':
# IFU targets are always inside slit
# Get centering
xcenter, ycenter = getCenter(exp_type, None)
# Calculate the 1-d wavelength and pathloss vectors
# for the source position
aperture = pathloss_model.apertures[0]
(wavelength_pointsource,
pathloss_pointsource_vector,
dummy) = calculate_pathloss_vector(aperture.pointsource_data,
aperture.pointsource_wcs,
xcenter, ycenter)
(wavelength_uniformsource,
pathloss_uniform_vector,
dummy) = calculate_pathloss_vector(aperture.uniform_data,
aperture.uniform_wcs,
xcenter, ycenter)
# Wavelengths in the reference file are in meters, need them to be
# in microns
wavelength_pointsource *= 1.0e6
wavelength_uniformsource *= 1.0e6
input_model.wavelength_pointsource = wavelength_pointsource
input_model.pathloss_pointsource = pathloss_pointsource_vector
input_model.wavelength_uniformsource = wavelength_uniformsource
input_model.pathloss_uniformsource = pathloss_uniform_vector
#
# Create the 2-d pathloss arrays, initialize with NaNs
wavelength_array = np.zeros(input_model.shape, dtype=np.float32)
wavelength_array.fill(np.nan)
for slice in NIRSPEC_IFU_SLICES:
slice_wcs = nirspec.nrs_wcs_set_input(input_model, slice)
x, y = wcstools.grid_from_bounding_box(slice_wcs.bounding_box)
xmin = int(x.min())
xmax = int(x.max())
ymin = int(y.min())
ymax = int(y.max())
ra, dec, wavelength = slice_wcs(x, y)
wavelength_array[ymin:ymax+1, xmin:xmax+1] = wavelength
pathloss_pointsource_2d = interpolate_onto_grid(wavelength_array,
wavelength_pointsource,
pathloss_pointsource_vector)
pathloss_uniformsource_2d = interpolate_onto_grid(wavelength_array,
wavelength_uniformsource,
pathloss_uniform_vector)
input_model.pathloss_pointsource2d = pathloss_pointsource_2d
input_model.pathloss_uniformsource2d = pathloss_uniformsource_2d
#
# This might be useful to other steps
input_model.wavelength = wavelength_array
input_model.meta.cal_step.pathloss = 'COMPLETE'
return input_model.copy()
