def pathtest(step_input_filename,
reffile,
comparison_filename,
writefile=True,
show_figs=True,
save_figs=False,
threshold_diff=1.0e-7,
debug=False):
"""
This function calculates the difference between the pipeline and
calculated pathloss values.
Args:
step_input_filename: str, full path name of sourcetype output fits file
reffile: str, path to the pathloss FS reference fits files
comparison_filename: str, path to comparison pipeline pathloss file
writefile: boolean, if True writes the fits files of
calculated pathloss and difference images
show_figs: boolean, whether to show plots or not
save_figs: boolean, save the plots
threshold_diff: float, threshold difference between pipeline output
and ESA file
debug: boolean, if true print statements will show on-screen
Returns:
- 1 plot, if told to save and/or show them.
- median_diff: Boolean, True if smaller or equal to threshold.
- log_msgs: list, all print statements are captured in this variable
"""
log_msgs = []
# start the timer
pathtest_start_time = time.time()
# get info from the rate file header
msg = 'step_input_filename=' + step_input_filename
print(msg)
log_msgs.append(msg)
exptype = fits.getval(step_input_filename, "EXP_TYPE", 0)
grat = fits.getval(step_input_filename, "GRATING", 0)
filt = fits.getval(step_input_filename, "FILTER", 0)
msg = "pathloss_file: Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype
print(msg)
log_msgs.append(msg)
# get the reference files
msg = "Using reference file: " + reffile
print(msg)
log_msgs.append(msg)
is_point_source = False
if writefile:
# create the fits list to hold the calculated flat values for each slit
hdu0 = fits.PrimaryHDU()
outfile = fits.HDUList()
outfile.append(hdu0)
# create fits list to hold pipeline-calculated difference values
hdu0 = fits.PrimaryHDU()
compfile = fits.HDUList()
compfile.append(hdu0)
# list to determine if pytest is passed or not
total_test_result = []
# get all the science extensions
if not is_point_source:
ext = 3 # only one option for IFU_UNI
# read in 2D spectra output prior to pathloss:
print("""Checking if files exist and obtaining datamodels.
This takes a few minutes...""")
if os.path.isfile(comparison_filename):
if debug:
print('Comparison file does exist.')
else:
result_msg = "Comparison file does NOT exist. Skipping pathloss test."
print(result_msg)
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the comparison data model
ifu_pipe_model = datamodels.open(comparison_filename)
if debug:
print('got comparison datamodel!')
if os.path.isfile(step_input_filename):
if debug:
print('Input file does exist.')
else:
result_msg = 'Input file does NOT exist. Skipping pathloss test.'
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the input data model
ifu_input_model = datamodels.open(step_input_filename)
if debug:
print('got input datamodel!')
# get slices (instead of using .slit)
pl_ifu_slits = nirspec.nrs_ifu_wcs(ifu_input_model)
print("got input slices")
plcor_ref_ext = fits.getdata(reffile, ext)
hdul = fits.open(reffile)
plcor_ref = hdul[1].data
print("PLCOR_REF.shape", plcor_ref.shape)
w = wcs.WCS(hdul[1].header)
w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref.shape[1], :plcor_ref.
shape[2]]
slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0)
# these are full 2048 * 2048 files:
previous_sci = fits.getdata(step_input_filename, "SCI")
comp_sci = fits.getdata(comparison_filename, "SCI")
pathloss_divided = comp_sci / previous_sci
# set up generals for all plots
font = {'weight': 'normal', 'size': 10}
matplotlib.rc('font', **font)
# loop through the slices
msg = " Looping through the slices... "
print(msg)
log_msgs.append(msg)
slit_list = np.ndarray.tolist(np.arange(0, 30))
for slit, slit_num in zip(pl_ifu_slits, slit_list):
print("working with slice {}".format(slit_num))
x, y = wcstools.grid_from_bounding_box(slit.bounding_box, step=(1, 1))
ra, dec, wave = slit(x, y)
wave_sci = wave * 10**(-6)
corr_vals = np.interp(wave_sci, wave_ref[:, 0, 0], plcor_ref_ext)
box = slit.bounding_box
small_y = box[0][0]
big_y = box[0][1]
small_x = box[1][0]
big_x = box[1][1]
left = int(math.trunc(small_x))
right = int(math.ceil(big_x))
bottom = int(math.trunc(small_y))
top = int(math.ceil(big_y))
full_cut2slice = previous_sci[left:right, bottom:top]
print("SHAPES", full_cut2slice.shape, corr_vals.shape)
if full_cut2slice.shape != corr_vals.shape:
value = 0
while ((((full_cut2slice.shape[0] - corr_vals.shape[0]) != 0) or
((full_cut2slice.shape[1] - corr_vals.shape[1]) != 0))
and value < 7): # can delete second criteria once all pass
if value == 6:
print("WARNING: may be in infinite loop!")
x_amount_off = full_cut2slice.shape[0] - corr_vals.shape[0]
if x_amount_off >= 1:
if x_amount_off % 2 == 0: # need 2 more vals: 1 per side
right = right - 1
left = left + 1
print("ALTERED SHAPE OF SLICE: V1")
value = value + 1
else: # just add one value
left = left + 1
print("ALTERED SHAPE OF SLICE: V2")
value = value + 1
elif x_amount_off <= -1:
if x_amount_off % 2 == 0:
right = right + 1
left = left - 1
print("ALTERED SHAPE OF SLICE: V3")
value = value + 1
else:
left = left - 1
print("ALTERED SHAPE OF SLICE: V4")
value = value + 1
y_amount_off = full_cut2slice.shape[1] - corr_vals.shape[1]
if y_amount_off >= 1:
if y_amount_off % 2 == 0:
bottom = bottom - 1
top = top + 1
print("ALTERED SHAPE OF SLICE: V5")
value = value + 1
else:
bottom = bottom + 1
print("ALTERED SHAPE OF SLICE: V6")
value = value + 1
elif y_amount_off <= -1:
if y_amount_off % 2 == 0:
top = top + 1
bottom = bottom - 1
print("ALTERED SHAPE OF SLICE: V7")
value = value + 1
else:
bottom = bottom - 1
print("ALTERED SHAPE OF SLICE: V8")
value = value + 1
full_cut2slice = previous_sci[left:right, bottom:top]
print("final left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
print("NEW SHAPE OF SLICE: {} and corr_vals.shape: {}".format(
full_cut2slice.shape, corr_vals.shape))
if full_cut2slice.shape != corr_vals.shape:
print("shapes did not match! full_cut2slice: {}, corr_vals {}".
format(full_cut2slice.shape, corr_vals.shape))
continue
corrected_array = full_cut2slice / corr_vals
pipe_correction = pathloss_divided[left:right, bottom:top]
if pipe_correction.shape != corr_vals.shape:
print("shapes did not match! pipe_correction: {}, corr_vals {}".
format(pipe_correction.shape, corr_vals.shape))
continue
prev_sci_slit = previous_sci[left:right, bottom:top]
if prev_sci_slit.shape != corr_vals.shape:
print(
"shapes did not match! prev_sci_slit: {}, corr_vals {}".format(
prev_sci_slit.shape, corr_vals.shape))
continue
comp_sci_slit = comp_sci[left:right, bottom:top]
if comp_sci_slit.shape != corr_vals.shape:
print(
"shapes did not match! comp_sci_slit: {}, corr_vals {}".format(
comp_sci_slit.shape, corr_vals.shape))
continue
# Plots:
step_input_filepath = step_input_filename.replace(".fits", "")
# my correction values
fig = plt.figure(figsize=(15, 15))
plt.subplot(221)
norm = ImageNormalize(corr_vals)
plt.imshow(corr_vals,
vmin=0.999995,
vmax=1.000005,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Calculated Correction')
plt.colorbar()
# pipe correction
plt.subplot(222)
norm = ImageNormalize(pipe_correction)
plt.imshow(pipe_correction,
vmin=0.999995,
vmax=1.000005,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Pipe Correction')
plt.colorbar()
# residuals (pipe correction - my correction)
if pipe_correction.shape == corr_vals.shape:
corr_residuals = pipe_correction - corr_vals
plt.subplot(223)
norm = ImageNormalize(corr_residuals)
plt.imshow(corr_residuals,
vmin=-0.000000005,
vmax=0.000000005,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Correction residuals')
plt.colorbar()
# my science data after pathloss
plt.subplot(224)
norm = ImageNormalize(corrected_array)
plt.imshow(corrected_array,
vmin=0,
vmax=300,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.title('My slit science data after pathloss')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.colorbar()
fig.suptitle("IFU UNI Pathloss Correction Testing")
if show_figs:
plt.show()
if save_figs:
plt_name = step_input_filepath + "_Pathloss_test_IFU_UNI_slit_" + str(
slit_num) + ".png"
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
if debug:
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
if debug:
print(msg)
log_msgs.append(msg)
plt.close()
# create fits file to hold the calculated pathloss for each slit
if writefile:
msg = "Saving the fits files with the calculated pathloss for each slit..."
print(msg)
log_msgs.append(msg)
# this is the file to hold the image of pipeline-calculated difference values
outfile_ext = fits.ImageHDU(corr_vals, name=str(slit_num))
outfile.append(outfile_ext)
# this is the file to hold the image of pipeline-calculated difference values
compfile_ext = fits.ImageHDU(corr_residuals, name=str(slit_num))
compfile.append(compfile_ext)
if corr_residuals[~np.isnan(corr_residuals)].size == 0:
msg1 = " * Unable to calculate statistics because difference array has all values as NaN. " \
"Test will be set to FAILED."
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
msg = "Calculating statistics... "
print(msg)
log_msgs.append(msg)
# ignore outliers:
corr_residuals = corr_residuals[np.where((corr_residuals != 999.0)
& (corr_residuals < 0.1)
& (corr_residuals > -0.1))]
if corr_residuals.size == 0:
msg1 = """Unable to calculate statistics because
difference array has all outlier values.
Test will be set to FAILED."""
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
stats_and_strings = auxfunc.print_stats(corr_residuals,
"Difference",
float(threshold_diff),
abs=True)
stats, stats_print_strings = stats_and_strings
corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats
for msg in stats_print_strings:
log_msgs.append(msg)
# This is the key argument for the assert pytest function
median_diff = False
if abs(corr_residuals_median) <= float(threshold_diff):
median_diff = True
if median_diff:
test_result = "PASSED"
else:
test_result = "FAILED"
msg = " *** Result of the test: " + test_result + "\n"
print(msg)
log_msgs.append(msg)
total_test_result.append(test_result)
if writefile:
outfile_name = step_input_filename.replace(
"srctype", "_calcuated_FS_UNI_pathloss")
compfile_name = step_input_filename.replace(
"srctype", "_comparison_FS_UNI_pathloss")
# create the fits list to hold the calculated flat values for each slit
outfile.writeto(outfile_name, overwrite=True)
# this is the file to hold the image of pipeline-calculated difference values
compfile.writeto(compfile_name, overwrite=True)
msg = "\nFits file with calculated pathloss values of each slit saved as: "
print(msg)
log_msgs.append(msg)
print(outfile_name)
log_msgs.append(outfile_name)
msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: "
print(msg)
log_msgs.append(msg)
print(compfile_name)
log_msgs.append(compfile_name)
# If all tests passed then pytest will be marked as PASSED, else FAILED
FINAL_TEST_RESULT = False
for t in total_test_result:
if t == "FAILED":
FINAL_TEST_RESULT = False
break
else:
FINAL_TEST_RESULT = True
if FINAL_TEST_RESULT:
msg = "\n *** Final result of pathloss test is PASSED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "All slits PASSED path_loss test."
else:
msg = "\n *** Final result of pathloss test is FAILED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "One or more slits FAILED path_loss test."
# end the timer
pathloss_end_time = time.time() - pathtest_start_time
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time / 60.0 # in minutes
pathloss_tot_time = "* Script ifu_uni.py took ", repr(
pathloss_end_time) + " minutes to finish."
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time / 60. # in hours
pathloss_tot_time = "* Script ifu_uni.py took ", repr(
pathloss_end_time) + " hours to finish."
else:
pathloss_tot_time = "* Script ifu_uni.py took ", repr(
pathloss_end_time) + " seconds to finish."
print(pathloss_tot_time)
log_msgs.append(pathloss_tot_time)
return FINAL_TEST_RESULT, result_msg, log_msgs
def flattest(step_input_filename,
dflatref_path=None,
sfile_path=None,
fflat_path=None,
writefile=False,
mk_all_slices_plt=False,
show_figs=True,
save_figs=False,
plot_name=None,
threshold_diff=1.0e-7,
debug=False):
"""
This function calculates the difference between the pipeline and the calculated flat field values.
The functions uses the output of the compute_world_coordinates.py script.
Args:
step_input_filename: str, name of the output fits file from the 2d_extract step (with full path)
dflatref_path: str, path of where the D-flat reference fits files
sfile_path: str, path of where the S-flat reference fits files
fflat_path: str, path of where the F-flat reference fits files
msa_conf_root: str, path to where the MSA configuration fits file lives
writefile: boolean, if True writes the fits files of the calculated flat and difference images
show_figs: boolean, whether to show plots or not
save_figs: boolean, save the plots (the 3 plots can be saved or not independently with the function call)
plot_name: string, desired name (if name is not given, the plot function will name the plot by
default)
threshold_diff: float, threshold difference between pipeline output and ESA file
debug: boolean, if true a series of print statements will show on-screen
Returns:
- 1 plot, if told to save and/or show.
- median_diff: Boolean, True if smaller or equal to 1e-14
- log_msgs: list, all print statements are captured in this variable
"""
log_msgs = []
# start the timer
flattest_start_time = time.time()
# get info from the flat field file
file_path = step_input_filename.replace(
os.path.basename(step_input_filename), "")
det = fits.getval(step_input_filename, "DETECTOR", 0)
exptype = fits.getval(step_input_filename, "EXP_TYPE", 0)
grat = fits.getval(step_input_filename, "GRATING", 0)
filt = fits.getval(step_input_filename, "FILTER", 0)
file_basename = os.path.basename(step_input_filename.replace(".fits", ""))
msg1 = 'step_input_filename=' + step_input_filename
msg2 = "flat_field_file --> Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype
print(msg1)
print(msg2)
log_msgs.append(msg1)
log_msgs.append(msg2)
# read in the on-the-fly flat image
flatfile = step_input_filename.replace("flat_field.fits",
"interpolatedflat.fits")
pipeflat = fits.getdata(flatfile, "SCI")
# get the reference files
msg = "Getting and reading the D-, S-, and F-flats for this specific IFU configuration... "
print(msg)
log_msgs.append(msg)
# D-Flat
dflat_ending = "f_01.03.fits"
dfile = dflatref_path + "_nrs1_" + dflat_ending
if det == "NRS2":
dfile = dfile.replace("nrs1", "nrs2")
msg = "Using D-flat: " + dfile
print(msg)
log_msgs.append(msg)
dfim = fits.getdata(dfile, "SCI") #1)
dfimdq = fits.getdata(dfile, "DQ") #4)
# need to flip/rotate the image into science orientation
ns = np.shape(dfim)
dfim = np.transpose(dfim, (
0, 2,
1)) # keep in mind that 0,1,2 = z,y,x in Python, whereas =x,y,z in IDL
dfimdq = np.transpose(dfimdq)
if det == "NRS2":
# rotate science data by 180 degrees for NRS2
dfim = dfim[..., ::-1, ::-1]
dfimdq = dfimdq[..., ::-1, ::-1]
naxis3 = fits.getval(dfile, "NAXIS3", "SCI") #1)
# get the wavelength values
dfwave = np.array([])
for i in range(naxis3):
keyword = "PFLAT_" + str(i + 1)
dfwave = np.append(dfwave, fits.getval(dfile, keyword, "SCI")) #1))
dfrqe = fits.getdata(dfile, 2)
# S-flat
tsp = exptype.split("_")
mode = tsp[1]
if filt == "F070LP":
flat = "FLAT4"
elif filt == "F100LP":
flat = "FLAT1"
elif filt == "F170LP":
flat = "FLAT2"
elif filt == "F290LP":
flat = "FLAT3"
elif filt == "CLEAR":
flat = "FLAT5"
else:
msg = "No filter correspondence. Exiting the program."
print(msg)
log_msgs.append(msg)
# This is the key argument for the assert pytest function
msg = "Test skiped because there is no flat correspondence for the filter in the data: {}".format(
filt)
median_diff = "skip"
return median_diff, msg
sflat_ending = "f_01.01.fits"
sfile = sfile_path + "_" + grat + "_OPAQUE_" + flat + "_nrs1_" + sflat_ending
if debug:
print("grat = ", grat)
print("flat = ", flat)
print("sfile used = ", sfile)
if det == "NRS2":
sfile = sfile.replace("nrs1", "nrs2")
msg = "Using S-flat: " + sfile
print(msg)
log_msgs.append(msg)
sfim = fits.getdata(sfile, "SCI") #1)
sfimdq = fits.getdata(sfile, "DQ") #3)
# need to flip/rotate image into science orientation
sfim = np.transpose(sfim)
sfimdq = np.transpose(sfimdq)
if det == "NRS2":
# rotate science data by 180 degrees for NRS2
sfim = sfim[..., ::-1, ::-1]
sfimdq = sfimdq[..., ::-1, ::-1]
sfv = fits.getdata(sfile, 5)
# F-Flat
fflat_ending = "_01.01.fits"
if mode in fflat_path:
ffile = fflat_path + "_" + filt + fflat_ending
else:
msg = "Wrong path in for mode F-flat. This script handles mode " + mode + "only."
print(msg)
log_msgs.append(msg)
# This is the key argument for the assert pytest function
result_msg = "Wrong path in for mode F-flat. Test skiped because mode is not IFU."
median_diff = "skip"
return median_diff, result_msg, log_msgs
msg = "Using F-flat: " + ffile
print(msg)
log_msgs.append(msg)
ffv = fits.getdata(ffile, "IFU") #1)
# now go through each pixel in the test data
if writefile:
# create the fits list to hold the calculated flat values for each slit
hdu0 = fits.PrimaryHDU()
outfile = fits.HDUList()
outfile.append(hdu0)
# create the fits list to hold the image of pipeline-calculated difference values
hdu0 = fits.PrimaryHDU()
complfile = fits.HDUList()
complfile.append(hdu0)
# get the datamodel from the assign_wcs output file
assign_wcs_file = step_input_filename.replace("_flat_field.fits",
"_assign_wcs.fits")
model = datamodels.ImageModel(assign_wcs_file)
ifu_slits = nirspec.nrs_ifu_wcs(model)
# loop over the slices
all_delfg_mean, all_delfg_mean_arr, all_delfg_median, all_test_result = [], [], [], []
msg = "\n Now looping through the slices, this may take some time... "
print(msg)
log_msgs.append(msg)
for n_ext, slice in enumerate(ifu_slits):
if n_ext < 10:
pslice = "0" + repr(n_ext)
else:
pslice = repr(n_ext)
msg = "\nWorking with slice: " + pslice
print(msg)
log_msgs.append(msg)
# get the wavelength
# slice.x(y)start are 1-based, turn them to 0-based for extraction
x, y = wcstools.grid_from_bounding_box(slice.bounding_box, (1, 1),
center=True)
ra, dec, wave = slice(x, y)
# get the subwindow origin (technically no subwindows for IFU, but need this for comparing to the
# full frame on-the-fly flat image).
px0 = model.meta.subarray.xstart - 1 + int(
_toindex(slice.bounding_box[0][0])) + 1
py0 = model.meta.subarray.xstart - 1 + int(
_toindex(slice.bounding_box[1][0])) + 1
n_p = np.shape(wave)
nx, ny = n_p[1], n_p[0]
nw = nx * ny
msg = " Subwindow origin: px0=" + repr(px0) + " py0=" + repr(py0)
print(msg)
log_msgs.append(msg)
if debug:
print("n_p = ", n_p)
print("nw = ", nw)
# initialize arrays of the right size
delf = np.zeros([nw]) + 999.0
flatcor = np.zeros([nw]) + 999.0
sffarr = np.zeros([nw])
calc_flat = np.zeros([2048, 2048]) + 999.0
# loop through the wavelengths
msg = " Looping through the wavelngth, this may take a little time ... "
print(msg)
log_msgs.append(msg)
flat_wave = wave.flatten()
wave_shape = np.shape(wave)
for j in range(0, nw):
if np.isfinite(flat_wave[j]): # skip if wavelength is NaN
# get the pixel indeces
jwav = flat_wave[j]
t = np.where(wave == jwav)
pind = [t[0][0] + py0 - 1, t[1][0] + px0 - 1
] # pind =[pixel_y, pixe_x] in python, [x, y] in IDL
if debug:
print('j, jwav, px0, py0 : ', j, jwav, px0, py0)
print('pind[0], pind[1] = ', pind[0], pind[1])
# get the pixel bandwidth **this needs to be modified for prism, since the dispersion is not linear!**
delw = 0.0
if (j != 0) and (int((j - 1) / nx) == int(j / nx)) and (int(
(j + 1) / nx) == int(j / nx)) and np.isfinite(
flat_wave[j + 1]) and np.isfinite(flat_wave[j - 1]):
delw = 0.5 * (flat_wave[j + 1] - flat_wave[j - 1])
if (j == 0) or not np.isfinite(flat_wave[j - 1]) or (int(
(j - 1) / nx) != int(j / nx)):
delw = 0.5 * (flat_wave[j + 1] - flat_wave[j])
if (j == nw - 1) or not np.isfinite(flat_wave[j + 1]) or (int(
(j + 1) / nx) != int(j / nx)):
delw = 0.5 * (flat_wave[j] - flat_wave[j - 1])
if debug:
#print("(j, (j-1), nx, (j-1)/nx, (j+1), (j+1)/nx)", j, (j-1), nx, int((j-1)/nx), (j+1), int((j+1)/nx))
#print("np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1])", np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1]))
#print("flat_wave[j+1], flat_wave[j-1] : ", np.isfinite(flat_wave[j+1]), flat_wave[j+1], flat_wave[j-1])
print("delw = ", delw)
# integrate over D-flat fast vector
dfrqe_wav = dfrqe.field("WAVELENGTH")
dfrqe_rqe = dfrqe.field("RQE")
iw = np.where((dfrqe_wav >= jwav - delw / 2.0)
& (dfrqe_wav <= jwav + delw / 2.0))
if np.size(iw) == 0:
iw = -1
int_tab = auxfunc.idl_tabulate(dfrqe_wav[iw], dfrqe_rqe[iw])
if int_tab == 0:
int_tab = np.interp(dfrqe_wav[iw], dfrqe_wav, dfrqe_rqe)
dff = int_tab
else:
first_dfrqe_wav, last_dfrqe_wav = dfrqe_wav[iw][
0], dfrqe_wav[iw][-1]
dff = int_tab / (last_dfrqe_wav - first_dfrqe_wav)
if debug:
#print("np.shape(dfrqe_wav) : ", np.shape(dfrqe_wav))
#print("np.shape(dfrqe_rqe) : ", np.shape(dfrqe_rqe))
#print("dfimdq[pind[0]][pind[1]] : ", dfimdq[pind[0]][pind[1]])
#print("np.shape(iw) =", np.shape(iw))
#print("np.shape(dfrqe_wav[iw[0]]) = ", np.shape(dfrqe_wav[iw[0]]))
#print("np.shape(dfrqe_rqe[iw[0]]) = ", np.shape(dfrqe_rqe[iw[0]]))
#print("int_tab=", int_tab)
print("np.shape(iw) = ", np.shape(iw))
print("iw = ", iw)
print("dff = ", dff)
# interpolate over D-flat cube
dfs = 1.0
if dfimdq[pind[0], pind[1]] == 0:
dfs = np.interp(jwav, dfwave, dfim[:, pind[0], pind[1]])
# integrate over S-flat fast vector
sfv_wav = sfv.field("WAVELENGTH")
sfv_dat = sfv.field("DATA")
if (jwav < 5.3) and (jwav > 0.6):
iw = np.where((sfv_wav >= jwav - delw / 2.0)
& (sfv_wav <= jwav + delw / 2.0))
if np.size(iw) == 0:
iw = -1
if np.size(iw) > 1:
int_tab = auxfunc.idl_tabulate(sfv_wav[iw],
sfv_dat[iw])
first_sfv_wav, last_sfv_wav = sfv_wav[iw][0], sfv_wav[
iw][-1]
sff = int_tab / (last_sfv_wav - first_sfv_wav)
elif np.size(iw) == 1:
sff = float(sfv_dat[iw])
else:
sff = 999.0
# get s-flat pixel-dependent correction
sfs = 1.0
if sfimdq[pind[0], pind[1]] == 0:
sfs = sfim[pind[0], pind[1]]
if debug:
print("jwav-delw/2.0 = ", jwav - delw / 2.0)
print("jwav+delw/2.0 = ", jwav + delw / 2.0)
print("np.shape(sfv_wav), sfv_wav[-1] = ",
np.shape(sfv_wav), sfv_wav[-1])
print("iw = ", iw)
print("sfv_wav[iw] = ", sfv_wav[iw])
print("int_tab = ", int_tab)
print("first_sfv_wav, last_sfv_wav = ", first_sfv_wav,
last_sfv_wav)
print("sfs = ", sfs)
print("sff = ", sff)
# integrate over f-flat fast vector
# reference file blue cutoff is 1 micron, so need to force solution for shorter wavs
ffv_wav = ffv.field("WAVELENGTH")
ffv_dat = ffv.field("DATA")
fff = 1.0
if jwav - delw / 2.0 >= 1.0:
iw = np.where((ffv_wav >= jwav - delw / 2.0)
& (ffv_wav <= jwav + delw / 2.0))
if np.size(iw) == 0:
iw = -1
if np.size(iw) > 1:
int_tab = auxfunc.idl_tabulate(ffv_wav[iw],
ffv_dat[iw])
first_ffv_wav, last_ffv_wav = ffv_wav[iw][0], ffv_wav[
iw][-1]
fff = int_tab / (last_ffv_wav - first_ffv_wav)
elif np.size(iw) == 1:
fff = float(ffv_dat[iw])
flatcor[j] = dff * dfs * sff * sfs * fff
sffarr[j] = sff
# To visually compare between the pipeline flat and the calculated one (e.g. in ds9), Phil Hodge
# suggested using the following line:
calc_flat[pind[0], pind[1]] = flatcor[j]
# this line writes the calculated flat into a full frame array
# then this new array needs to be written into a file. This part has not been done yet.
# Difference between pipeline and calculated values
delf[j] = pipeflat[pind[0], pind[1]] - flatcor[j]
# Remove all pixels with values=1 (mainly inter-slit pixels) for statistics
if pipeflat[pind[0], pind[1]] == 1:
delf[j] = 999.0
if np.isnan(jwav):
flatcor[j] = 1.0 # no correction if no wavelength
if debug:
print("np.shape(iw) = ", np.shape(iw))
print("fff = ", fff)
print("flatcor[j] = ", flatcor[j])
print("delf[j] = ", delf[j])
# ignore outliers for calculating median
delfg = delf[np.where(delf != 999.0)]
#delfg_median, delfg_std = np.median(delfg), np.std(delfg)
msg = "Flat value differences for slice number: " + pslice
print(msg)
log_msgs.append(msg)
#print(" median = ", delfg_median, " stdev =", delfg_std)
stats_and_strings = auxfunc.print_stats(delfg,
"Flat Difference",
float(threshold_diff),
abs=True)
stats, stats_print_strings = stats_and_strings
delfg_mean, delfg_median, delfg_std = stats
for msg in stats_print_strings:
log_msgs.append(msg)
if debug:
print("np.shape(delf) = ", np.shape(delf))
print("np.shape(delfg) = ", np.shape(delfg))
all_delfg_mean.append(delfg_mean)
all_delfg_median.append(delfg_median)
# make the slice plot
if np.isfinite(delfg_median) and (len(delfg) != 0):
if show_figs or save_figs:
msg = "Making the plot for this slice..."
print(msg)
log_msgs.append(msg)
# create histogram
t = (file_basename, det, pslice, "IFUflatcomp_histogram")
title = filt + " " + grat + " SLICE=" + pslice + "\n"
plot_name = "".join((file_path, ("_".join(t)) + ".pdf"))
#mk_hist(title, delfg, delfg_mean, delfg_median, delfg_std, save_figs, show_figs, plot_name=plot_name)
bins = None # binning for the histograms, if None the function will select them automatically
title = title + "Residuals"
info_img = [title, "x (pixels)", "y (pixels)"]
xlabel, ylabel = "flat$_{pipe}$ - flat$_{calc}$", "N"
info_hist = [xlabel, ylabel, bins, stats]
if delfg[1] is np.nan:
msg = "Unable to create plot of relative wavelength difference."
print(msg)
log_msgs.append(msg)
else:
plt_name = os.path.join(file_path, plot_name)
difference_img = (pipeflat - calc_flat) #/calc_flat
in_slit = np.logical_and(
difference_img < 900.0, difference_img >
-900.0) # ignore points out of the slit,
difference_img[
~in_slit] = np.nan # Set values outside the slit to NaN
nanind = np.isnan(
difference_img) # get all the nan indexes
difference_img[
nanind] = np.nan # set all nan indexes to have a value of nan
plt_origin = None
limits = [px0 - 5, px0 + 1500, py0 - 5, py0 + 55]
vminmax = [
-5 * delfg_std, 5 * delfg_std
] # set the range of values to be shown in the image, will affect color scale
auxfunc.plt_two_2Dimgandhist(difference_img,
delfg,
info_img,
info_hist,
plt_name=plt_name,
limits=limits,
vminmax=vminmax,
plt_origin=plt_origin,
show_figs=show_figs,
save_figs=save_figs)
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
print(msg)
log_msgs.append(msg)
if writefile:
# this is the file to hold the image of pipeline-calculated difference values
outfile_ext = fits.ImageHDU(flatcor.reshape(wave_shape),
name=pslice)
outfile.append(outfile_ext)
# this is the file to hold the image of pipeline-calculated difference values
complfile_ext = fits.ImageHDU(delf.reshape(wave_shape),
name=pslice)
complfile.append(complfile_ext)
# the file is not yet written, indicate that this slit was appended to list to be written
msg = "Extension " + repr(
n_ext
) + " appended to list to be written into calculated and comparison fits files."
print(msg)
log_msgs.append(msg)
# This is the key argument for the assert pytest function
median_diff = False
if abs(delfg_median) <= float(threshold_diff):
median_diff = True
if median_diff:
test_result = "PASSED"
else:
test_result = "FAILED"
msg = " *** Result of the test: " + test_result + "\n"
print(msg)
log_msgs.append(msg)
all_test_result.append(test_result)
# if the test is failed exit the script
if (delfg_median == 999.0) or not np.isfinite(delfg_median):
msg = "Unable to determine mean, meadian, and std_dev for the slice" + pslice
print(msg)
log_msgs.append(msg)
if mk_all_slices_plt:
if show_figs or save_figs:
# create histogram
t = (file_basename, det, "all_slices_IFU_flatcomp_histogram")
title = ("_".join(t))
# calculate median of medians and std_dev of medians
all_delfg_median_arr = np.array(all_delfg_median)
mean_of_delfg_mean = np.mean(all_delfg_mean_arr)
median_of_delfg_median = np.median(all_delfg_median_arr)
medians_std = np.std(median_of_delfg_median)
plot_name = "".join((file_path, title, ".pdf"))
mk_hist(title,
all_delfg_median_arr,
mean_of_delfg_mean,
median_of_delfg_median,
medians_std,
save_figs,
show_figs,
plot_name=plot_name)
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
print(msg)
log_msgs.append(msg)
# create fits file to hold the calculated flat for each slice
if writefile:
outfile_name = step_input_filename.replace("flat_field.fits",
det + "_flat_calc.fits")
complfile_name = step_input_filename.replace("flat_field.fits",
det + "_flat_comp.fits")
# create the fits list to hold the calculated flat values for each slit
outfile.writeto(outfile_name, overwrite=True)
# this is the file to hold the image of pipeline-calculated difference values
complfile.writeto(complfile_name, overwrite=True)
msg = "Fits file with calculated flat values of each slice saved as: "
print(msg)
print(outfile_name)
log_msgs.append(msg)
log_msgs.append(outfile_name)
msg = "Fits file with comparison (pipeline flat - calculated flat) saved as: "
print(msg)
print(complfile_name)
log_msgs.append(msg)
log_msgs.append(complfile_name)
# If all tests passed then pytest will be marked as PASSED, else it will be FAILED
FINAL_TEST_RESULT = True
for t in all_test_result:
if t == "FAILED":
FINAL_TEST_RESULT = False
break
if FINAL_TEST_RESULT:
msg = "\n *** Final result for flat_field test will be reported as PASSED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "All slices PASSED flat_field test."
else:
msg = "\n *** Final result for flat_field test will be reported as FAILED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "One or more slices FAILED flat_field test."
# end the timer
flattest_end_time = time.time() - flattest_start_time
if flattest_end_time > 60.0:
flattest_end_time = flattest_end_time / 60.0 # in minutes
flattest_tot_time = "* Script flattest_ifu.py script took ", repr(
flattest_end_time) + " minutes to finish."
if flattest_end_time > 60.0:
flattest_end_time = flattest_end_time / 60. # in hours
flattest_tot_time = "* Script flattest_ifu.py took ", repr(
flattest_end_time) + " hours to finish."
else:
flattest_tot_time = "* Script flattest_ifu.py took ", repr(
flattest_end_time) + " seconds to finish."
print(flattest_tot_time)
log_msgs.append(flattest_tot_time)
return FINAL_TEST_RESULT, result_msg, log_msgs
def ifu_coords(fname, output=None):
"""
Computes wavelengths, and space coordinates of a NIRSPEC
IFU exposure after running assign_wcs.
Parameters
----------
fname : str
The name of a file after assign_wcs was run.
output : str
The name of the output file. If None the root of the input
file is used with an extension world_coordinates.
Examples
--------
>>> compute_world_coordinates('nrs1_fixed_assign_wcs_extract_2d.fits')
"""
model = datamodels.ImageModel(fname)
if model.meta.exposure.type.lower() != 'nrs_ifu':
raise ValueError("Expected an IFU observation,"
"(EXP_TYPE=NRS_IFU), got {0}".format(
model.meta.exposure.type))
ifu_slits = nirspec.nrs_ifu_wcs(model)
hdulist = fits.HDUList()
phdu = fits.PrimaryHDU()
phdu.header['filename'] = model.meta.filename
phdu.header['data'] = 'world coordinates'
hdulist.append(phdu)
output_frame = ifu_slits[0].available_frames[-1]
for i, slit in enumerate(ifu_slits):
x, y = wcstools.grid_from_bounding_box(slit.bounding_box, (1, 1),
center=True)
ra, dec, lam = slit(x, y)
detector2slit = slit.get_transform('detector', 'slit_frame')
sx, sy, ls = detector2slit(x, y)
world_coordinates = np.array([lam, ra, dec, sy])
imhdu = fits.ImageHDU(data=world_coordinates)
imhdu.header['PLANE1'] = 'lambda, microns'
imhdu.header['PLANE2'] = '{0}_x, arcsec'.format(output_frame)
imhdu.header['PLANE3'] = '{0}_y, arcsec'.format(output_frame)
imhdu.header['PLANE4'] = 'slit_y, relative to center (0, 0)'
imhdu.header['SLIT'] = "SLIT_{0}".format(i)
# -1 and +1 are to express this in 1-based coordinates
imhdu.header['CRVAL1'] = model.meta.subarray.xstart - 1 + int(
_toindex(slit.bounding_box[0][0])) + 1
imhdu.header['CRVAL2'] = model.meta.subarray.xstart - 1 + int(
_toindex(slit.bounding_box[1][0])) + 1
# Input coordinates will be 1-based.
imhdu.header['CRPIX1'] = 1
imhdu.header['CRPIX2'] = 1
imhdu.header['CTYPE1'] = 'pixel'
imhdu.header['CTYPE2'] = 'pixel'
hdulist.append(imhdu)
if output is not None:
base, ext = os.path.splitext(output)
if ext != "fits":
ext = "fits"
if not base.endswith('world_coordinates'):
"".join([base, '_world_coordinates'])
"".join([base, ext])
else:
root = model.meta.filename.split('_')
output = "".join([root[0], '_world_coordinates', '.fits'])
hdulist.writeto(output, overwrite=True)
del hdulist
model.close()
def pathtest(step_input_filename,
reffile,
comparison_filename,
writefile=True,
show_figs=False,
save_figs=True,
threshold_diff=1.0e-7,
debug=False):
"""
This function calculates the difference between the pipeline and
calculated pathloss values.
Args:
step_input_filename: str, full path name of sourcetype output fits file
reffile: str, path to the pathloss IFU reference fits file
comparison_filename: str, path to comparison pipeline pathloss file
writefile: boolean, if True writes calculated flat and
difference image fits files
show_figs: boolean, whether to show plots or not
save_figs: boolean, save the plots
threshold_diff: float, threshold diff between pipeline and ESA file
debug: boolean, if true print statements will show on-screen
Returns:
- 1 plot, if told to save and/or show them.
- median_diff: Boolean, True if smaller or equal to threshold.
- log_msgs: list, all print statements are captured in this variable
"""
log_msgs = []
# start the timer
pathtest_start_time = time.time()
# get info from the input previous pipeline step file/datamodel
print(
"Checking if files exist and obtaining datamodels. This takes a few minutes..."
)
if isinstance(step_input_filename, str):
if os.path.isfile(step_input_filename):
if debug:
print('Input file does exist.')
msg = 'step_input_filename=' + step_input_filename
print(msg)
log_msgs.append(msg)
# get the input data model
ifu_input_model = datamodels.open(step_input_filename)
if debug:
print('got input datamodel!')
else:
result_msg = 'Input file does NOT exist. Skipping pathloss test.'
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
else:
ifu_input_model = step_input_filename
# get comparison data
if isinstance(comparison_filename, str):
if os.path.isfile(comparison_filename):
if debug:
msg = 'Comparison file does exist.'
print(msg)
else:
result_msg = "Comparison file does NOT exist. Skipping pathloss test."
print(result_msg)
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
# get the comparison data model
ifu_pipe_model = datamodels.open(comparison_filename)
if debug:
print('Retrieved comparison datamodel.')
else:
ifu_pipe_model = comparison_filename
# get info from data model
det = ifu_input_model.meta.instrument.detector
lamp = ifu_input_model.meta.instrument.lamp_state
grat = ifu_input_model.meta.instrument.grating
filt = ifu_input_model.meta.instrument.filter
exptype = ifu_input_model.meta.exposure.type
msg = "from datamodel --> Detector: " + det + " Grating: " + grat + " Filter: " + \
filt + " Lamp: " + lamp + " EXP_TYPE: " + exptype
print(msg)
log_msgs.append(msg)
msg = "Using reference file: " + reffile
print(msg)
log_msgs.append(msg)
is_point_source = True
if writefile:
# create the fits list to hold the calculated pathloss values for each slit
hdu0 = fits.PrimaryHDU()
outfile = fits.HDUList()
outfile.append(hdu0)
# create fits list to hold pipeline-calculated difference values
hdu0 = fits.PrimaryHDU()
compfile = fits.HDUList()
compfile.append(hdu0)
# list to determine if pytest is passed or not
total_test_result = []
# get all the science extensions
if is_point_source:
ext = 1 # for all PS IFU
# get slices (instead of using .slit)
pl_ifu_slits = nirspec.nrs_ifu_wcs(ifu_input_model)
print("got input slices")
plcor_ref_ext = fits.getdata(reffile, ext)
hdul = fits.open(reffile)
plcor_ref = hdul[1].data
w = wcs.WCS(hdul[1].header)
w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref.shape[1], :plcor_ref.
shape[2]]
slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0)
# these are full 2048 * 2048 files:
previous_sci = ifu_input_model.data
comp_sci = ifu_pipe_model.data
pathloss_divided = comp_sci / previous_sci
# set up generals for all plots
font = {'weight': 'normal', 'size': 12}
matplotlib.rc('font', **font)
# loop through the slices
msg = " Looping through the slices... "
print(msg)
log_msgs.append(msg)
slit_list = np.ndarray.tolist(np.arange(0, 30))
for slit, slit_num, pipeslit in zip(pl_ifu_slits, slit_list):
print("working with slice {}".format(slit_num))
# NIRSpec implementation
x, y = wcstools.grid_from_bounding_box(slit.bounding_box, step=(1, 1))
ra, dec, wave = slit(x, y)
slit_x = 0 # This assumption is made for IFU sources only
slit_y = 0
if debug:
print("slit_x, slit_y", slit_x, slit_y)
correction_array = np.array([])
lambda_array = np.array([])
wave_sci = wave * 10**(-6) # microns --> meters
wave_sci_flat = wave_sci.reshape(wave_sci.size)
wave_ref_flat = wave_ref.reshape(wave_ref.size)
ref_xy = np.column_stack((slitx_ref.reshape(slitx_ref.size),
slity_ref.reshape(slitx_ref.size)))
# loop through slices in lambda from reference file
shape = 0
for lambda_val in wave_ref_flat:
# loop through every lambda value
# flattened so that looping works smoothly
shape = shape + 1
index = np.where(wave_ref[:, 0, 0] == lambda_val)
# index of closest lambda value in reffile to given sci lambda
# took index of only the first slice of wave_ref because
# the others were repetitive & we got extra indices
# take slice where lambda=index:
plcor_slice = plcor_ref_ext[index[0][0]].reshape(
plcor_ref_ext[index[0][0]].size)
# do 2d interpolation to get a single correction factor for each slice
corr_val = scipy.interpolate.griddata(ref_xy[:plcor_slice.size],
plcor_slice,
np.asarray([slit_x, slit_y]),
method='linear')
# append values from loop to create a vector of correction factors
correction_array = np.append(correction_array, corr_val[0])
# map to array with corresponding lambda
lambda_array = np.append(lambda_array, lambda_val)
# get correction value for each pixel
corr_vals = np.interp(wave_sci_flat, lambda_array, correction_array)
corr_vals = corr_vals.reshape(wave_sci.shape)
box = slit.bounding_box
small_y = box[0][0]
big_y = box[0][1]
small_x = box[1][0]
big_x = box[1][1]
left = int(math.trunc(small_x))
right = int(math.ceil(big_x))
bottom = int(math.trunc(small_y))
top = int(math.ceil(big_y))
full_cut2slice = previous_sci[left:right, bottom:top]
print("shapes:", full_cut2slice.shape, corr_vals.shape)
if full_cut2slice.shape != corr_vals.shape:
value = 0
while (((full_cut2slice.shape[0] - corr_vals.shape[0]) != 0) or
((full_cut2slice.shape[1] - corr_vals.shape[1]) != 0)) and \
value < 7: # can delete second criteria once all pass
if value == 6:
print("WARNING: may be in infinite loop!")
x_amount_off = full_cut2slice.shape[0] - corr_vals.shape[0]
if x_amount_off >= 1:
if x_amount_off % 2 == 0: # need to add two values, so do one on each side
right = right - 1
left = left + 1
print("ALTERED SHAPE OF SLICE: V1")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
else: # just add one value
left = left + 1
print("ALTERED SHAPE OF SLICE: V2")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
elif x_amount_off <= -1:
if x_amount_off % 2 == 0:
right = right + 1
left = left - 1
print("ALTERED SHAPE OF SLICE: V3")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
else:
left = left - 1
print("ALTERED SHAPE OF SLICE: V4")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
y_amount_off = full_cut2slice.shape[1] - corr_vals.shape[1]
if y_amount_off >= 1:
if y_amount_off % 2 == 0:
bottom = bottom - 1
top = top + 1
print("ALTERED SHAPE OF SLICE: V5")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
else:
bottom = bottom + 1
print("ALTERED SHAPE OF SLICE: V6")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
elif y_amount_off <= -1:
if y_amount_off % 2 == 0:
top = top + 1
bottom = bottom - 1
print("ALTERED SHAPE OF SLICE: V7")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
else:
bottom = bottom - 1
print("ALTERED SHAPE OF SLICE: V8")
value = value + 1
print("left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
full_cut2slice = previous_sci[left:right, bottom:top]
print("final left {}, right {}, top {}, bottom {}".format(
left, right, top, bottom))
print("NEW SHAPE OF SLICE: {} and corr_vals.shape: {}".format(
full_cut2slice.shape, corr_vals.shape))
if full_cut2slice.shape != corr_vals.shape:
print("shapes did not match! full_cut2slice: {}, corr_vals {}".
format(full_cut2slice.shape, corr_vals.shape))
continue
corrected_array = full_cut2slice / corr_vals
pipe_correction = pathloss_divided[left:right, bottom:top]
if pipe_correction.shape != corr_vals.shape:
print("shapes did not match! pipe_correction: {}, corr_vals {}".
format(pipe_correction.shape, corr_vals.shape))
continue
prev_sci_slit = previous_sci[left:right, bottom:top]
if prev_sci_slit.shape != corr_vals.shape:
print(
"shapes did not match! prev_sci_slit: {}, corr_vals {}".format(
prev_sci_slit.shape, corr_vals.shape))
continue
comp_sci_slit = comp_sci[left:right, bottom:top]
if comp_sci_slit.shape != corr_vals.shape:
print(
"shapes did not match! comp_sci_slit: {}, corr_vals {}".format(
comp_sci_slit.shape, corr_vals.shape))
continue
# Plots:
# my correction values
fig = plt.figure(figsize=(15, 15))
plt.subplot(221)
norm = ImageNormalize(corr_vals)
plt.imshow(corr_vals,
vmin=0.999995,
vmax=1.000005,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Calculated Correction')
plt.colorbar()
# pipe correction
plt.subplot(222)
norm = ImageNormalize(pipe_correction)
plt.imshow(pipe_correction,
vmin=0.999995,
vmax=1.000005,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Pipe Correction')
plt.colorbar()
# residuals (pipe correction - my correction)
if pipe_correction.shape == corr_vals.shape:
corr_residuals = pipe_correction - corr_vals
plt.subplot(223)
norm = ImageNormalize(corr_residuals)
plt.imshow(corr_residuals,
vmin=-0.000000005,
vmax=0.000000005,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.title('Correction residuals')
plt.colorbar()
# my science data after pathloss
plt.subplot(224)
norm = ImageNormalize(corrected_array)
plt.imshow(corrected_array,
vmin=0,
vmax=300,
aspect=10.0,
origin='lower',
cmap='viridis')
plt.title('Corrected Data After Pathloss')
plt.xlabel('dispersion in pixels')
plt.ylabel('y in pixels')
plt.colorbar()
fig.suptitle("IFU PS Pathloss Correction Testing" +
str(corrected_array.shape))
fig.tight_layout(pad=3.0)
if show_figs:
plt.show()
if save_figs:
step_input_filepath = step_input_filename.replace(".fits", "")
plt_name = step_input_filepath + "_Pathloss_test_slitlet_IFU_PS_" + str(
slit_num) + ".png"
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
if debug:
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
if debug:
print(msg)
log_msgs.append(msg)
plt.clf()
# create fits file to hold the calculated pathloss for each slit
if writefile:
msg = "Saving the fits files with the calculated pathloss for each slit..."
print(msg)
log_msgs.append(msg)
# this is the file to hold the image of pipeline-calculated difference values
outfile_ext = fits.ImageHDU(corr_vals, name=str(slit_num))
outfile.append(outfile_ext)
# this is the file to hold the image of pipeline-calculated difference values
compfile_ext = fits.ImageHDU(corr_residuals, name=str(slit_num))
compfile.append(compfile_ext)
if corr_residuals[~np.isnan(corr_residuals)].size == 0:
msg1 = " * Unable to calculate statistics because difference array has all values as NaN. " \
"Test will be set to FAILED."
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
msg = "Calculating statistics... "
print(msg)
log_msgs.append(msg)
corr_residuals = corr_residuals[
np.where((corr_residuals != 999.0) & (corr_residuals < 0.1)
& (corr_residuals > -0.1))] # ignore outliers
if corr_residuals.size == 0:
msg1 = """ * Unable to calculate statistics because
difference array has all outlier values.
Test will be set to FAILED."""
print(msg1)
log_msgs.append(msg1)
test_result = "FAILED"
else:
stats_and_strings = auxfunc.print_stats(corr_residuals,
"Difference",
float(threshold_diff),
absolute=True)
stats, stats_print_strings = stats_and_strings
corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats
for msg in stats_print_strings:
log_msgs.append(msg)
# This is the key argument for the assert pytest function
median_diff = False
if abs(corr_residuals_median) <= float(threshold_diff):
median_diff = True
if median_diff:
test_result = "PASSED"
else:
test_result = "FAILED"
msg = " *** Result of the test: " + test_result + "\n"
print(msg)
log_msgs.append(msg)
total_test_result.append(test_result)
hdul.close()
ifu_pipe_model.close()
ifu_input_model.close()
if writefile:
outfile_name = step_input_filename.replace("srctype",
"_calcuated_pathloss")
compfile_name = step_input_filename.replace("srctype",
"_comparison_pathloss")
# create the fits list to hold the calculated pathloss values for each slit
outfile.writeto(outfile_name, overwrite=True)
# this is the file to hold the image of pipeline-calculated difference values
compfile.writeto(compfile_name, overwrite=True)
msg = "\nFits file with calculated pathloss values of each slit saved as: "
print(msg)
log_msgs.append(msg)
print(outfile_name)
log_msgs.append(outfile_name)
msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: "
print(msg)
log_msgs.append(msg)
print(compfile_name)
log_msgs.append(compfile_name)
# If all tests passed then pytest will be marked as PASSED, else FAILED
FINAL_TEST_RESULT = False
for t in total_test_result:
if t == "FAILED":
FINAL_TEST_RESULT = False
break
else:
FINAL_TEST_RESULT = True
if FINAL_TEST_RESULT:
msg = "\n *** Final pathloss test result reported as PASSED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "All slits PASSED path_loss test."
else:
msg = "\n *** Final pathloss test result reported as FAILED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "One or more slits FAILED path_loss test."
# end the timer
pathloss_end_time = time.time() - pathtest_start_time
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time / 60.0 # in minutes
pathloss_tot_time = "* Script IFU_PS.py took ", repr(
pathloss_end_time) + " minutes to finish."
if pathloss_end_time > 60.0:
pathloss_end_time = pathloss_end_time / 60. # in hours
pathloss_tot_time = "* Script IFU_PS.py took ", repr(
pathloss_end_time) + " hours to finish."
else:
pathloss_tot_time = "* Script IFU_PS.py took ", repr(
pathloss_end_time) + " seconds to finish."
print(pathloss_tot_time)
log_msgs.append(pathloss_tot_time)
return FINAL_TEST_RESULT, result_msg, log_msgs
