def imaging_coords(fname, output=None):
"""
Computes wavelengths, and space coordinates of a NIRSPEC
imaging exposure after running assign_wcs.
Parameters
----------
fname : str
The name of a file after assign_wcs was run.
output : str
The name of the output file. If None the root of the input
file is used with an extension world_coordinates.
Examples
--------
>>> compute_world_coordinates('nrs1_fixed_assign_wcs_extract_2d.fits')
"""
model = datamodels.ImageModel(fname)
if model.meta.exposure.type.lower() not in imaging_modes:
raise ValueError("Observation mode {0} is not supported.".format(model.meta.exposure.type))
hdulist = fits.HDUList()
phdu = fits.PrimaryHDU()
phdu.header['filename'] = model.meta.filename
phdu.header['data'] = 'world coordinates'
hdulist.append(phdu)
output_frame = model.available_frames[-1]
bb = model.meta.wcs.bounding_box
x, y = wcstools.grid_from_bounding_box(bb, step=(1, 1), center=True)
ra, dec, lam = slit(x + 1, y + 1)
world_coordinates = np.array([lam, ra, dec])
imhdu = fits.ImageHDU(data=world_coordinates)
imhdu.header['PLANE1'] = 'lambda, microns'
imhdu.header['PLANE2'] = '{0}_x, deg'.format(output_frame)
imhdu.header['PLANE3'] = '{0}_y, deg'.format(output_frame)
imhdu.header['SLIT'] = "SLIT_{0}".format(i)
# add the overall subarray offset
imhdu.header['CRVAL1'] = model.meta.subarray.xstart - 1 + int(_toindex(bb[0][0]))
imhdu.header['CRVAL2'] = model.meta.subarray.ystart - 1 + int(_toindex(bb[1][0]))
# Input coordinates will be 1-based.
imhdu.header['CRPIX1'] = 1
imhdu.header['CRPIX2'] = 1
imhdu.header['CTYPE1'] = 'pixel'
imhdu.header['CTYPE2'] = 'pixel'
hdulist.append(imhdu)
if output is not None:
base, ext = os.path.splitext(output)
if ext != "fits":
ext = "fits"
if not base.endswith('world_coordinates'):
"".join([base, '_world_coordinates'])
"".join([base, ext])
else:
root = model.meta.filename.split('_')
output = "".join([root[0], '_world_coordinates', '.fits'])
hdulist.writeto(output, overwrite=True)
del hdulist
model.close()
def set_slit_attributes(output_model, slit, xlo, xhi, ylo, yhi):
"""
Set the slit attributes.
Parameters
----------
output_model : `~jwst.datamodels.multislit.MultiSlitModel`
The output model representing a slit.
nslit : int
The index o fthis slit in the `~jwst.datamodels.multislit.MultiSlitModel`.
xlo, xhi, ylo, yhi : float
Indices into the data array where extraction should be done.
"""
xlo_ind, xhi_ind, ylo_ind, yhi_ind = _toindex(
(xlo, xhi, ylo, yhi)).astype(np.int16)
output_model.name = str(slit.name)
output_model.xstart = xlo_ind + 1
output_model.xsize = (xhi_ind - xlo_ind) + 1
output_model.ystart = ylo_ind + 1
output_model.ysize = (yhi_ind - ylo_ind) + 1
if output_model.meta.exposure.type.lower() == 'nrs_msaspec':
output_model.source_id = int(slit.source_id)
output_model.source_name = slit.source_name
output_model.source_alias = slit.source_alias
output_model.stellarity = float(slit.stellarity)
output_model.source_xpos = float(slit.source_xpos)
output_model.source_ypos = float(slit.source_ypos)
output_model.slitlet_id = int(slit.name)
# for pathloss correction
output_model.shutter_state = slit.shutter_state
log.info('set slit_attributes completed')
def offset_wcs(slit_wcs):
"""
Prepend a Shift transform to the slit WCS to account for subarrays.
Parameters
----------
slit_wcs : `~gwcs.wcs.WCS`
The WCS for this slit.
slit_name : str
The name of the slit.
"""
xlo, xhi = _toindex(slit_wcs.bounding_box[0])
ylo, yhi = _toindex(slit_wcs.bounding_box[1])
# Add the slit offset to each slit WCS object
tr = slit_wcs.get_transform('detector', 'sca')
tr = Shift(xlo) & Shift(ylo) | tr
slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca'))
return xlo, xhi, ylo, yhi
def offset_wcs(slit_wcs):
"""
Prepend a Shift transform to the slit WCS to account for subarrays.
Parameters
----------
slit_wcs : `~gwcs.wcs.WCS`
The WCS for this slit.
slit_name : str
The name of the slit.
"""
xlo, xhi = _toindex(slit_wcs.bounding_box[0])
ylo, yhi = _toindex(slit_wcs.bounding_box[1])
# Add the slit offset to each slit WCS object
tr = slit_wcs.get_transform('detector', 'sca')
tr = Shift(xlo) & Shift(ylo) | tr
slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca'))
return xlo, xhi, ylo, yhi
def offset_wcs(slit_wcs, slit_name):
"""
Prepend a Shift to the slit WCS to account for subarrays.
Parameters
----------
slit_wcs : `~gwcs.wcs.WCS`
The WCS for this slit.
slit_name : str
The name of the slit.
"""
xlo, xhi = _toindex(slit_wcs.bounding_box[0])
ylo, yhi = _toindex(slit_wcs.bounding_box[1])
# Add the slit offset to each slit WCS object
tr = slit_wcs.get_transform('detector', 'sca')
tr = Shift(xlo) & Shift(ylo) | tr
slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca'))
log.info('Name of subarray extracted: %s', slit_name)
log.info('Subarray x-extents are: %s %s', xlo, xhi)
log.info('Subarray y-extents are: %s %s', ylo, yhi)
return xlo, xhi, ylo, yhi
def set_slit_attributes(output_model, slit, xlo, xhi, ylo, yhi):
"""
Set the slit attributes.
Parameters
----------
output_model : `~jwst.datamodels.multislit.MultiSlitModel`
The output model representing a slit.
slit : namedtuple
A `~jwst.transforms.models.Slit` object representing a slit.
xlo, xhi, ylo, yhi : float
Indices into the data array where extraction should be done.
These are converted to "pixel indices" - the center of a pixel.
"""
xlo_ind, xhi_ind, ylo_ind, yhi_ind = _toindex(
(xlo, xhi, ylo, yhi)).astype(np.int16)
output_model.name = str(slit.name)
output_model.xstart = xlo_ind + 1
output_model.xsize = (xhi_ind - xlo_ind) + 1
output_model.ystart = ylo_ind + 1
output_model.ysize = (yhi_ind - ylo_ind) + 1
if output_model.meta.exposure.type.lower() in [
'nrs_msaspec', 'nrs_autoflat'
]:
output_model.source_id = int(slit.source_id)
output_model.source_name = slit.source_name
output_model.source_alias = slit.source_alias
output_model.stellarity = float(slit.stellarity)
output_model.source_xpos = float(slit.source_xpos)
output_model.source_ypos = float(slit.source_ypos)
output_model.slitlet_id = int(slit.name)
output_model.quadrant = int(slit.quadrant)
output_model.xcen = int(slit.xcen)
output_model.ycen = int(slit.ycen)
# for pathloss correction
output_model.shutter_state = slit.shutter_state
log.info('set slit_attributes completed')
def ifu_coords(fname, output=None):
"""
Computes wavelengths, and space coordinates of a NIRSPEC
IFU exposure after running assign_wcs.
Parameters
----------
fname : str
The name of a file after assign_wcs was run.
output : str
The name of the output file. If None the root of the input
file is used with an extension world_coordinates.
Examples
--------
>>> compute_world_coordinates('nrs1_fixed_assign_wcs_extract_2d.fits')
"""
model = datamodels.ImageModel(fname)
if model.meta.exposure.type.lower() != 'nrs_ifu':
raise ValueError("Expected an IFU observation,"
"(EXP_TYPE=NRS_IFU), got {0}".format(
model.meta.exposure.type))
ifu_slits = nirspec.nrs_ifu_wcs(model)
hdulist = fits.HDUList()
phdu = fits.PrimaryHDU()
phdu.header['filename'] = model.meta.filename
phdu.header['data'] = 'world coordinates'
hdulist.append(phdu)
output_frame = ifu_slits[0].available_frames[-1]
for i, slit in enumerate(ifu_slits):
x, y = wcstools.grid_from_bounding_box(slit.bounding_box, (1, 1),
center=True)
ra, dec, lam = slit(x, y)
detector2slit = slit.get_transform('detector', 'slit_frame')
sx, sy, ls = detector2slit(x, y)
world_coordinates = np.array([lam, ra, dec, sy])
imhdu = fits.ImageHDU(data=world_coordinates)
imhdu.header['PLANE1'] = 'lambda, microns'
imhdu.header['PLANE2'] = '{0}_x, arcsec'.format(output_frame)
imhdu.header['PLANE3'] = '{0}_y, arcsec'.format(output_frame)
imhdu.header['PLANE4'] = 'slit_y, relative to center (0, 0)'
imhdu.header['SLIT'] = "SLIT_{0}".format(i)
# -1 and +1 are to express this in 1-based coordinates
imhdu.header['CRVAL1'] = model.meta.subarray.xstart - 1 + int(
_toindex(slit.bounding_box[0][0])) + 1
imhdu.header['CRVAL2'] = model.meta.subarray.xstart - 1 + int(
_toindex(slit.bounding_box[1][0])) + 1
# Input coordinates will be 1-based.
imhdu.header['CRPIX1'] = 1
imhdu.header['CRPIX2'] = 1
imhdu.header['CTYPE1'] = 'pixel'
imhdu.header['CTYPE2'] = 'pixel'
hdulist.append(imhdu)
if output is not None:
base, ext = os.path.splitext(output)
if ext != "fits":
ext = "fits"
if not base.endswith('world_coordinates'):
"".join([base, '_world_coordinates'])
"".join([base, ext])
else:
root = model.meta.filename.split('_')
output = "".join([root[0], '_world_coordinates', '.fits'])
hdulist.writeto(output, overwrite=True)
del hdulist
model.close()
def flattest(step_input_filename,
dflatref_path=None,
sfile_path=None,
fflat_path=None,
writefile=False,
mk_all_slices_plt=False,
show_figs=True,
save_figs=False,
plot_name=None,
threshold_diff=1.0e-7,
debug=False):
"""
This function calculates the difference between the pipeline and the calculated flat field values.
The functions uses the output of the compute_world_coordinates.py script.
Args:
step_input_filename: str, name of the output fits file from the 2d_extract step (with full path)
dflatref_path: str, path of where the D-flat reference fits files
sfile_path: str, path of where the S-flat reference fits files
fflat_path: str, path of where the F-flat reference fits files
msa_conf_root: str, path to where the MSA configuration fits file lives
writefile: boolean, if True writes the fits files of the calculated flat and difference images
show_figs: boolean, whether to show plots or not
save_figs: boolean, save the plots (the 3 plots can be saved or not independently with the function call)
plot_name: string, desired name (if name is not given, the plot function will name the plot by
default)
threshold_diff: float, threshold difference between pipeline output and ESA file
debug: boolean, if true a series of print statements will show on-screen
Returns:
- 1 plot, if told to save and/or show.
- median_diff: Boolean, True if smaller or equal to 1e-14
- log_msgs: list, all print statements are captured in this variable
"""
log_msgs = []
# start the timer
flattest_start_time = time.time()
# get info from the flat field file
file_path = step_input_filename.replace(
os.path.basename(step_input_filename), "")
det = fits.getval(step_input_filename, "DETECTOR", 0)
exptype = fits.getval(step_input_filename, "EXP_TYPE", 0)
grat = fits.getval(step_input_filename, "GRATING", 0)
filt = fits.getval(step_input_filename, "FILTER", 0)
file_basename = os.path.basename(step_input_filename.replace(".fits", ""))
msg1 = 'step_input_filename=' + step_input_filename
msg2 = "flat_field_file --> Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype
print(msg1)
print(msg2)
log_msgs.append(msg1)
log_msgs.append(msg2)
# read in the on-the-fly flat image
flatfile = step_input_filename.replace("flat_field.fits",
"interpolatedflat.fits")
pipeflat = fits.getdata(flatfile, "SCI")
# get the reference files
msg = "Getting and reading the D-, S-, and F-flats for this specific IFU configuration... "
print(msg)
log_msgs.append(msg)
# D-Flat
dflat_ending = "f_01.03.fits"
dfile = dflatref_path + "_nrs1_" + dflat_ending
if det == "NRS2":
dfile = dfile.replace("nrs1", "nrs2")
msg = "Using D-flat: " + dfile
print(msg)
log_msgs.append(msg)
dfim = fits.getdata(dfile, "SCI") #1)
dfimdq = fits.getdata(dfile, "DQ") #4)
# need to flip/rotate the image into science orientation
ns = np.shape(dfim)
dfim = np.transpose(dfim, (
0, 2,
1)) # keep in mind that 0,1,2 = z,y,x in Python, whereas =x,y,z in IDL
dfimdq = np.transpose(dfimdq)
if det == "NRS2":
# rotate science data by 180 degrees for NRS2
dfim = dfim[..., ::-1, ::-1]
dfimdq = dfimdq[..., ::-1, ::-1]
naxis3 = fits.getval(dfile, "NAXIS3", "SCI") #1)
# get the wavelength values
dfwave = np.array([])
for i in range(naxis3):
keyword = "PFLAT_" + str(i + 1)
dfwave = np.append(dfwave, fits.getval(dfile, keyword, "SCI")) #1))
dfrqe = fits.getdata(dfile, 2)
# S-flat
tsp = exptype.split("_")
mode = tsp[1]
if filt == "F070LP":
flat = "FLAT4"
elif filt == "F100LP":
flat = "FLAT1"
elif filt == "F170LP":
flat = "FLAT2"
elif filt == "F290LP":
flat = "FLAT3"
elif filt == "CLEAR":
flat = "FLAT5"
else:
msg = "No filter correspondence. Exiting the program."
print(msg)
log_msgs.append(msg)
# This is the key argument for the assert pytest function
msg = "Test skiped because there is no flat correspondence for the filter in the data: {}".format(
filt)
median_diff = "skip"
return median_diff, msg
sflat_ending = "f_01.01.fits"
sfile = sfile_path + "_" + grat + "_OPAQUE_" + flat + "_nrs1_" + sflat_ending
if debug:
print("grat = ", grat)
print("flat = ", flat)
print("sfile used = ", sfile)
if det == "NRS2":
sfile = sfile.replace("nrs1", "nrs2")
msg = "Using S-flat: " + sfile
print(msg)
log_msgs.append(msg)
sfim = fits.getdata(sfile, "SCI") #1)
sfimdq = fits.getdata(sfile, "DQ") #3)
# need to flip/rotate image into science orientation
sfim = np.transpose(sfim)
sfimdq = np.transpose(sfimdq)
if det == "NRS2":
# rotate science data by 180 degrees for NRS2
sfim = sfim[..., ::-1, ::-1]
sfimdq = sfimdq[..., ::-1, ::-1]
sfv = fits.getdata(sfile, 5)
# F-Flat
fflat_ending = "_01.01.fits"
if mode in fflat_path:
ffile = fflat_path + "_" + filt + fflat_ending
else:
msg = "Wrong path in for mode F-flat. This script handles mode " + mode + "only."
print(msg)
log_msgs.append(msg)
# This is the key argument for the assert pytest function
result_msg = "Wrong path in for mode F-flat. Test skiped because mode is not IFU."
median_diff = "skip"
return median_diff, result_msg, log_msgs
msg = "Using F-flat: " + ffile
print(msg)
log_msgs.append(msg)
ffv = fits.getdata(ffile, "IFU") #1)
# now go through each pixel in the test data
if writefile:
# create the fits list to hold the calculated flat values for each slit
hdu0 = fits.PrimaryHDU()
outfile = fits.HDUList()
outfile.append(hdu0)
# create the fits list to hold the image of pipeline-calculated difference values
hdu0 = fits.PrimaryHDU()
complfile = fits.HDUList()
complfile.append(hdu0)
# get the datamodel from the assign_wcs output file
assign_wcs_file = step_input_filename.replace("_flat_field.fits",
"_assign_wcs.fits")
model = datamodels.ImageModel(assign_wcs_file)
ifu_slits = nirspec.nrs_ifu_wcs(model)
# loop over the slices
all_delfg_mean, all_delfg_mean_arr, all_delfg_median, all_test_result = [], [], [], []
msg = "\n Now looping through the slices, this may take some time... "
print(msg)
log_msgs.append(msg)
for n_ext, slice in enumerate(ifu_slits):
if n_ext < 10:
pslice = "0" + repr(n_ext)
else:
pslice = repr(n_ext)
msg = "\nWorking with slice: " + pslice
print(msg)
log_msgs.append(msg)
# get the wavelength
# slice.x(y)start are 1-based, turn them to 0-based for extraction
x, y = wcstools.grid_from_bounding_box(slice.bounding_box, (1, 1),
center=True)
ra, dec, wave = slice(x, y)
# get the subwindow origin (technically no subwindows for IFU, but need this for comparing to the
# full frame on-the-fly flat image).
px0 = model.meta.subarray.xstart - 1 + int(
_toindex(slice.bounding_box[0][0])) + 1
py0 = model.meta.subarray.xstart - 1 + int(
_toindex(slice.bounding_box[1][0])) + 1
n_p = np.shape(wave)
nx, ny = n_p[1], n_p[0]
nw = nx * ny
msg = " Subwindow origin: px0=" + repr(px0) + " py0=" + repr(py0)
print(msg)
log_msgs.append(msg)
if debug:
print("n_p = ", n_p)
print("nw = ", nw)
# initialize arrays of the right size
delf = np.zeros([nw]) + 999.0
flatcor = np.zeros([nw]) + 999.0
sffarr = np.zeros([nw])
calc_flat = np.zeros([2048, 2048]) + 999.0
# loop through the wavelengths
msg = " Looping through the wavelngth, this may take a little time ... "
print(msg)
log_msgs.append(msg)
flat_wave = wave.flatten()
wave_shape = np.shape(wave)
for j in range(0, nw):
if np.isfinite(flat_wave[j]): # skip if wavelength is NaN
# get the pixel indeces
jwav = flat_wave[j]
t = np.where(wave == jwav)
pind = [t[0][0] + py0 - 1, t[1][0] + px0 - 1
] # pind =[pixel_y, pixe_x] in python, [x, y] in IDL
if debug:
print('j, jwav, px0, py0 : ', j, jwav, px0, py0)
print('pind[0], pind[1] = ', pind[0], pind[1])
# get the pixel bandwidth **this needs to be modified for prism, since the dispersion is not linear!**
delw = 0.0
if (j != 0) and (int((j - 1) / nx) == int(j / nx)) and (int(
(j + 1) / nx) == int(j / nx)) and np.isfinite(
flat_wave[j + 1]) and np.isfinite(flat_wave[j - 1]):
delw = 0.5 * (flat_wave[j + 1] - flat_wave[j - 1])
if (j == 0) or not np.isfinite(flat_wave[j - 1]) or (int(
(j - 1) / nx) != int(j / nx)):
delw = 0.5 * (flat_wave[j + 1] - flat_wave[j])
if (j == nw - 1) or not np.isfinite(flat_wave[j + 1]) or (int(
(j + 1) / nx) != int(j / nx)):
delw = 0.5 * (flat_wave[j] - flat_wave[j - 1])
if debug:
#print("(j, (j-1), nx, (j-1)/nx, (j+1), (j+1)/nx)", j, (j-1), nx, int((j-1)/nx), (j+1), int((j+1)/nx))
#print("np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1])", np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1]))
#print("flat_wave[j+1], flat_wave[j-1] : ", np.isfinite(flat_wave[j+1]), flat_wave[j+1], flat_wave[j-1])
print("delw = ", delw)
# integrate over D-flat fast vector
dfrqe_wav = dfrqe.field("WAVELENGTH")
dfrqe_rqe = dfrqe.field("RQE")
iw = np.where((dfrqe_wav >= jwav - delw / 2.0)
& (dfrqe_wav <= jwav + delw / 2.0))
if np.size(iw) == 0:
iw = -1
int_tab = auxfunc.idl_tabulate(dfrqe_wav[iw], dfrqe_rqe[iw])
if int_tab == 0:
int_tab = np.interp(dfrqe_wav[iw], dfrqe_wav, dfrqe_rqe)
dff = int_tab
else:
first_dfrqe_wav, last_dfrqe_wav = dfrqe_wav[iw][
0], dfrqe_wav[iw][-1]
dff = int_tab / (last_dfrqe_wav - first_dfrqe_wav)
if debug:
#print("np.shape(dfrqe_wav) : ", np.shape(dfrqe_wav))
#print("np.shape(dfrqe_rqe) : ", np.shape(dfrqe_rqe))
#print("dfimdq[pind[0]][pind[1]] : ", dfimdq[pind[0]][pind[1]])
#print("np.shape(iw) =", np.shape(iw))
#print("np.shape(dfrqe_wav[iw[0]]) = ", np.shape(dfrqe_wav[iw[0]]))
#print("np.shape(dfrqe_rqe[iw[0]]) = ", np.shape(dfrqe_rqe[iw[0]]))
#print("int_tab=", int_tab)
print("np.shape(iw) = ", np.shape(iw))
print("iw = ", iw)
print("dff = ", dff)
# interpolate over D-flat cube
dfs = 1.0
if dfimdq[pind[0], pind[1]] == 0:
dfs = np.interp(jwav, dfwave, dfim[:, pind[0], pind[1]])
# integrate over S-flat fast vector
sfv_wav = sfv.field("WAVELENGTH")
sfv_dat = sfv.field("DATA")
if (jwav < 5.3) and (jwav > 0.6):
iw = np.where((sfv_wav >= jwav - delw / 2.0)
& (sfv_wav <= jwav + delw / 2.0))
if np.size(iw) == 0:
iw = -1
if np.size(iw) > 1:
int_tab = auxfunc.idl_tabulate(sfv_wav[iw],
sfv_dat[iw])
first_sfv_wav, last_sfv_wav = sfv_wav[iw][0], sfv_wav[
iw][-1]
sff = int_tab / (last_sfv_wav - first_sfv_wav)
elif np.size(iw) == 1:
sff = float(sfv_dat[iw])
else:
sff = 999.0
# get s-flat pixel-dependent correction
sfs = 1.0
if sfimdq[pind[0], pind[1]] == 0:
sfs = sfim[pind[0], pind[1]]
if debug:
print("jwav-delw/2.0 = ", jwav - delw / 2.0)
print("jwav+delw/2.0 = ", jwav + delw / 2.0)
print("np.shape(sfv_wav), sfv_wav[-1] = ",
np.shape(sfv_wav), sfv_wav[-1])
print("iw = ", iw)
print("sfv_wav[iw] = ", sfv_wav[iw])
print("int_tab = ", int_tab)
print("first_sfv_wav, last_sfv_wav = ", first_sfv_wav,
last_sfv_wav)
print("sfs = ", sfs)
print("sff = ", sff)
# integrate over f-flat fast vector
# reference file blue cutoff is 1 micron, so need to force solution for shorter wavs
ffv_wav = ffv.field("WAVELENGTH")
ffv_dat = ffv.field("DATA")
fff = 1.0
if jwav - delw / 2.0 >= 1.0:
iw = np.where((ffv_wav >= jwav - delw / 2.0)
& (ffv_wav <= jwav + delw / 2.0))
if np.size(iw) == 0:
iw = -1
if np.size(iw) > 1:
int_tab = auxfunc.idl_tabulate(ffv_wav[iw],
ffv_dat[iw])
first_ffv_wav, last_ffv_wav = ffv_wav[iw][0], ffv_wav[
iw][-1]
fff = int_tab / (last_ffv_wav - first_ffv_wav)
elif np.size(iw) == 1:
fff = float(ffv_dat[iw])
flatcor[j] = dff * dfs * sff * sfs * fff
sffarr[j] = sff
# To visually compare between the pipeline flat and the calculated one (e.g. in ds9), Phil Hodge
# suggested using the following line:
calc_flat[pind[0], pind[1]] = flatcor[j]
# this line writes the calculated flat into a full frame array
# then this new array needs to be written into a file. This part has not been done yet.
# Difference between pipeline and calculated values
delf[j] = pipeflat[pind[0], pind[1]] - flatcor[j]
# Remove all pixels with values=1 (mainly inter-slit pixels) for statistics
if pipeflat[pind[0], pind[1]] == 1:
delf[j] = 999.0
if np.isnan(jwav):
flatcor[j] = 1.0 # no correction if no wavelength
if debug:
print("np.shape(iw) = ", np.shape(iw))
print("fff = ", fff)
print("flatcor[j] = ", flatcor[j])
print("delf[j] = ", delf[j])
# ignore outliers for calculating median
delfg = delf[np.where(delf != 999.0)]
#delfg_median, delfg_std = np.median(delfg), np.std(delfg)
msg = "Flat value differences for slice number: " + pslice
print(msg)
log_msgs.append(msg)
#print(" median = ", delfg_median, " stdev =", delfg_std)
stats_and_strings = auxfunc.print_stats(delfg,
"Flat Difference",
float(threshold_diff),
abs=True)
stats, stats_print_strings = stats_and_strings
delfg_mean, delfg_median, delfg_std = stats
for msg in stats_print_strings:
log_msgs.append(msg)
if debug:
print("np.shape(delf) = ", np.shape(delf))
print("np.shape(delfg) = ", np.shape(delfg))
all_delfg_mean.append(delfg_mean)
all_delfg_median.append(delfg_median)
# make the slice plot
if np.isfinite(delfg_median) and (len(delfg) != 0):
if show_figs or save_figs:
msg = "Making the plot for this slice..."
print(msg)
log_msgs.append(msg)
# create histogram
t = (file_basename, det, pslice, "IFUflatcomp_histogram")
title = filt + " " + grat + " SLICE=" + pslice + "\n"
plot_name = "".join((file_path, ("_".join(t)) + ".pdf"))
#mk_hist(title, delfg, delfg_mean, delfg_median, delfg_std, save_figs, show_figs, plot_name=plot_name)
bins = None # binning for the histograms, if None the function will select them automatically
title = title + "Residuals"
info_img = [title, "x (pixels)", "y (pixels)"]
xlabel, ylabel = "flat$_{pipe}$ - flat$_{calc}$", "N"
info_hist = [xlabel, ylabel, bins, stats]
if delfg[1] is np.nan:
msg = "Unable to create plot of relative wavelength difference."
print(msg)
log_msgs.append(msg)
else:
plt_name = os.path.join(file_path, plot_name)
difference_img = (pipeflat - calc_flat) #/calc_flat
in_slit = np.logical_and(
difference_img < 900.0, difference_img >
-900.0) # ignore points out of the slit,
difference_img[
~in_slit] = np.nan # Set values outside the slit to NaN
nanind = np.isnan(
difference_img) # get all the nan indexes
difference_img[
nanind] = np.nan # set all nan indexes to have a value of nan
plt_origin = None
limits = [px0 - 5, px0 + 1500, py0 - 5, py0 + 55]
vminmax = [
-5 * delfg_std, 5 * delfg_std
] # set the range of values to be shown in the image, will affect color scale
auxfunc.plt_two_2Dimgandhist(difference_img,
delfg,
info_img,
info_hist,
plt_name=plt_name,
limits=limits,
vminmax=vminmax,
plt_origin=plt_origin,
show_figs=show_figs,
save_figs=save_figs)
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
print(msg)
log_msgs.append(msg)
if writefile:
# this is the file to hold the image of pipeline-calculated difference values
outfile_ext = fits.ImageHDU(flatcor.reshape(wave_shape),
name=pslice)
outfile.append(outfile_ext)
# this is the file to hold the image of pipeline-calculated difference values
complfile_ext = fits.ImageHDU(delf.reshape(wave_shape),
name=pslice)
complfile.append(complfile_ext)
# the file is not yet written, indicate that this slit was appended to list to be written
msg = "Extension " + repr(
n_ext
) + " appended to list to be written into calculated and comparison fits files."
print(msg)
log_msgs.append(msg)
# This is the key argument for the assert pytest function
median_diff = False
if abs(delfg_median) <= float(threshold_diff):
median_diff = True
if median_diff:
test_result = "PASSED"
else:
test_result = "FAILED"
msg = " *** Result of the test: " + test_result + "\n"
print(msg)
log_msgs.append(msg)
all_test_result.append(test_result)
# if the test is failed exit the script
if (delfg_median == 999.0) or not np.isfinite(delfg_median):
msg = "Unable to determine mean, meadian, and std_dev for the slice" + pslice
print(msg)
log_msgs.append(msg)
if mk_all_slices_plt:
if show_figs or save_figs:
# create histogram
t = (file_basename, det, "all_slices_IFU_flatcomp_histogram")
title = ("_".join(t))
# calculate median of medians and std_dev of medians
all_delfg_median_arr = np.array(all_delfg_median)
mean_of_delfg_mean = np.mean(all_delfg_mean_arr)
median_of_delfg_median = np.median(all_delfg_median_arr)
medians_std = np.std(median_of_delfg_median)
plot_name = "".join((file_path, title, ".pdf"))
mk_hist(title,
all_delfg_median_arr,
mean_of_delfg_mean,
median_of_delfg_median,
medians_std,
save_figs,
show_figs,
plot_name=plot_name)
elif not save_figs and not show_figs:
msg = "Not making plots because both show_figs and save_figs were set to False."
print(msg)
log_msgs.append(msg)
elif not save_figs:
msg = "Not saving plots because save_figs was set to False."
print(msg)
log_msgs.append(msg)
# create fits file to hold the calculated flat for each slice
if writefile:
outfile_name = step_input_filename.replace("flat_field.fits",
det + "_flat_calc.fits")
complfile_name = step_input_filename.replace("flat_field.fits",
det + "_flat_comp.fits")
# create the fits list to hold the calculated flat values for each slit
outfile.writeto(outfile_name, overwrite=True)
# this is the file to hold the image of pipeline-calculated difference values
complfile.writeto(complfile_name, overwrite=True)
msg = "Fits file with calculated flat values of each slice saved as: "
print(msg)
print(outfile_name)
log_msgs.append(msg)
log_msgs.append(outfile_name)
msg = "Fits file with comparison (pipeline flat - calculated flat) saved as: "
print(msg)
print(complfile_name)
log_msgs.append(msg)
log_msgs.append(complfile_name)
# If all tests passed then pytest will be marked as PASSED, else it will be FAILED
FINAL_TEST_RESULT = True
for t in all_test_result:
if t == "FAILED":
FINAL_TEST_RESULT = False
break
if FINAL_TEST_RESULT:
msg = "\n *** Final result for flat_field test will be reported as PASSED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "All slices PASSED flat_field test."
else:
msg = "\n *** Final result for flat_field test will be reported as FAILED *** \n"
print(msg)
log_msgs.append(msg)
result_msg = "One or more slices FAILED flat_field test."
# end the timer
flattest_end_time = time.time() - flattest_start_time
if flattest_end_time > 60.0:
flattest_end_time = flattest_end_time / 60.0 # in minutes
flattest_tot_time = "* Script flattest_ifu.py script took ", repr(
flattest_end_time) + " minutes to finish."
if flattest_end_time > 60.0:
flattest_end_time = flattest_end_time / 60. # in hours
flattest_tot_time = "* Script flattest_ifu.py took ", repr(
flattest_end_time) + " hours to finish."
else:
flattest_tot_time = "* Script flattest_ifu.py took ", repr(
flattest_end_time) + " seconds to finish."
print(flattest_tot_time)
log_msgs.append(flattest_tot_time)
return FINAL_TEST_RESULT, result_msg, log_msgs
def lrs(input_model, reference_files):
"""
The LRS-FIXEDSLIT and LRS-SLITLESS WCS pipeline.
It has two coordinate frames: "detecor" and "world".
Uses the "specwcs" and "distortion" reference files.
"""
# Setup the frames.
detector = cf.Frame2D(name='detector',
axes_order=(0, 1),
unit=(u.pix, u.pix))
spec = cf.SpectralFrame(name='wavelength',
axes_order=(2, ),
unit=(u.micron, ),
axes_names=('lambda', ))
sky = cf.CelestialFrame(reference_frame=coord.ICRS(), name='sky')
world = cf.CompositeFrame(name="world", frames=[sky, spec])
# Determine the distortion model.
subarray2full = subarray_transform(input_model)
with DistortionModel(reference_files['distortion']) as dist:
distortion = dist.model
full_distortion = subarray2full | distortion
# Load and process the reference data.
with fits.open(reference_files['specwcs']) as ref:
lrsdata = np.array([l for l in ref[1].data])
# Get the zero point from the reference data.
# The zero_point is X, Y (which should be COLUMN, ROW)
# TODO: Are imx, imy 0- or 1-indexed? We are treating them here as
# 0-indexed. Since they are FITS, they are probably 1-indexed.
if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit':
zero_point = ref[1].header['imx'], ref[1].header['imy']
elif input_model.meta.exposure.type.lower() == 'mir_lrs-slitless':
#zero_point = ref[1].header['imxsltl'], ref[1].header['imysltl']
zero_point = [35, 442] # [35, 763] # account for subarray
# Create the bounding_box
x0 = lrsdata[:, 3]
y0 = lrsdata[:, 4]
x1 = lrsdata[:, 5]
bb = ((x0.min() - 0.5 + zero_point[0], x1.max() + 0.5 + zero_point[0]),
(y0.min() - 0.5 + zero_point[1], y0.max() + 0.5 + zero_point[1]))
# Find the ROW of the zero point which should be the [1] of zero_point
row_zero_point = zero_point[1]
# Compute the v2v3 to sky.
tel2sky = pointing.v23tosky(input_model)
# Compute the V2/V3 for each pixel in this row
# x.shape will be something like (1, 388)
y, x = np.mgrid[row_zero_point:row_zero_point + 1,
0:input_model.data.shape[1]]
spatial_transform = full_distortion | tel2sky
radec = np.array(spatial_transform(x, y))[:, 0, :]
ra_full = np.matlib.repmat(radec[0],
_toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1)
dec_full = np.matlib.repmat(radec[1],
_toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1)
ra_t2d = models.Tabular2D(lookup_table=ra_full,
name='xtable',
bounds_error=False,
fill_value=np.nan)
dec_t2d = models.Tabular2D(lookup_table=dec_full,
name='ytable',
bounds_error=False,
fill_value=np.nan)
# Create the model transforms.
lrs_wav_model = jwmodels.LRSWavelength(lrsdata, zero_point)
# Incorporate the small rotation
angle = np.arctan(0.00421924)
rot = models.Rotation2D(angle)
radec_t2d = ra_t2d & dec_t2d | rot
# Account for the subarray when computing spatial coordinates.
xshift = -bb[0][0]
yshift = -bb[1][0]
det2world = models.Mapping((1, 0, 1, 0, 0, 1)) | models.Shift(yshift, name='yshift1') & \
models.Shift(xshift, name='xshift1') & \
models.Shift(yshift, name='yshift2') & models.Shift(xshift, name='xshift2') & \
models.Identity(2) | radec_t2d & lrs_wav_model
det2world.bounding_box = bb[::-1]
# Now the actual pipeline.
pipeline = [(detector, det2world), (world, None)]
return pipeline
def extract2d(input_model, which_subarray=None):
supported_modes = [
'NRS_FIXEDSLIT', 'NRS_MSASPEC', 'NRS_BRIGHTOBJ', 'NRS_LAMP'
]
exp_type = input_model.meta.exposure.type.upper()
log.info('EXP_TYPE is {0}'.format(exp_type))
if exp_type not in supported_modes:
input_model.meta.cal_step.extract_2d = 'SKIPPED'
return input_model
else:
output_model = datamodels.MultiSlitModel()
output_model.update(input_model)
if exp_type in supported_modes:
slit2msa = input_model.meta.wcs.get_transform('slit_frame',
'msa_frame')
# This is a cludge but will work for now.
# This model keeps open_slits as an attribute.
open_slits = slit2msa[1].slits[:]
if which_subarray is not None:
open_slits = [
sub for sub in open_slits if sub.name == which_subarray
]
log.debug('open slits {0}'.format(open_slits))
for slit in open_slits:
slit_wcs = nirspec.nrs_wcs_set_input(input_model, slit.name)
xlo, xhi = _toindex(slit_wcs.bounding_box[0])
ylo, yhi = _toindex(slit_wcs.bounding_box[1])
# Add the slit offset to each slit WCS object
tr = slit_wcs.get_transform('detector', 'sca')
tr = Shift(xlo) & Shift(ylo) | tr
slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca'))
log.info('Name of subarray extracted: %s', slit.name)
log.info('Subarray x-extents are: %s %s', xlo, xhi)
log.info('Subarray y-extents are: %s %s', ylo, yhi)
ext_data = input_model.data[ylo:yhi + 1, xlo:xhi + 1].copy()
ext_err = input_model.err[ylo:yhi + 1, xlo:xhi + 1].copy()
ext_dq = input_model.dq[ylo:yhi + 1, xlo:xhi + 1].copy()
new_model = datamodels.ImageModel(data=ext_data,
err=ext_err,
dq=ext_dq)
shape = ext_data.shape
bounding_box = ((0, shape[1] - 1), (0, shape[0] - 1))
slit_wcs.bounding_box = bounding_box
new_model.meta.wcs = slit_wcs
output_model.slits.append(new_model)
# set x/ystart values relative to the image (screen) frame.
# The overall subarray offset is recorded in model.meta.subarray.
nslit = len(output_model.slits) - 1
xlo_ind, xhi_ind, ylo_ind, yhi_ind = _toindex(
(xlo, xhi, ylo, yhi)).astype(np.int16)
output_model.slits[nslit].name = str(slit.name)
output_model.slits[nslit].xstart = xlo_ind + 1
output_model.slits[nslit].xsize = (xhi_ind - xlo_ind) + 1
output_model.slits[nslit].ystart = ylo_ind + 1
output_model.slits[nslit].ysize = (yhi_ind - ylo_ind) + 1
if exp_type.lower() == 'nrs_msaspec':
output_model.slits[nslit].source_id = int(slit.source_id)
output_model.slits[nslit].source_name = slit.source_name
output_model.slits[nslit].source_alias = slit.source_alias
output_model.slits[nslit].catalog_id = slit.catalog_id
output_model.slits[nslit].stellarity = float(slit.stellarity)
output_model.slits[nslit].source_xpos = float(slit.source_xpos)
output_model.slits[nslit].source_ypos = float(slit.source_ypos)
output_model.slits[nslit].slitlet_id = int(slit.name)
# for pathloss correction
output_model.slits[nslit].nshutters = int(slit.nshutters)
del input_model
# Set the step status to COMPLETE
output_model.meta.cal_step.extract_2d = 'COMPLETE'
return output_model
def lrs(input_model, reference_files):
"""
The LRS-FIXEDSLIT and LRS-SLITLESS WCS pipeline.
It has two coordinate frames: "detecor" and "world".
Uses the "specwcs" and "distortion" reference files.
"""
# Setup the frames.
detector = cf.Frame2D(name='detector', axes_order=(0, 1), unit=(u.pix, u.pix))
spec = cf.SpectralFrame(name='wavelength', axes_order=(2,), unit=(u.micron,),
axes_names=('lambda',))
sky = cf.CelestialFrame(reference_frame=coord.ICRS(), name='sky')
world = cf.CompositeFrame(name="world", frames=[sky, spec])
# Determine the distortion model.
subarray2full = subarray_transform(input_model)
with DistortionModel(reference_files['distortion']) as dist:
distortion = dist.model
full_distortion = subarray2full | distortion
# Load and process the reference data.
with fits.open(reference_files['specwcs']) as ref:
lrsdata = np.array([l for l in ref[1].data])
# Get the zero point from the reference data.
# The zero_point is X, Y (which should be COLUMN, ROW)
# TODO: Are imx, imy 0- or 1-indexed? We are treating them here as
# 0-indexed. Since they are FITS, they are probably 1-indexed.
if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit':
zero_point = ref[1].header['imx'], ref[1].header['imy']
elif input_model.meta.exposure.type.lower() == 'mir_lrs-slitless':
#zero_point = ref[1].header['imxsltl'], ref[1].header['imysltl']
zero_point = [35, 442] # [35, 763] # account for subarray
# Create the bounding_box
x0 = lrsdata[:, 3]
y0 = lrsdata[:, 4]
x1 = lrsdata[:, 5]
bb = ((x0.min() - 0.5 + zero_point[0], x1.max() + 0.5 + zero_point[0]),
(y0.min() - 0.5 + zero_point[1], y0.max() + 0.5 + zero_point[1]))
# Find the ROW of the zero point which should be the [1] of zero_point
row_zero_point = zero_point[1]
# Compute the v2v3 to sky.
tel2sky = pointing.v23tosky(input_model)
# Compute the V2/V3 for each pixel in this row
# x.shape will be something like (1, 388)
y, x = np.mgrid[row_zero_point:row_zero_point + 1, 0:input_model.data.shape[1]]
spatial_transform = full_distortion | tel2sky
radec = np.array(spatial_transform(x, y))[:, 0, :]
ra_full = np.matlib.repmat(radec[0], _toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1)
dec_full = np.matlib.repmat(radec[1], _toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1)
ra_t2d = models.Tabular2D(lookup_table=ra_full, name='xtable',
bounds_error=False, fill_value=np.nan)
dec_t2d = models.Tabular2D(lookup_table=dec_full, name='ytable',
bounds_error=False, fill_value=np.nan)
# Create the model transforms.
lrs_wav_model = jwmodels.LRSWavelength(lrsdata, zero_point)
try:
velosys = input_model.meta.wcsinfo.velosys
except AttributeError:
pass
else:
if velosys is not None:
velocity_corr = velocity_correction(input_model.meta.wcsinfo.velosys)
lrs_wav_model = lrs_wav_model | velocity_corr
log.info("Applied Barycentric velocity correction : {}".format(velocity_corr[1].amplitude.value))
# Incorporate the small rotation
angle = np.arctan(0.00421924)
rot = models.Rotation2D(angle)
radec_t2d = ra_t2d & dec_t2d | rot
# Account for the subarray when computing spatial coordinates.
xshift = -bb[0][0]
yshift = -bb[1][0]
det2world = models.Mapping((1, 0, 1, 0, 0, 1)) | models.Shift(yshift, name='yshift1') & \
models.Shift(xshift, name='xshift1') & \
models.Shift(yshift, name='yshift2') & models.Shift(xshift, name='xshift2') & \
models.Identity(2) | radec_t2d & lrs_wav_model
det2world.bounding_box = bb[::-1]
# Now the actual pipeline.
pipeline = [(detector, det2world),
(world, None)
]
return pipeline
