def imaging_coords(fname, output=None): """ Computes wavelengths, and space coordinates of a NIRSPEC imaging exposure after running assign_wcs. Parameters ---------- fname : str The name of a file after assign_wcs was run. output : str The name of the output file. If None the root of the input file is used with an extension world_coordinates. Examples -------- >>> compute_world_coordinates('nrs1_fixed_assign_wcs_extract_2d.fits') """ model = datamodels.ImageModel(fname) if model.meta.exposure.type.lower() not in imaging_modes: raise ValueError("Observation mode {0} is not supported.".format(model.meta.exposure.type)) hdulist = fits.HDUList() phdu = fits.PrimaryHDU() phdu.header['filename'] = model.meta.filename phdu.header['data'] = 'world coordinates' hdulist.append(phdu) output_frame = model.available_frames[-1] bb = model.meta.wcs.bounding_box x, y = wcstools.grid_from_bounding_box(bb, step=(1, 1), center=True) ra, dec, lam = slit(x + 1, y + 1) world_coordinates = np.array([lam, ra, dec]) imhdu = fits.ImageHDU(data=world_coordinates) imhdu.header['PLANE1'] = 'lambda, microns' imhdu.header['PLANE2'] = '{0}_x, deg'.format(output_frame) imhdu.header['PLANE3'] = '{0}_y, deg'.format(output_frame) imhdu.header['SLIT'] = "SLIT_{0}".format(i) # add the overall subarray offset imhdu.header['CRVAL1'] = model.meta.subarray.xstart - 1 + int(_toindex(bb[0][0])) imhdu.header['CRVAL2'] = model.meta.subarray.ystart - 1 + int(_toindex(bb[1][0])) # Input coordinates will be 1-based. imhdu.header['CRPIX1'] = 1 imhdu.header['CRPIX2'] = 1 imhdu.header['CTYPE1'] = 'pixel' imhdu.header['CTYPE2'] = 'pixel' hdulist.append(imhdu) if output is not None: base, ext = os.path.splitext(output) if ext != "fits": ext = "fits" if not base.endswith('world_coordinates'): "".join([base, '_world_coordinates']) "".join([base, ext]) else: root = model.meta.filename.split('_') output = "".join([root[0], '_world_coordinates', '.fits']) hdulist.writeto(output, overwrite=True) del hdulist model.close()
def set_slit_attributes(output_model, slit, xlo, xhi, ylo, yhi): """ Set the slit attributes. Parameters ---------- output_model : `~jwst.datamodels.multislit.MultiSlitModel` The output model representing a slit. nslit : int The index o fthis slit in the `~jwst.datamodels.multislit.MultiSlitModel`. xlo, xhi, ylo, yhi : float Indices into the data array where extraction should be done. """ xlo_ind, xhi_ind, ylo_ind, yhi_ind = _toindex( (xlo, xhi, ylo, yhi)).astype(np.int16) output_model.name = str(slit.name) output_model.xstart = xlo_ind + 1 output_model.xsize = (xhi_ind - xlo_ind) + 1 output_model.ystart = ylo_ind + 1 output_model.ysize = (yhi_ind - ylo_ind) + 1 if output_model.meta.exposure.type.lower() == 'nrs_msaspec': output_model.source_id = int(slit.source_id) output_model.source_name = slit.source_name output_model.source_alias = slit.source_alias output_model.stellarity = float(slit.stellarity) output_model.source_xpos = float(slit.source_xpos) output_model.source_ypos = float(slit.source_ypos) output_model.slitlet_id = int(slit.name) # for pathloss correction output_model.shutter_state = slit.shutter_state log.info('set slit_attributes completed')
def offset_wcs(slit_wcs): """ Prepend a Shift transform to the slit WCS to account for subarrays. Parameters ---------- slit_wcs : `~gwcs.wcs.WCS` The WCS for this slit. slit_name : str The name of the slit. """ xlo, xhi = _toindex(slit_wcs.bounding_box[0]) ylo, yhi = _toindex(slit_wcs.bounding_box[1]) # Add the slit offset to each slit WCS object tr = slit_wcs.get_transform('detector', 'sca') tr = Shift(xlo) & Shift(ylo) | tr slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca')) return xlo, xhi, ylo, yhi
def offset_wcs(slit_wcs): """ Prepend a Shift transform to the slit WCS to account for subarrays. Parameters ---------- slit_wcs : `~gwcs.wcs.WCS` The WCS for this slit. slit_name : str The name of the slit. """ xlo, xhi = _toindex(slit_wcs.bounding_box[0]) ylo, yhi = _toindex(slit_wcs.bounding_box[1]) # Add the slit offset to each slit WCS object tr = slit_wcs.get_transform('detector', 'sca') tr = Shift(xlo) & Shift(ylo) | tr slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca')) return xlo, xhi, ylo, yhi
def offset_wcs(slit_wcs, slit_name): """ Prepend a Shift to the slit WCS to account for subarrays. Parameters ---------- slit_wcs : `~gwcs.wcs.WCS` The WCS for this slit. slit_name : str The name of the slit. """ xlo, xhi = _toindex(slit_wcs.bounding_box[0]) ylo, yhi = _toindex(slit_wcs.bounding_box[1]) # Add the slit offset to each slit WCS object tr = slit_wcs.get_transform('detector', 'sca') tr = Shift(xlo) & Shift(ylo) | tr slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca')) log.info('Name of subarray extracted: %s', slit_name) log.info('Subarray x-extents are: %s %s', xlo, xhi) log.info('Subarray y-extents are: %s %s', ylo, yhi) return xlo, xhi, ylo, yhi
def set_slit_attributes(output_model, slit, xlo, xhi, ylo, yhi): """ Set the slit attributes. Parameters ---------- output_model : `~jwst.datamodels.multislit.MultiSlitModel` The output model representing a slit. slit : namedtuple A `~jwst.transforms.models.Slit` object representing a slit. xlo, xhi, ylo, yhi : float Indices into the data array where extraction should be done. These are converted to "pixel indices" - the center of a pixel. """ xlo_ind, xhi_ind, ylo_ind, yhi_ind = _toindex( (xlo, xhi, ylo, yhi)).astype(np.int16) output_model.name = str(slit.name) output_model.xstart = xlo_ind + 1 output_model.xsize = (xhi_ind - xlo_ind) + 1 output_model.ystart = ylo_ind + 1 output_model.ysize = (yhi_ind - ylo_ind) + 1 if output_model.meta.exposure.type.lower() in [ 'nrs_msaspec', 'nrs_autoflat' ]: output_model.source_id = int(slit.source_id) output_model.source_name = slit.source_name output_model.source_alias = slit.source_alias output_model.stellarity = float(slit.stellarity) output_model.source_xpos = float(slit.source_xpos) output_model.source_ypos = float(slit.source_ypos) output_model.slitlet_id = int(slit.name) output_model.quadrant = int(slit.quadrant) output_model.xcen = int(slit.xcen) output_model.ycen = int(slit.ycen) # for pathloss correction output_model.shutter_state = slit.shutter_state log.info('set slit_attributes completed')
def ifu_coords(fname, output=None): """ Computes wavelengths, and space coordinates of a NIRSPEC IFU exposure after running assign_wcs. Parameters ---------- fname : str The name of a file after assign_wcs was run. output : str The name of the output file. If None the root of the input file is used with an extension world_coordinates. Examples -------- >>> compute_world_coordinates('nrs1_fixed_assign_wcs_extract_2d.fits') """ model = datamodels.ImageModel(fname) if model.meta.exposure.type.lower() != 'nrs_ifu': raise ValueError("Expected an IFU observation," "(EXP_TYPE=NRS_IFU), got {0}".format( model.meta.exposure.type)) ifu_slits = nirspec.nrs_ifu_wcs(model) hdulist = fits.HDUList() phdu = fits.PrimaryHDU() phdu.header['filename'] = model.meta.filename phdu.header['data'] = 'world coordinates' hdulist.append(phdu) output_frame = ifu_slits[0].available_frames[-1] for i, slit in enumerate(ifu_slits): x, y = wcstools.grid_from_bounding_box(slit.bounding_box, (1, 1), center=True) ra, dec, lam = slit(x, y) detector2slit = slit.get_transform('detector', 'slit_frame') sx, sy, ls = detector2slit(x, y) world_coordinates = np.array([lam, ra, dec, sy]) imhdu = fits.ImageHDU(data=world_coordinates) imhdu.header['PLANE1'] = 'lambda, microns' imhdu.header['PLANE2'] = '{0}_x, arcsec'.format(output_frame) imhdu.header['PLANE3'] = '{0}_y, arcsec'.format(output_frame) imhdu.header['PLANE4'] = 'slit_y, relative to center (0, 0)' imhdu.header['SLIT'] = "SLIT_{0}".format(i) # -1 and +1 are to express this in 1-based coordinates imhdu.header['CRVAL1'] = model.meta.subarray.xstart - 1 + int( _toindex(slit.bounding_box[0][0])) + 1 imhdu.header['CRVAL2'] = model.meta.subarray.xstart - 1 + int( _toindex(slit.bounding_box[1][0])) + 1 # Input coordinates will be 1-based. imhdu.header['CRPIX1'] = 1 imhdu.header['CRPIX2'] = 1 imhdu.header['CTYPE1'] = 'pixel' imhdu.header['CTYPE2'] = 'pixel' hdulist.append(imhdu) if output is not None: base, ext = os.path.splitext(output) if ext != "fits": ext = "fits" if not base.endswith('world_coordinates'): "".join([base, '_world_coordinates']) "".join([base, ext]) else: root = model.meta.filename.split('_') output = "".join([root[0], '_world_coordinates', '.fits']) hdulist.writeto(output, overwrite=True) del hdulist model.close()
def flattest(step_input_filename, dflatref_path=None, sfile_path=None, fflat_path=None, writefile=False, mk_all_slices_plt=False, show_figs=True, save_figs=False, plot_name=None, threshold_diff=1.0e-7, debug=False): """ This function calculates the difference between the pipeline and the calculated flat field values. The functions uses the output of the compute_world_coordinates.py script. Args: step_input_filename: str, name of the output fits file from the 2d_extract step (with full path) dflatref_path: str, path of where the D-flat reference fits files sfile_path: str, path of where the S-flat reference fits files fflat_path: str, path of where the F-flat reference fits files msa_conf_root: str, path to where the MSA configuration fits file lives writefile: boolean, if True writes the fits files of the calculated flat and difference images show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots (the 3 plots can be saved or not independently with the function call) plot_name: string, desired name (if name is not given, the plot function will name the plot by default) threshold_diff: float, threshold difference between pipeline output and ESA file debug: boolean, if true a series of print statements will show on-screen Returns: - 1 plot, if told to save and/or show. - median_diff: Boolean, True if smaller or equal to 1e-14 - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer flattest_start_time = time.time() # get info from the flat field file file_path = step_input_filename.replace( os.path.basename(step_input_filename), "") det = fits.getval(step_input_filename, "DETECTOR", 0) exptype = fits.getval(step_input_filename, "EXP_TYPE", 0) grat = fits.getval(step_input_filename, "GRATING", 0) filt = fits.getval(step_input_filename, "FILTER", 0) file_basename = os.path.basename(step_input_filename.replace(".fits", "")) msg1 = 'step_input_filename=' + step_input_filename msg2 = "flat_field_file --> Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype print(msg1) print(msg2) log_msgs.append(msg1) log_msgs.append(msg2) # read in the on-the-fly flat image flatfile = step_input_filename.replace("flat_field.fits", "interpolatedflat.fits") pipeflat = fits.getdata(flatfile, "SCI") # get the reference files msg = "Getting and reading the D-, S-, and F-flats for this specific IFU configuration... " print(msg) log_msgs.append(msg) # D-Flat dflat_ending = "f_01.03.fits" dfile = dflatref_path + "_nrs1_" + dflat_ending if det == "NRS2": dfile = dfile.replace("nrs1", "nrs2") msg = "Using D-flat: " + dfile print(msg) log_msgs.append(msg) dfim = fits.getdata(dfile, "SCI") #1) dfimdq = fits.getdata(dfile, "DQ") #4) # need to flip/rotate the image into science orientation ns = np.shape(dfim) dfim = np.transpose(dfim, ( 0, 2, 1)) # keep in mind that 0,1,2 = z,y,x in Python, whereas =x,y,z in IDL dfimdq = np.transpose(dfimdq) if det == "NRS2": # rotate science data by 180 degrees for NRS2 dfim = dfim[..., ::-1, ::-1] dfimdq = dfimdq[..., ::-1, ::-1] naxis3 = fits.getval(dfile, "NAXIS3", "SCI") #1) # get the wavelength values dfwave = np.array([]) for i in range(naxis3): keyword = "PFLAT_" + str(i + 1) dfwave = np.append(dfwave, fits.getval(dfile, keyword, "SCI")) #1)) dfrqe = fits.getdata(dfile, 2) # S-flat tsp = exptype.split("_") mode = tsp[1] if filt == "F070LP": flat = "FLAT4" elif filt == "F100LP": flat = "FLAT1" elif filt == "F170LP": flat = "FLAT2" elif filt == "F290LP": flat = "FLAT3" elif filt == "CLEAR": flat = "FLAT5" else: msg = "No filter correspondence. Exiting the program." print(msg) log_msgs.append(msg) # This is the key argument for the assert pytest function msg = "Test skiped because there is no flat correspondence for the filter in the data: {}".format( filt) median_diff = "skip" return median_diff, msg sflat_ending = "f_01.01.fits" sfile = sfile_path + "_" + grat + "_OPAQUE_" + flat + "_nrs1_" + sflat_ending if debug: print("grat = ", grat) print("flat = ", flat) print("sfile used = ", sfile) if det == "NRS2": sfile = sfile.replace("nrs1", "nrs2") msg = "Using S-flat: " + sfile print(msg) log_msgs.append(msg) sfim = fits.getdata(sfile, "SCI") #1) sfimdq = fits.getdata(sfile, "DQ") #3) # need to flip/rotate image into science orientation sfim = np.transpose(sfim) sfimdq = np.transpose(sfimdq) if det == "NRS2": # rotate science data by 180 degrees for NRS2 sfim = sfim[..., ::-1, ::-1] sfimdq = sfimdq[..., ::-1, ::-1] sfv = fits.getdata(sfile, 5) # F-Flat fflat_ending = "_01.01.fits" if mode in fflat_path: ffile = fflat_path + "_" + filt + fflat_ending else: msg = "Wrong path in for mode F-flat. This script handles mode " + mode + "only." print(msg) log_msgs.append(msg) # This is the key argument for the assert pytest function result_msg = "Wrong path in for mode F-flat. Test skiped because mode is not IFU." median_diff = "skip" return median_diff, result_msg, log_msgs msg = "Using F-flat: " + ffile print(msg) log_msgs.append(msg) ffv = fits.getdata(ffile, "IFU") #1) # now go through each pixel in the test data if writefile: # create the fits list to hold the calculated flat values for each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create the fits list to hold the image of pipeline-calculated difference values hdu0 = fits.PrimaryHDU() complfile = fits.HDUList() complfile.append(hdu0) # get the datamodel from the assign_wcs output file assign_wcs_file = step_input_filename.replace("_flat_field.fits", "_assign_wcs.fits") model = datamodels.ImageModel(assign_wcs_file) ifu_slits = nirspec.nrs_ifu_wcs(model) # loop over the slices all_delfg_mean, all_delfg_mean_arr, all_delfg_median, all_test_result = [], [], [], [] msg = "\n Now looping through the slices, this may take some time... " print(msg) log_msgs.append(msg) for n_ext, slice in enumerate(ifu_slits): if n_ext < 10: pslice = "0" + repr(n_ext) else: pslice = repr(n_ext) msg = "\nWorking with slice: " + pslice print(msg) log_msgs.append(msg) # get the wavelength # slice.x(y)start are 1-based, turn them to 0-based for extraction x, y = wcstools.grid_from_bounding_box(slice.bounding_box, (1, 1), center=True) ra, dec, wave = slice(x, y) # get the subwindow origin (technically no subwindows for IFU, but need this for comparing to the # full frame on-the-fly flat image). px0 = model.meta.subarray.xstart - 1 + int( _toindex(slice.bounding_box[0][0])) + 1 py0 = model.meta.subarray.xstart - 1 + int( _toindex(slice.bounding_box[1][0])) + 1 n_p = np.shape(wave) nx, ny = n_p[1], n_p[0] nw = nx * ny msg = " Subwindow origin: px0=" + repr(px0) + " py0=" + repr(py0) print(msg) log_msgs.append(msg) if debug: print("n_p = ", n_p) print("nw = ", nw) # initialize arrays of the right size delf = np.zeros([nw]) + 999.0 flatcor = np.zeros([nw]) + 999.0 sffarr = np.zeros([nw]) calc_flat = np.zeros([2048, 2048]) + 999.0 # loop through the wavelengths msg = " Looping through the wavelngth, this may take a little time ... " print(msg) log_msgs.append(msg) flat_wave = wave.flatten() wave_shape = np.shape(wave) for j in range(0, nw): if np.isfinite(flat_wave[j]): # skip if wavelength is NaN # get the pixel indeces jwav = flat_wave[j] t = np.where(wave == jwav) pind = [t[0][0] + py0 - 1, t[1][0] + px0 - 1 ] # pind =[pixel_y, pixe_x] in python, [x, y] in IDL if debug: print('j, jwav, px0, py0 : ', j, jwav, px0, py0) print('pind[0], pind[1] = ', pind[0], pind[1]) # get the pixel bandwidth **this needs to be modified for prism, since the dispersion is not linear!** delw = 0.0 if (j != 0) and (int((j - 1) / nx) == int(j / nx)) and (int( (j + 1) / nx) == int(j / nx)) and np.isfinite( flat_wave[j + 1]) and np.isfinite(flat_wave[j - 1]): delw = 0.5 * (flat_wave[j + 1] - flat_wave[j - 1]) if (j == 0) or not np.isfinite(flat_wave[j - 1]) or (int( (j - 1) / nx) != int(j / nx)): delw = 0.5 * (flat_wave[j + 1] - flat_wave[j]) if (j == nw - 1) or not np.isfinite(flat_wave[j + 1]) or (int( (j + 1) / nx) != int(j / nx)): delw = 0.5 * (flat_wave[j] - flat_wave[j - 1]) if debug: #print("(j, (j-1), nx, (j-1)/nx, (j+1), (j+1)/nx)", j, (j-1), nx, int((j-1)/nx), (j+1), int((j+1)/nx)) #print("np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1])", np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1])) #print("flat_wave[j+1], flat_wave[j-1] : ", np.isfinite(flat_wave[j+1]), flat_wave[j+1], flat_wave[j-1]) print("delw = ", delw) # integrate over D-flat fast vector dfrqe_wav = dfrqe.field("WAVELENGTH") dfrqe_rqe = dfrqe.field("RQE") iw = np.where((dfrqe_wav >= jwav - delw / 2.0) & (dfrqe_wav <= jwav + delw / 2.0)) if np.size(iw) == 0: iw = -1 int_tab = auxfunc.idl_tabulate(dfrqe_wav[iw], dfrqe_rqe[iw]) if int_tab == 0: int_tab = np.interp(dfrqe_wav[iw], dfrqe_wav, dfrqe_rqe) dff = int_tab else: first_dfrqe_wav, last_dfrqe_wav = dfrqe_wav[iw][ 0], dfrqe_wav[iw][-1] dff = int_tab / (last_dfrqe_wav - first_dfrqe_wav) if debug: #print("np.shape(dfrqe_wav) : ", np.shape(dfrqe_wav)) #print("np.shape(dfrqe_rqe) : ", np.shape(dfrqe_rqe)) #print("dfimdq[pind[0]][pind[1]] : ", dfimdq[pind[0]][pind[1]]) #print("np.shape(iw) =", np.shape(iw)) #print("np.shape(dfrqe_wav[iw[0]]) = ", np.shape(dfrqe_wav[iw[0]])) #print("np.shape(dfrqe_rqe[iw[0]]) = ", np.shape(dfrqe_rqe[iw[0]])) #print("int_tab=", int_tab) print("np.shape(iw) = ", np.shape(iw)) print("iw = ", iw) print("dff = ", dff) # interpolate over D-flat cube dfs = 1.0 if dfimdq[pind[0], pind[1]] == 0: dfs = np.interp(jwav, dfwave, dfim[:, pind[0], pind[1]]) # integrate over S-flat fast vector sfv_wav = sfv.field("WAVELENGTH") sfv_dat = sfv.field("DATA") if (jwav < 5.3) and (jwav > 0.6): iw = np.where((sfv_wav >= jwav - delw / 2.0) & (sfv_wav <= jwav + delw / 2.0)) if np.size(iw) == 0: iw = -1 if np.size(iw) > 1: int_tab = auxfunc.idl_tabulate(sfv_wav[iw], sfv_dat[iw]) first_sfv_wav, last_sfv_wav = sfv_wav[iw][0], sfv_wav[ iw][-1] sff = int_tab / (last_sfv_wav - first_sfv_wav) elif np.size(iw) == 1: sff = float(sfv_dat[iw]) else: sff = 999.0 # get s-flat pixel-dependent correction sfs = 1.0 if sfimdq[pind[0], pind[1]] == 0: sfs = sfim[pind[0], pind[1]] if debug: print("jwav-delw/2.0 = ", jwav - delw / 2.0) print("jwav+delw/2.0 = ", jwav + delw / 2.0) print("np.shape(sfv_wav), sfv_wav[-1] = ", np.shape(sfv_wav), sfv_wav[-1]) print("iw = ", iw) print("sfv_wav[iw] = ", sfv_wav[iw]) print("int_tab = ", int_tab) print("first_sfv_wav, last_sfv_wav = ", first_sfv_wav, last_sfv_wav) print("sfs = ", sfs) print("sff = ", sff) # integrate over f-flat fast vector # reference file blue cutoff is 1 micron, so need to force solution for shorter wavs ffv_wav = ffv.field("WAVELENGTH") ffv_dat = ffv.field("DATA") fff = 1.0 if jwav - delw / 2.0 >= 1.0: iw = np.where((ffv_wav >= jwav - delw / 2.0) & (ffv_wav <= jwav + delw / 2.0)) if np.size(iw) == 0: iw = -1 if np.size(iw) > 1: int_tab = auxfunc.idl_tabulate(ffv_wav[iw], ffv_dat[iw]) first_ffv_wav, last_ffv_wav = ffv_wav[iw][0], ffv_wav[ iw][-1] fff = int_tab / (last_ffv_wav - first_ffv_wav) elif np.size(iw) == 1: fff = float(ffv_dat[iw]) flatcor[j] = dff * dfs * sff * sfs * fff sffarr[j] = sff # To visually compare between the pipeline flat and the calculated one (e.g. in ds9), Phil Hodge # suggested using the following line: calc_flat[pind[0], pind[1]] = flatcor[j] # this line writes the calculated flat into a full frame array # then this new array needs to be written into a file. This part has not been done yet. # Difference between pipeline and calculated values delf[j] = pipeflat[pind[0], pind[1]] - flatcor[j] # Remove all pixels with values=1 (mainly inter-slit pixels) for statistics if pipeflat[pind[0], pind[1]] == 1: delf[j] = 999.0 if np.isnan(jwav): flatcor[j] = 1.0 # no correction if no wavelength if debug: print("np.shape(iw) = ", np.shape(iw)) print("fff = ", fff) print("flatcor[j] = ", flatcor[j]) print("delf[j] = ", delf[j]) # ignore outliers for calculating median delfg = delf[np.where(delf != 999.0)] #delfg_median, delfg_std = np.median(delfg), np.std(delfg) msg = "Flat value differences for slice number: " + pslice print(msg) log_msgs.append(msg) #print(" median = ", delfg_median, " stdev =", delfg_std) stats_and_strings = auxfunc.print_stats(delfg, "Flat Difference", float(threshold_diff), abs=True) stats, stats_print_strings = stats_and_strings delfg_mean, delfg_median, delfg_std = stats for msg in stats_print_strings: log_msgs.append(msg) if debug: print("np.shape(delf) = ", np.shape(delf)) print("np.shape(delfg) = ", np.shape(delfg)) all_delfg_mean.append(delfg_mean) all_delfg_median.append(delfg_median) # make the slice plot if np.isfinite(delfg_median) and (len(delfg) != 0): if show_figs or save_figs: msg = "Making the plot for this slice..." print(msg) log_msgs.append(msg) # create histogram t = (file_basename, det, pslice, "IFUflatcomp_histogram") title = filt + " " + grat + " SLICE=" + pslice + "\n" plot_name = "".join((file_path, ("_".join(t)) + ".pdf")) #mk_hist(title, delfg, delfg_mean, delfg_median, delfg_std, save_figs, show_figs, plot_name=plot_name) bins = None # binning for the histograms, if None the function will select them automatically title = title + "Residuals" info_img = [title, "x (pixels)", "y (pixels)"] xlabel, ylabel = "flat$_{pipe}$ - flat$_{calc}$", "N" info_hist = [xlabel, ylabel, bins, stats] if delfg[1] is np.nan: msg = "Unable to create plot of relative wavelength difference." print(msg) log_msgs.append(msg) else: plt_name = os.path.join(file_path, plot_name) difference_img = (pipeflat - calc_flat) #/calc_flat in_slit = np.logical_and( difference_img < 900.0, difference_img > -900.0) # ignore points out of the slit, difference_img[ ~in_slit] = np.nan # Set values outside the slit to NaN nanind = np.isnan( difference_img) # get all the nan indexes difference_img[ nanind] = np.nan # set all nan indexes to have a value of nan plt_origin = None limits = [px0 - 5, px0 + 1500, py0 - 5, py0 + 55] vminmax = [ -5 * delfg_std, 5 * delfg_std ] # set the range of values to be shown in the image, will affect color scale auxfunc.plt_two_2Dimgandhist(difference_img, delfg, info_img, info_hist, plt_name=plt_name, limits=limits, vminmax=vminmax, plt_origin=plt_origin, show_figs=show_figs, save_figs=save_figs) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." print(msg) log_msgs.append(msg) if writefile: # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(flatcor.reshape(wave_shape), name=pslice) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values complfile_ext = fits.ImageHDU(delf.reshape(wave_shape), name=pslice) complfile.append(complfile_ext) # the file is not yet written, indicate that this slit was appended to list to be written msg = "Extension " + repr( n_ext ) + " appended to list to be written into calculated and comparison fits files." print(msg) log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(delfg_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) all_test_result.append(test_result) # if the test is failed exit the script if (delfg_median == 999.0) or not np.isfinite(delfg_median): msg = "Unable to determine mean, meadian, and std_dev for the slice" + pslice print(msg) log_msgs.append(msg) if mk_all_slices_plt: if show_figs or save_figs: # create histogram t = (file_basename, det, "all_slices_IFU_flatcomp_histogram") title = ("_".join(t)) # calculate median of medians and std_dev of medians all_delfg_median_arr = np.array(all_delfg_median) mean_of_delfg_mean = np.mean(all_delfg_mean_arr) median_of_delfg_median = np.median(all_delfg_median_arr) medians_std = np.std(median_of_delfg_median) plot_name = "".join((file_path, title, ".pdf")) mk_hist(title, all_delfg_median_arr, mean_of_delfg_mean, median_of_delfg_median, medians_std, save_figs, show_figs, plot_name=plot_name) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." print(msg) log_msgs.append(msg) # create fits file to hold the calculated flat for each slice if writefile: outfile_name = step_input_filename.replace("flat_field.fits", det + "_flat_calc.fits") complfile_name = step_input_filename.replace("flat_field.fits", det + "_flat_comp.fits") # create the fits list to hold the calculated flat values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values complfile.writeto(complfile_name, overwrite=True) msg = "Fits file with calculated flat values of each slice saved as: " print(msg) print(outfile_name) log_msgs.append(msg) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline flat - calculated flat) saved as: " print(msg) print(complfile_name) log_msgs.append(msg) log_msgs.append(complfile_name) # If all tests passed then pytest will be marked as PASSED, else it will be FAILED FINAL_TEST_RESULT = True for t in all_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break if FINAL_TEST_RESULT: msg = "\n *** Final result for flat_field test will be reported as PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slices PASSED flat_field test." else: msg = "\n *** Final result for flat_field test will be reported as FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slices FAILED flat_field test." # end the timer flattest_end_time = time.time() - flattest_start_time if flattest_end_time > 60.0: flattest_end_time = flattest_end_time / 60.0 # in minutes flattest_tot_time = "* Script flattest_ifu.py script took ", repr( flattest_end_time) + " minutes to finish." if flattest_end_time > 60.0: flattest_end_time = flattest_end_time / 60. # in hours flattest_tot_time = "* Script flattest_ifu.py took ", repr( flattest_end_time) + " hours to finish." else: flattest_tot_time = "* Script flattest_ifu.py took ", repr( flattest_end_time) + " seconds to finish." print(flattest_tot_time) log_msgs.append(flattest_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs
def lrs(input_model, reference_files): """ The LRS-FIXEDSLIT and LRS-SLITLESS WCS pipeline. It has two coordinate frames: "detecor" and "world". Uses the "specwcs" and "distortion" reference files. """ # Setup the frames. detector = cf.Frame2D(name='detector', axes_order=(0, 1), unit=(u.pix, u.pix)) spec = cf.SpectralFrame(name='wavelength', axes_order=(2, ), unit=(u.micron, ), axes_names=('lambda', )) sky = cf.CelestialFrame(reference_frame=coord.ICRS(), name='sky') world = cf.CompositeFrame(name="world", frames=[sky, spec]) # Determine the distortion model. subarray2full = subarray_transform(input_model) with DistortionModel(reference_files['distortion']) as dist: distortion = dist.model full_distortion = subarray2full | distortion # Load and process the reference data. with fits.open(reference_files['specwcs']) as ref: lrsdata = np.array([l for l in ref[1].data]) # Get the zero point from the reference data. # The zero_point is X, Y (which should be COLUMN, ROW) # TODO: Are imx, imy 0- or 1-indexed? We are treating them here as # 0-indexed. Since they are FITS, they are probably 1-indexed. if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit': zero_point = ref[1].header['imx'], ref[1].header['imy'] elif input_model.meta.exposure.type.lower() == 'mir_lrs-slitless': #zero_point = ref[1].header['imxsltl'], ref[1].header['imysltl'] zero_point = [35, 442] # [35, 763] # account for subarray # Create the bounding_box x0 = lrsdata[:, 3] y0 = lrsdata[:, 4] x1 = lrsdata[:, 5] bb = ((x0.min() - 0.5 + zero_point[0], x1.max() + 0.5 + zero_point[0]), (y0.min() - 0.5 + zero_point[1], y0.max() + 0.5 + zero_point[1])) # Find the ROW of the zero point which should be the [1] of zero_point row_zero_point = zero_point[1] # Compute the v2v3 to sky. tel2sky = pointing.v23tosky(input_model) # Compute the V2/V3 for each pixel in this row # x.shape will be something like (1, 388) y, x = np.mgrid[row_zero_point:row_zero_point + 1, 0:input_model.data.shape[1]] spatial_transform = full_distortion | tel2sky radec = np.array(spatial_transform(x, y))[:, 0, :] ra_full = np.matlib.repmat(radec[0], _toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1) dec_full = np.matlib.repmat(radec[1], _toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1) ra_t2d = models.Tabular2D(lookup_table=ra_full, name='xtable', bounds_error=False, fill_value=np.nan) dec_t2d = models.Tabular2D(lookup_table=dec_full, name='ytable', bounds_error=False, fill_value=np.nan) # Create the model transforms. lrs_wav_model = jwmodels.LRSWavelength(lrsdata, zero_point) # Incorporate the small rotation angle = np.arctan(0.00421924) rot = models.Rotation2D(angle) radec_t2d = ra_t2d & dec_t2d | rot # Account for the subarray when computing spatial coordinates. xshift = -bb[0][0] yshift = -bb[1][0] det2world = models.Mapping((1, 0, 1, 0, 0, 1)) | models.Shift(yshift, name='yshift1') & \ models.Shift(xshift, name='xshift1') & \ models.Shift(yshift, name='yshift2') & models.Shift(xshift, name='xshift2') & \ models.Identity(2) | radec_t2d & lrs_wav_model det2world.bounding_box = bb[::-1] # Now the actual pipeline. pipeline = [(detector, det2world), (world, None)] return pipeline
def extract2d(input_model, which_subarray=None): supported_modes = [ 'NRS_FIXEDSLIT', 'NRS_MSASPEC', 'NRS_BRIGHTOBJ', 'NRS_LAMP' ] exp_type = input_model.meta.exposure.type.upper() log.info('EXP_TYPE is {0}'.format(exp_type)) if exp_type not in supported_modes: input_model.meta.cal_step.extract_2d = 'SKIPPED' return input_model else: output_model = datamodels.MultiSlitModel() output_model.update(input_model) if exp_type in supported_modes: slit2msa = input_model.meta.wcs.get_transform('slit_frame', 'msa_frame') # This is a cludge but will work for now. # This model keeps open_slits as an attribute. open_slits = slit2msa[1].slits[:] if which_subarray is not None: open_slits = [ sub for sub in open_slits if sub.name == which_subarray ] log.debug('open slits {0}'.format(open_slits)) for slit in open_slits: slit_wcs = nirspec.nrs_wcs_set_input(input_model, slit.name) xlo, xhi = _toindex(slit_wcs.bounding_box[0]) ylo, yhi = _toindex(slit_wcs.bounding_box[1]) # Add the slit offset to each slit WCS object tr = slit_wcs.get_transform('detector', 'sca') tr = Shift(xlo) & Shift(ylo) | tr slit_wcs.set_transform('detector', 'sca', tr.rename('dms2sca')) log.info('Name of subarray extracted: %s', slit.name) log.info('Subarray x-extents are: %s %s', xlo, xhi) log.info('Subarray y-extents are: %s %s', ylo, yhi) ext_data = input_model.data[ylo:yhi + 1, xlo:xhi + 1].copy() ext_err = input_model.err[ylo:yhi + 1, xlo:xhi + 1].copy() ext_dq = input_model.dq[ylo:yhi + 1, xlo:xhi + 1].copy() new_model = datamodels.ImageModel(data=ext_data, err=ext_err, dq=ext_dq) shape = ext_data.shape bounding_box = ((0, shape[1] - 1), (0, shape[0] - 1)) slit_wcs.bounding_box = bounding_box new_model.meta.wcs = slit_wcs output_model.slits.append(new_model) # set x/ystart values relative to the image (screen) frame. # The overall subarray offset is recorded in model.meta.subarray. nslit = len(output_model.slits) - 1 xlo_ind, xhi_ind, ylo_ind, yhi_ind = _toindex( (xlo, xhi, ylo, yhi)).astype(np.int16) output_model.slits[nslit].name = str(slit.name) output_model.slits[nslit].xstart = xlo_ind + 1 output_model.slits[nslit].xsize = (xhi_ind - xlo_ind) + 1 output_model.slits[nslit].ystart = ylo_ind + 1 output_model.slits[nslit].ysize = (yhi_ind - ylo_ind) + 1 if exp_type.lower() == 'nrs_msaspec': output_model.slits[nslit].source_id = int(slit.source_id) output_model.slits[nslit].source_name = slit.source_name output_model.slits[nslit].source_alias = slit.source_alias output_model.slits[nslit].catalog_id = slit.catalog_id output_model.slits[nslit].stellarity = float(slit.stellarity) output_model.slits[nslit].source_xpos = float(slit.source_xpos) output_model.slits[nslit].source_ypos = float(slit.source_ypos) output_model.slits[nslit].slitlet_id = int(slit.name) # for pathloss correction output_model.slits[nslit].nshutters = int(slit.nshutters) del input_model # Set the step status to COMPLETE output_model.meta.cal_step.extract_2d = 'COMPLETE' return output_model
def lrs(input_model, reference_files): """ The LRS-FIXEDSLIT and LRS-SLITLESS WCS pipeline. It has two coordinate frames: "detecor" and "world". Uses the "specwcs" and "distortion" reference files. """ # Setup the frames. detector = cf.Frame2D(name='detector', axes_order=(0, 1), unit=(u.pix, u.pix)) spec = cf.SpectralFrame(name='wavelength', axes_order=(2,), unit=(u.micron,), axes_names=('lambda',)) sky = cf.CelestialFrame(reference_frame=coord.ICRS(), name='sky') world = cf.CompositeFrame(name="world", frames=[sky, spec]) # Determine the distortion model. subarray2full = subarray_transform(input_model) with DistortionModel(reference_files['distortion']) as dist: distortion = dist.model full_distortion = subarray2full | distortion # Load and process the reference data. with fits.open(reference_files['specwcs']) as ref: lrsdata = np.array([l for l in ref[1].data]) # Get the zero point from the reference data. # The zero_point is X, Y (which should be COLUMN, ROW) # TODO: Are imx, imy 0- or 1-indexed? We are treating them here as # 0-indexed. Since they are FITS, they are probably 1-indexed. if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit': zero_point = ref[1].header['imx'], ref[1].header['imy'] elif input_model.meta.exposure.type.lower() == 'mir_lrs-slitless': #zero_point = ref[1].header['imxsltl'], ref[1].header['imysltl'] zero_point = [35, 442] # [35, 763] # account for subarray # Create the bounding_box x0 = lrsdata[:, 3] y0 = lrsdata[:, 4] x1 = lrsdata[:, 5] bb = ((x0.min() - 0.5 + zero_point[0], x1.max() + 0.5 + zero_point[0]), (y0.min() - 0.5 + zero_point[1], y0.max() + 0.5 + zero_point[1])) # Find the ROW of the zero point which should be the [1] of zero_point row_zero_point = zero_point[1] # Compute the v2v3 to sky. tel2sky = pointing.v23tosky(input_model) # Compute the V2/V3 for each pixel in this row # x.shape will be something like (1, 388) y, x = np.mgrid[row_zero_point:row_zero_point + 1, 0:input_model.data.shape[1]] spatial_transform = full_distortion | tel2sky radec = np.array(spatial_transform(x, y))[:, 0, :] ra_full = np.matlib.repmat(radec[0], _toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1) dec_full = np.matlib.repmat(radec[1], _toindex(bb[1][1]) + 1 - _toindex(bb[1][0]), 1) ra_t2d = models.Tabular2D(lookup_table=ra_full, name='xtable', bounds_error=False, fill_value=np.nan) dec_t2d = models.Tabular2D(lookup_table=dec_full, name='ytable', bounds_error=False, fill_value=np.nan) # Create the model transforms. lrs_wav_model = jwmodels.LRSWavelength(lrsdata, zero_point) try: velosys = input_model.meta.wcsinfo.velosys except AttributeError: pass else: if velosys is not None: velocity_corr = velocity_correction(input_model.meta.wcsinfo.velosys) lrs_wav_model = lrs_wav_model | velocity_corr log.info("Applied Barycentric velocity correction : {}".format(velocity_corr[1].amplitude.value)) # Incorporate the small rotation angle = np.arctan(0.00421924) rot = models.Rotation2D(angle) radec_t2d = ra_t2d & dec_t2d | rot # Account for the subarray when computing spatial coordinates. xshift = -bb[0][0] yshift = -bb[1][0] det2world = models.Mapping((1, 0, 1, 0, 0, 1)) | models.Shift(yshift, name='yshift1') & \ models.Shift(xshift, name='xshift1') & \ models.Shift(yshift, name='yshift2') & models.Shift(xshift, name='xshift2') & \ models.Identity(2) | radec_t2d & lrs_wav_model det2world.bounding_box = bb[::-1] # Now the actual pipeline. pipeline = [(detector, det2world), (world, None) ] return pipeline