def pathtest(step_input_filename, reffile, comparison_filename, writefile=True, show_figs=True, save_figs=False, threshold_diff=1.0e-7, debug=False): """ This function calculates the difference between the pipeline and calculated pathloss values. Args: step_input_filename: str, full path name of sourcetype output fits file reffile: str, path to the pathloss FS reference fits files comparison_filename: str, path to comparison pipeline pathloss file writefile: boolean, if True writes the fits files of calculated pathloss and difference images show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots threshold_diff: float, threshold difference between pipeline output and ESA file debug: boolean, if true print statements will show on-screen Returns: - 1 plot, if told to save and/or show them. - median_diff: Boolean, True if smaller or equal to threshold. - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer pathtest_start_time = time.time() # get info from the rate file header msg = 'step_input_filename=' + step_input_filename print(msg) log_msgs.append(msg) exptype = fits.getval(step_input_filename, "EXP_TYPE", 0) grat = fits.getval(step_input_filename, "GRATING", 0) filt = fits.getval(step_input_filename, "FILTER", 0) msg = "pathloss_file: Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype print(msg) log_msgs.append(msg) # get the reference files msg = "Using reference file: " + reffile print(msg) log_msgs.append(msg) is_point_source = False if writefile: # create the fits list to hold the calculated flat values for each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create fits list to hold pipeline-calculated difference values hdu0 = fits.PrimaryHDU() compfile = fits.HDUList() compfile.append(hdu0) # list to determine if pytest is passed or not total_test_result = [] # get all the science extensions if not is_point_source: ext = 3 # only one option for IFU_UNI # read in 2D spectra output prior to pathloss: print("""Checking if files exist and obtaining datamodels. This takes a few minutes...""") if os.path.isfile(comparison_filename): if debug: print('Comparison file does exist.') else: result_msg = "Comparison file does NOT exist. Skipping pathloss test." print(result_msg) log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the comparison data model ifu_pipe_model = datamodels.open(comparison_filename) if debug: print('got comparison datamodel!') if os.path.isfile(step_input_filename): if debug: print('Input file does exist.') else: result_msg = 'Input file does NOT exist. Skipping pathloss test.' log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the input data model ifu_input_model = datamodels.open(step_input_filename) if debug: print('got input datamodel!') # get slices (instead of using .slit) pl_ifu_slits = nirspec.nrs_ifu_wcs(ifu_input_model) print("got input slices") plcor_ref_ext = fits.getdata(reffile, ext) hdul = fits.open(reffile) plcor_ref = hdul[1].data print("PLCOR_REF.shape", plcor_ref.shape) w = wcs.WCS(hdul[1].header) w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref.shape[1], :plcor_ref. shape[2]] slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0) # these are full 2048 * 2048 files: previous_sci = fits.getdata(step_input_filename, "SCI") comp_sci = fits.getdata(comparison_filename, "SCI") pathloss_divided = comp_sci / previous_sci # set up generals for all plots font = {'weight': 'normal', 'size': 10} matplotlib.rc('font', **font) # loop through the slices msg = " Looping through the slices... " print(msg) log_msgs.append(msg) slit_list = np.ndarray.tolist(np.arange(0, 30)) for slit, slit_num in zip(pl_ifu_slits, slit_list): print("working with slice {}".format(slit_num)) x, y = wcstools.grid_from_bounding_box(slit.bounding_box, step=(1, 1)) ra, dec, wave = slit(x, y) wave_sci = wave * 10**(-6) corr_vals = np.interp(wave_sci, wave_ref[:, 0, 0], plcor_ref_ext) box = slit.bounding_box small_y = box[0][0] big_y = box[0][1] small_x = box[1][0] big_x = box[1][1] left = int(math.trunc(small_x)) right = int(math.ceil(big_x)) bottom = int(math.trunc(small_y)) top = int(math.ceil(big_y)) full_cut2slice = previous_sci[left:right, bottom:top] print("SHAPES", full_cut2slice.shape, corr_vals.shape) if full_cut2slice.shape != corr_vals.shape: value = 0 while ((((full_cut2slice.shape[0] - corr_vals.shape[0]) != 0) or ((full_cut2slice.shape[1] - corr_vals.shape[1]) != 0)) and value < 7): # can delete second criteria once all pass if value == 6: print("WARNING: may be in infinite loop!") x_amount_off = full_cut2slice.shape[0] - corr_vals.shape[0] if x_amount_off >= 1: if x_amount_off % 2 == 0: # need 2 more vals: 1 per side right = right - 1 left = left + 1 print("ALTERED SHAPE OF SLICE: V1") value = value + 1 else: # just add one value left = left + 1 print("ALTERED SHAPE OF SLICE: V2") value = value + 1 elif x_amount_off <= -1: if x_amount_off % 2 == 0: right = right + 1 left = left - 1 print("ALTERED SHAPE OF SLICE: V3") value = value + 1 else: left = left - 1 print("ALTERED SHAPE OF SLICE: V4") value = value + 1 y_amount_off = full_cut2slice.shape[1] - corr_vals.shape[1] if y_amount_off >= 1: if y_amount_off % 2 == 0: bottom = bottom - 1 top = top + 1 print("ALTERED SHAPE OF SLICE: V5") value = value + 1 else: bottom = bottom + 1 print("ALTERED SHAPE OF SLICE: V6") value = value + 1 elif y_amount_off <= -1: if y_amount_off % 2 == 0: top = top + 1 bottom = bottom - 1 print("ALTERED SHAPE OF SLICE: V7") value = value + 1 else: bottom = bottom - 1 print("ALTERED SHAPE OF SLICE: V8") value = value + 1 full_cut2slice = previous_sci[left:right, bottom:top] print("final left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) print("NEW SHAPE OF SLICE: {} and corr_vals.shape: {}".format( full_cut2slice.shape, corr_vals.shape)) if full_cut2slice.shape != corr_vals.shape: print("shapes did not match! full_cut2slice: {}, corr_vals {}". format(full_cut2slice.shape, corr_vals.shape)) continue corrected_array = full_cut2slice / corr_vals pipe_correction = pathloss_divided[left:right, bottom:top] if pipe_correction.shape != corr_vals.shape: print("shapes did not match! pipe_correction: {}, corr_vals {}". format(pipe_correction.shape, corr_vals.shape)) continue prev_sci_slit = previous_sci[left:right, bottom:top] if prev_sci_slit.shape != corr_vals.shape: print( "shapes did not match! prev_sci_slit: {}, corr_vals {}".format( prev_sci_slit.shape, corr_vals.shape)) continue comp_sci_slit = comp_sci[left:right, bottom:top] if comp_sci_slit.shape != corr_vals.shape: print( "shapes did not match! comp_sci_slit: {}, corr_vals {}".format( comp_sci_slit.shape, corr_vals.shape)) continue # Plots: step_input_filepath = step_input_filename.replace(".fits", "") # my correction values fig = plt.figure(figsize=(15, 15)) plt.subplot(221) norm = ImageNormalize(corr_vals) plt.imshow(corr_vals, vmin=0.999995, vmax=1.000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Calculated Correction') plt.colorbar() # pipe correction plt.subplot(222) norm = ImageNormalize(pipe_correction) plt.imshow(pipe_correction, vmin=0.999995, vmax=1.000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Pipe Correction') plt.colorbar() # residuals (pipe correction - my correction) if pipe_correction.shape == corr_vals.shape: corr_residuals = pipe_correction - corr_vals plt.subplot(223) norm = ImageNormalize(corr_residuals) plt.imshow(corr_residuals, vmin=-0.000000005, vmax=0.000000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Correction residuals') plt.colorbar() # my science data after pathloss plt.subplot(224) norm = ImageNormalize(corrected_array) plt.imshow(corrected_array, vmin=0, vmax=300, aspect=10.0, origin='lower', cmap='viridis') plt.title('My slit science data after pathloss') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.colorbar() fig.suptitle("IFU UNI Pathloss Correction Testing") if show_figs: plt.show() if save_figs: plt_name = step_input_filepath + "_Pathloss_test_IFU_UNI_slit_" + str( slit_num) + ".png" plt.savefig(plt_name) print('Figure saved as: ', plt_name) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." if debug: print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." if debug: print(msg) log_msgs.append(msg) plt.close() # create fits file to hold the calculated pathloss for each slit if writefile: msg = "Saving the fits files with the calculated pathloss for each slit..." print(msg) log_msgs.append(msg) # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(corr_vals, name=str(slit_num)) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values compfile_ext = fits.ImageHDU(corr_residuals, name=str(slit_num)) compfile.append(compfile_ext) if corr_residuals[~np.isnan(corr_residuals)].size == 0: msg1 = " * Unable to calculate statistics because difference array has all values as NaN. " \ "Test will be set to FAILED." print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: msg = "Calculating statistics... " print(msg) log_msgs.append(msg) # ignore outliers: corr_residuals = corr_residuals[np.where((corr_residuals != 999.0) & (corr_residuals < 0.1) & (corr_residuals > -0.1))] if corr_residuals.size == 0: msg1 = """Unable to calculate statistics because difference array has all outlier values. Test will be set to FAILED.""" print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: stats_and_strings = auxfunc.print_stats(corr_residuals, "Difference", float(threshold_diff), abs=True) stats, stats_print_strings = stats_and_strings corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats for msg in stats_print_strings: log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(corr_residuals_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) total_test_result.append(test_result) if writefile: outfile_name = step_input_filename.replace( "srctype", "_calcuated_FS_UNI_pathloss") compfile_name = step_input_filename.replace( "srctype", "_comparison_FS_UNI_pathloss") # create the fits list to hold the calculated flat values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values compfile.writeto(compfile_name, overwrite=True) msg = "\nFits file with calculated pathloss values of each slit saved as: " print(msg) log_msgs.append(msg) print(outfile_name) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: " print(msg) log_msgs.append(msg) print(compfile_name) log_msgs.append(compfile_name) # If all tests passed then pytest will be marked as PASSED, else FAILED FINAL_TEST_RESULT = False for t in total_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break else: FINAL_TEST_RESULT = True if FINAL_TEST_RESULT: msg = "\n *** Final result of pathloss test is PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slits PASSED path_loss test." else: msg = "\n *** Final result of pathloss test is FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slits FAILED path_loss test." # end the timer pathloss_end_time = time.time() - pathtest_start_time if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60.0 # in minutes pathloss_tot_time = "* Script ifu_uni.py took ", repr( pathloss_end_time) + " minutes to finish." if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60. # in hours pathloss_tot_time = "* Script ifu_uni.py took ", repr( pathloss_end_time) + " hours to finish." else: pathloss_tot_time = "* Script ifu_uni.py took ", repr( pathloss_end_time) + " seconds to finish." print(pathloss_tot_time) log_msgs.append(pathloss_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs
def flattest(step_input_filename, dflatref_path=None, sfile_path=None, fflat_path=None, writefile=False, mk_all_slices_plt=False, show_figs=True, save_figs=False, plot_name=None, threshold_diff=1.0e-7, debug=False): """ This function calculates the difference between the pipeline and the calculated flat field values. The functions uses the output of the compute_world_coordinates.py script. Args: step_input_filename: str, name of the output fits file from the 2d_extract step (with full path) dflatref_path: str, path of where the D-flat reference fits files sfile_path: str, path of where the S-flat reference fits files fflat_path: str, path of where the F-flat reference fits files msa_conf_root: str, path to where the MSA configuration fits file lives writefile: boolean, if True writes the fits files of the calculated flat and difference images show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots (the 3 plots can be saved or not independently with the function call) plot_name: string, desired name (if name is not given, the plot function will name the plot by default) threshold_diff: float, threshold difference between pipeline output and ESA file debug: boolean, if true a series of print statements will show on-screen Returns: - 1 plot, if told to save and/or show. - median_diff: Boolean, True if smaller or equal to 1e-14 - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer flattest_start_time = time.time() # get info from the flat field file file_path = step_input_filename.replace( os.path.basename(step_input_filename), "") det = fits.getval(step_input_filename, "DETECTOR", 0) exptype = fits.getval(step_input_filename, "EXP_TYPE", 0) grat = fits.getval(step_input_filename, "GRATING", 0) filt = fits.getval(step_input_filename, "FILTER", 0) file_basename = os.path.basename(step_input_filename.replace(".fits", "")) msg1 = 'step_input_filename=' + step_input_filename msg2 = "flat_field_file --> Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype print(msg1) print(msg2) log_msgs.append(msg1) log_msgs.append(msg2) # read in the on-the-fly flat image flatfile = step_input_filename.replace("flat_field.fits", "interpolatedflat.fits") pipeflat = fits.getdata(flatfile, "SCI") # get the reference files msg = "Getting and reading the D-, S-, and F-flats for this specific IFU configuration... " print(msg) log_msgs.append(msg) # D-Flat dflat_ending = "f_01.03.fits" dfile = dflatref_path + "_nrs1_" + dflat_ending if det == "NRS2": dfile = dfile.replace("nrs1", "nrs2") msg = "Using D-flat: " + dfile print(msg) log_msgs.append(msg) dfim = fits.getdata(dfile, "SCI") #1) dfimdq = fits.getdata(dfile, "DQ") #4) # need to flip/rotate the image into science orientation ns = np.shape(dfim) dfim = np.transpose(dfim, ( 0, 2, 1)) # keep in mind that 0,1,2 = z,y,x in Python, whereas =x,y,z in IDL dfimdq = np.transpose(dfimdq) if det == "NRS2": # rotate science data by 180 degrees for NRS2 dfim = dfim[..., ::-1, ::-1] dfimdq = dfimdq[..., ::-1, ::-1] naxis3 = fits.getval(dfile, "NAXIS3", "SCI") #1) # get the wavelength values dfwave = np.array([]) for i in range(naxis3): keyword = "PFLAT_" + str(i + 1) dfwave = np.append(dfwave, fits.getval(dfile, keyword, "SCI")) #1)) dfrqe = fits.getdata(dfile, 2) # S-flat tsp = exptype.split("_") mode = tsp[1] if filt == "F070LP": flat = "FLAT4" elif filt == "F100LP": flat = "FLAT1" elif filt == "F170LP": flat = "FLAT2" elif filt == "F290LP": flat = "FLAT3" elif filt == "CLEAR": flat = "FLAT5" else: msg = "No filter correspondence. Exiting the program." print(msg) log_msgs.append(msg) # This is the key argument for the assert pytest function msg = "Test skiped because there is no flat correspondence for the filter in the data: {}".format( filt) median_diff = "skip" return median_diff, msg sflat_ending = "f_01.01.fits" sfile = sfile_path + "_" + grat + "_OPAQUE_" + flat + "_nrs1_" + sflat_ending if debug: print("grat = ", grat) print("flat = ", flat) print("sfile used = ", sfile) if det == "NRS2": sfile = sfile.replace("nrs1", "nrs2") msg = "Using S-flat: " + sfile print(msg) log_msgs.append(msg) sfim = fits.getdata(sfile, "SCI") #1) sfimdq = fits.getdata(sfile, "DQ") #3) # need to flip/rotate image into science orientation sfim = np.transpose(sfim) sfimdq = np.transpose(sfimdq) if det == "NRS2": # rotate science data by 180 degrees for NRS2 sfim = sfim[..., ::-1, ::-1] sfimdq = sfimdq[..., ::-1, ::-1] sfv = fits.getdata(sfile, 5) # F-Flat fflat_ending = "_01.01.fits" if mode in fflat_path: ffile = fflat_path + "_" + filt + fflat_ending else: msg = "Wrong path in for mode F-flat. This script handles mode " + mode + "only." print(msg) log_msgs.append(msg) # This is the key argument for the assert pytest function result_msg = "Wrong path in for mode F-flat. Test skiped because mode is not IFU." median_diff = "skip" return median_diff, result_msg, log_msgs msg = "Using F-flat: " + ffile print(msg) log_msgs.append(msg) ffv = fits.getdata(ffile, "IFU") #1) # now go through each pixel in the test data if writefile: # create the fits list to hold the calculated flat values for each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create the fits list to hold the image of pipeline-calculated difference values hdu0 = fits.PrimaryHDU() complfile = fits.HDUList() complfile.append(hdu0) # get the datamodel from the assign_wcs output file assign_wcs_file = step_input_filename.replace("_flat_field.fits", "_assign_wcs.fits") model = datamodels.ImageModel(assign_wcs_file) ifu_slits = nirspec.nrs_ifu_wcs(model) # loop over the slices all_delfg_mean, all_delfg_mean_arr, all_delfg_median, all_test_result = [], [], [], [] msg = "\n Now looping through the slices, this may take some time... " print(msg) log_msgs.append(msg) for n_ext, slice in enumerate(ifu_slits): if n_ext < 10: pslice = "0" + repr(n_ext) else: pslice = repr(n_ext) msg = "\nWorking with slice: " + pslice print(msg) log_msgs.append(msg) # get the wavelength # slice.x(y)start are 1-based, turn them to 0-based for extraction x, y = wcstools.grid_from_bounding_box(slice.bounding_box, (1, 1), center=True) ra, dec, wave = slice(x, y) # get the subwindow origin (technically no subwindows for IFU, but need this for comparing to the # full frame on-the-fly flat image). px0 = model.meta.subarray.xstart - 1 + int( _toindex(slice.bounding_box[0][0])) + 1 py0 = model.meta.subarray.xstart - 1 + int( _toindex(slice.bounding_box[1][0])) + 1 n_p = np.shape(wave) nx, ny = n_p[1], n_p[0] nw = nx * ny msg = " Subwindow origin: px0=" + repr(px0) + " py0=" + repr(py0) print(msg) log_msgs.append(msg) if debug: print("n_p = ", n_p) print("nw = ", nw) # initialize arrays of the right size delf = np.zeros([nw]) + 999.0 flatcor = np.zeros([nw]) + 999.0 sffarr = np.zeros([nw]) calc_flat = np.zeros([2048, 2048]) + 999.0 # loop through the wavelengths msg = " Looping through the wavelngth, this may take a little time ... " print(msg) log_msgs.append(msg) flat_wave = wave.flatten() wave_shape = np.shape(wave) for j in range(0, nw): if np.isfinite(flat_wave[j]): # skip if wavelength is NaN # get the pixel indeces jwav = flat_wave[j] t = np.where(wave == jwav) pind = [t[0][0] + py0 - 1, t[1][0] + px0 - 1 ] # pind =[pixel_y, pixe_x] in python, [x, y] in IDL if debug: print('j, jwav, px0, py0 : ', j, jwav, px0, py0) print('pind[0], pind[1] = ', pind[0], pind[1]) # get the pixel bandwidth **this needs to be modified for prism, since the dispersion is not linear!** delw = 0.0 if (j != 0) and (int((j - 1) / nx) == int(j / nx)) and (int( (j + 1) / nx) == int(j / nx)) and np.isfinite( flat_wave[j + 1]) and np.isfinite(flat_wave[j - 1]): delw = 0.5 * (flat_wave[j + 1] - flat_wave[j - 1]) if (j == 0) or not np.isfinite(flat_wave[j - 1]) or (int( (j - 1) / nx) != int(j / nx)): delw = 0.5 * (flat_wave[j + 1] - flat_wave[j]) if (j == nw - 1) or not np.isfinite(flat_wave[j + 1]) or (int( (j + 1) / nx) != int(j / nx)): delw = 0.5 * (flat_wave[j] - flat_wave[j - 1]) if debug: #print("(j, (j-1), nx, (j-1)/nx, (j+1), (j+1)/nx)", j, (j-1), nx, int((j-1)/nx), (j+1), int((j+1)/nx)) #print("np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1])", np.isfinite(flat_wave[j+1]), np.isfinite(flat_wave[j-1])) #print("flat_wave[j+1], flat_wave[j-1] : ", np.isfinite(flat_wave[j+1]), flat_wave[j+1], flat_wave[j-1]) print("delw = ", delw) # integrate over D-flat fast vector dfrqe_wav = dfrqe.field("WAVELENGTH") dfrqe_rqe = dfrqe.field("RQE") iw = np.where((dfrqe_wav >= jwav - delw / 2.0) & (dfrqe_wav <= jwav + delw / 2.0)) if np.size(iw) == 0: iw = -1 int_tab = auxfunc.idl_tabulate(dfrqe_wav[iw], dfrqe_rqe[iw]) if int_tab == 0: int_tab = np.interp(dfrqe_wav[iw], dfrqe_wav, dfrqe_rqe) dff = int_tab else: first_dfrqe_wav, last_dfrqe_wav = dfrqe_wav[iw][ 0], dfrqe_wav[iw][-1] dff = int_tab / (last_dfrqe_wav - first_dfrqe_wav) if debug: #print("np.shape(dfrqe_wav) : ", np.shape(dfrqe_wav)) #print("np.shape(dfrqe_rqe) : ", np.shape(dfrqe_rqe)) #print("dfimdq[pind[0]][pind[1]] : ", dfimdq[pind[0]][pind[1]]) #print("np.shape(iw) =", np.shape(iw)) #print("np.shape(dfrqe_wav[iw[0]]) = ", np.shape(dfrqe_wav[iw[0]])) #print("np.shape(dfrqe_rqe[iw[0]]) = ", np.shape(dfrqe_rqe[iw[0]])) #print("int_tab=", int_tab) print("np.shape(iw) = ", np.shape(iw)) print("iw = ", iw) print("dff = ", dff) # interpolate over D-flat cube dfs = 1.0 if dfimdq[pind[0], pind[1]] == 0: dfs = np.interp(jwav, dfwave, dfim[:, pind[0], pind[1]]) # integrate over S-flat fast vector sfv_wav = sfv.field("WAVELENGTH") sfv_dat = sfv.field("DATA") if (jwav < 5.3) and (jwav > 0.6): iw = np.where((sfv_wav >= jwav - delw / 2.0) & (sfv_wav <= jwav + delw / 2.0)) if np.size(iw) == 0: iw = -1 if np.size(iw) > 1: int_tab = auxfunc.idl_tabulate(sfv_wav[iw], sfv_dat[iw]) first_sfv_wav, last_sfv_wav = sfv_wav[iw][0], sfv_wav[ iw][-1] sff = int_tab / (last_sfv_wav - first_sfv_wav) elif np.size(iw) == 1: sff = float(sfv_dat[iw]) else: sff = 999.0 # get s-flat pixel-dependent correction sfs = 1.0 if sfimdq[pind[0], pind[1]] == 0: sfs = sfim[pind[0], pind[1]] if debug: print("jwav-delw/2.0 = ", jwav - delw / 2.0) print("jwav+delw/2.0 = ", jwav + delw / 2.0) print("np.shape(sfv_wav), sfv_wav[-1] = ", np.shape(sfv_wav), sfv_wav[-1]) print("iw = ", iw) print("sfv_wav[iw] = ", sfv_wav[iw]) print("int_tab = ", int_tab) print("first_sfv_wav, last_sfv_wav = ", first_sfv_wav, last_sfv_wav) print("sfs = ", sfs) print("sff = ", sff) # integrate over f-flat fast vector # reference file blue cutoff is 1 micron, so need to force solution for shorter wavs ffv_wav = ffv.field("WAVELENGTH") ffv_dat = ffv.field("DATA") fff = 1.0 if jwav - delw / 2.0 >= 1.0: iw = np.where((ffv_wav >= jwav - delw / 2.0) & (ffv_wav <= jwav + delw / 2.0)) if np.size(iw) == 0: iw = -1 if np.size(iw) > 1: int_tab = auxfunc.idl_tabulate(ffv_wav[iw], ffv_dat[iw]) first_ffv_wav, last_ffv_wav = ffv_wav[iw][0], ffv_wav[ iw][-1] fff = int_tab / (last_ffv_wav - first_ffv_wav) elif np.size(iw) == 1: fff = float(ffv_dat[iw]) flatcor[j] = dff * dfs * sff * sfs * fff sffarr[j] = sff # To visually compare between the pipeline flat and the calculated one (e.g. in ds9), Phil Hodge # suggested using the following line: calc_flat[pind[0], pind[1]] = flatcor[j] # this line writes the calculated flat into a full frame array # then this new array needs to be written into a file. This part has not been done yet. # Difference between pipeline and calculated values delf[j] = pipeflat[pind[0], pind[1]] - flatcor[j] # Remove all pixels with values=1 (mainly inter-slit pixels) for statistics if pipeflat[pind[0], pind[1]] == 1: delf[j] = 999.0 if np.isnan(jwav): flatcor[j] = 1.0 # no correction if no wavelength if debug: print("np.shape(iw) = ", np.shape(iw)) print("fff = ", fff) print("flatcor[j] = ", flatcor[j]) print("delf[j] = ", delf[j]) # ignore outliers for calculating median delfg = delf[np.where(delf != 999.0)] #delfg_median, delfg_std = np.median(delfg), np.std(delfg) msg = "Flat value differences for slice number: " + pslice print(msg) log_msgs.append(msg) #print(" median = ", delfg_median, " stdev =", delfg_std) stats_and_strings = auxfunc.print_stats(delfg, "Flat Difference", float(threshold_diff), abs=True) stats, stats_print_strings = stats_and_strings delfg_mean, delfg_median, delfg_std = stats for msg in stats_print_strings: log_msgs.append(msg) if debug: print("np.shape(delf) = ", np.shape(delf)) print("np.shape(delfg) = ", np.shape(delfg)) all_delfg_mean.append(delfg_mean) all_delfg_median.append(delfg_median) # make the slice plot if np.isfinite(delfg_median) and (len(delfg) != 0): if show_figs or save_figs: msg = "Making the plot for this slice..." print(msg) log_msgs.append(msg) # create histogram t = (file_basename, det, pslice, "IFUflatcomp_histogram") title = filt + " " + grat + " SLICE=" + pslice + "\n" plot_name = "".join((file_path, ("_".join(t)) + ".pdf")) #mk_hist(title, delfg, delfg_mean, delfg_median, delfg_std, save_figs, show_figs, plot_name=plot_name) bins = None # binning for the histograms, if None the function will select them automatically title = title + "Residuals" info_img = [title, "x (pixels)", "y (pixels)"] xlabel, ylabel = "flat$_{pipe}$ - flat$_{calc}$", "N" info_hist = [xlabel, ylabel, bins, stats] if delfg[1] is np.nan: msg = "Unable to create plot of relative wavelength difference." print(msg) log_msgs.append(msg) else: plt_name = os.path.join(file_path, plot_name) difference_img = (pipeflat - calc_flat) #/calc_flat in_slit = np.logical_and( difference_img < 900.0, difference_img > -900.0) # ignore points out of the slit, difference_img[ ~in_slit] = np.nan # Set values outside the slit to NaN nanind = np.isnan( difference_img) # get all the nan indexes difference_img[ nanind] = np.nan # set all nan indexes to have a value of nan plt_origin = None limits = [px0 - 5, px0 + 1500, py0 - 5, py0 + 55] vminmax = [ -5 * delfg_std, 5 * delfg_std ] # set the range of values to be shown in the image, will affect color scale auxfunc.plt_two_2Dimgandhist(difference_img, delfg, info_img, info_hist, plt_name=plt_name, limits=limits, vminmax=vminmax, plt_origin=plt_origin, show_figs=show_figs, save_figs=save_figs) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." print(msg) log_msgs.append(msg) if writefile: # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(flatcor.reshape(wave_shape), name=pslice) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values complfile_ext = fits.ImageHDU(delf.reshape(wave_shape), name=pslice) complfile.append(complfile_ext) # the file is not yet written, indicate that this slit was appended to list to be written msg = "Extension " + repr( n_ext ) + " appended to list to be written into calculated and comparison fits files." print(msg) log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(delfg_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) all_test_result.append(test_result) # if the test is failed exit the script if (delfg_median == 999.0) or not np.isfinite(delfg_median): msg = "Unable to determine mean, meadian, and std_dev for the slice" + pslice print(msg) log_msgs.append(msg) if mk_all_slices_plt: if show_figs or save_figs: # create histogram t = (file_basename, det, "all_slices_IFU_flatcomp_histogram") title = ("_".join(t)) # calculate median of medians and std_dev of medians all_delfg_median_arr = np.array(all_delfg_median) mean_of_delfg_mean = np.mean(all_delfg_mean_arr) median_of_delfg_median = np.median(all_delfg_median_arr) medians_std = np.std(median_of_delfg_median) plot_name = "".join((file_path, title, ".pdf")) mk_hist(title, all_delfg_median_arr, mean_of_delfg_mean, median_of_delfg_median, medians_std, save_figs, show_figs, plot_name=plot_name) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." print(msg) log_msgs.append(msg) # create fits file to hold the calculated flat for each slice if writefile: outfile_name = step_input_filename.replace("flat_field.fits", det + "_flat_calc.fits") complfile_name = step_input_filename.replace("flat_field.fits", det + "_flat_comp.fits") # create the fits list to hold the calculated flat values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values complfile.writeto(complfile_name, overwrite=True) msg = "Fits file with calculated flat values of each slice saved as: " print(msg) print(outfile_name) log_msgs.append(msg) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline flat - calculated flat) saved as: " print(msg) print(complfile_name) log_msgs.append(msg) log_msgs.append(complfile_name) # If all tests passed then pytest will be marked as PASSED, else it will be FAILED FINAL_TEST_RESULT = True for t in all_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break if FINAL_TEST_RESULT: msg = "\n *** Final result for flat_field test will be reported as PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slices PASSED flat_field test." else: msg = "\n *** Final result for flat_field test will be reported as FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slices FAILED flat_field test." # end the timer flattest_end_time = time.time() - flattest_start_time if flattest_end_time > 60.0: flattest_end_time = flattest_end_time / 60.0 # in minutes flattest_tot_time = "* Script flattest_ifu.py script took ", repr( flattest_end_time) + " minutes to finish." if flattest_end_time > 60.0: flattest_end_time = flattest_end_time / 60. # in hours flattest_tot_time = "* Script flattest_ifu.py took ", repr( flattest_end_time) + " hours to finish." else: flattest_tot_time = "* Script flattest_ifu.py took ", repr( flattest_end_time) + " seconds to finish." print(flattest_tot_time) log_msgs.append(flattest_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs
def ifu_coords(fname, output=None): """ Computes wavelengths, and space coordinates of a NIRSPEC IFU exposure after running assign_wcs. Parameters ---------- fname : str The name of a file after assign_wcs was run. output : str The name of the output file. If None the root of the input file is used with an extension world_coordinates. Examples -------- >>> compute_world_coordinates('nrs1_fixed_assign_wcs_extract_2d.fits') """ model = datamodels.ImageModel(fname) if model.meta.exposure.type.lower() != 'nrs_ifu': raise ValueError("Expected an IFU observation," "(EXP_TYPE=NRS_IFU), got {0}".format( model.meta.exposure.type)) ifu_slits = nirspec.nrs_ifu_wcs(model) hdulist = fits.HDUList() phdu = fits.PrimaryHDU() phdu.header['filename'] = model.meta.filename phdu.header['data'] = 'world coordinates' hdulist.append(phdu) output_frame = ifu_slits[0].available_frames[-1] for i, slit in enumerate(ifu_slits): x, y = wcstools.grid_from_bounding_box(slit.bounding_box, (1, 1), center=True) ra, dec, lam = slit(x, y) detector2slit = slit.get_transform('detector', 'slit_frame') sx, sy, ls = detector2slit(x, y) world_coordinates = np.array([lam, ra, dec, sy]) imhdu = fits.ImageHDU(data=world_coordinates) imhdu.header['PLANE1'] = 'lambda, microns' imhdu.header['PLANE2'] = '{0}_x, arcsec'.format(output_frame) imhdu.header['PLANE3'] = '{0}_y, arcsec'.format(output_frame) imhdu.header['PLANE4'] = 'slit_y, relative to center (0, 0)' imhdu.header['SLIT'] = "SLIT_{0}".format(i) # -1 and +1 are to express this in 1-based coordinates imhdu.header['CRVAL1'] = model.meta.subarray.xstart - 1 + int( _toindex(slit.bounding_box[0][0])) + 1 imhdu.header['CRVAL2'] = model.meta.subarray.xstart - 1 + int( _toindex(slit.bounding_box[1][0])) + 1 # Input coordinates will be 1-based. imhdu.header['CRPIX1'] = 1 imhdu.header['CRPIX2'] = 1 imhdu.header['CTYPE1'] = 'pixel' imhdu.header['CTYPE2'] = 'pixel' hdulist.append(imhdu) if output is not None: base, ext = os.path.splitext(output) if ext != "fits": ext = "fits" if not base.endswith('world_coordinates'): "".join([base, '_world_coordinates']) "".join([base, ext]) else: root = model.meta.filename.split('_') output = "".join([root[0], '_world_coordinates', '.fits']) hdulist.writeto(output, overwrite=True) del hdulist model.close()
def pathtest(step_input_filename, reffile, comparison_filename, writefile=True, show_figs=False, save_figs=True, threshold_diff=1.0e-7, debug=False): """ This function calculates the difference between the pipeline and calculated pathloss values. Args: step_input_filename: str, full path name of sourcetype output fits file reffile: str, path to the pathloss IFU reference fits file comparison_filename: str, path to comparison pipeline pathloss file writefile: boolean, if True writes calculated flat and difference image fits files show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots threshold_diff: float, threshold diff between pipeline and ESA file debug: boolean, if true print statements will show on-screen Returns: - 1 plot, if told to save and/or show them. - median_diff: Boolean, True if smaller or equal to threshold. - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer pathtest_start_time = time.time() # get info from the input previous pipeline step file/datamodel print( "Checking if files exist and obtaining datamodels. This takes a few minutes..." ) if isinstance(step_input_filename, str): if os.path.isfile(step_input_filename): if debug: print('Input file does exist.') msg = 'step_input_filename=' + step_input_filename print(msg) log_msgs.append(msg) # get the input data model ifu_input_model = datamodels.open(step_input_filename) if debug: print('got input datamodel!') else: result_msg = 'Input file does NOT exist. Skipping pathloss test.' log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs else: ifu_input_model = step_input_filename # get comparison data if isinstance(comparison_filename, str): if os.path.isfile(comparison_filename): if debug: msg = 'Comparison file does exist.' print(msg) else: result_msg = "Comparison file does NOT exist. Skipping pathloss test." print(result_msg) log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the comparison data model ifu_pipe_model = datamodels.open(comparison_filename) if debug: print('Retrieved comparison datamodel.') else: ifu_pipe_model = comparison_filename # get info from data model det = ifu_input_model.meta.instrument.detector lamp = ifu_input_model.meta.instrument.lamp_state grat = ifu_input_model.meta.instrument.grating filt = ifu_input_model.meta.instrument.filter exptype = ifu_input_model.meta.exposure.type msg = "from datamodel --> Detector: " + det + " Grating: " + grat + " Filter: " + \ filt + " Lamp: " + lamp + " EXP_TYPE: " + exptype print(msg) log_msgs.append(msg) msg = "Using reference file: " + reffile print(msg) log_msgs.append(msg) is_point_source = True if writefile: # create the fits list to hold the calculated pathloss values for each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create fits list to hold pipeline-calculated difference values hdu0 = fits.PrimaryHDU() compfile = fits.HDUList() compfile.append(hdu0) # list to determine if pytest is passed or not total_test_result = [] # get all the science extensions if is_point_source: ext = 1 # for all PS IFU # get slices (instead of using .slit) pl_ifu_slits = nirspec.nrs_ifu_wcs(ifu_input_model) print("got input slices") plcor_ref_ext = fits.getdata(reffile, ext) hdul = fits.open(reffile) plcor_ref = hdul[1].data w = wcs.WCS(hdul[1].header) w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref.shape[1], :plcor_ref. shape[2]] slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0) # these are full 2048 * 2048 files: previous_sci = ifu_input_model.data comp_sci = ifu_pipe_model.data pathloss_divided = comp_sci / previous_sci # set up generals for all plots font = {'weight': 'normal', 'size': 12} matplotlib.rc('font', **font) # loop through the slices msg = " Looping through the slices... " print(msg) log_msgs.append(msg) slit_list = np.ndarray.tolist(np.arange(0, 30)) for slit, slit_num, pipeslit in zip(pl_ifu_slits, slit_list): print("working with slice {}".format(slit_num)) # NIRSpec implementation x, y = wcstools.grid_from_bounding_box(slit.bounding_box, step=(1, 1)) ra, dec, wave = slit(x, y) slit_x = 0 # This assumption is made for IFU sources only slit_y = 0 if debug: print("slit_x, slit_y", slit_x, slit_y) correction_array = np.array([]) lambda_array = np.array([]) wave_sci = wave * 10**(-6) # microns --> meters wave_sci_flat = wave_sci.reshape(wave_sci.size) wave_ref_flat = wave_ref.reshape(wave_ref.size) ref_xy = np.column_stack((slitx_ref.reshape(slitx_ref.size), slity_ref.reshape(slitx_ref.size))) # loop through slices in lambda from reference file shape = 0 for lambda_val in wave_ref_flat: # loop through every lambda value # flattened so that looping works smoothly shape = shape + 1 index = np.where(wave_ref[:, 0, 0] == lambda_val) # index of closest lambda value in reffile to given sci lambda # took index of only the first slice of wave_ref because # the others were repetitive & we got extra indices # take slice where lambda=index: plcor_slice = plcor_ref_ext[index[0][0]].reshape( plcor_ref_ext[index[0][0]].size) # do 2d interpolation to get a single correction factor for each slice corr_val = scipy.interpolate.griddata(ref_xy[:plcor_slice.size], plcor_slice, np.asarray([slit_x, slit_y]), method='linear') # append values from loop to create a vector of correction factors correction_array = np.append(correction_array, corr_val[0]) # map to array with corresponding lambda lambda_array = np.append(lambda_array, lambda_val) # get correction value for each pixel corr_vals = np.interp(wave_sci_flat, lambda_array, correction_array) corr_vals = corr_vals.reshape(wave_sci.shape) box = slit.bounding_box small_y = box[0][0] big_y = box[0][1] small_x = box[1][0] big_x = box[1][1] left = int(math.trunc(small_x)) right = int(math.ceil(big_x)) bottom = int(math.trunc(small_y)) top = int(math.ceil(big_y)) full_cut2slice = previous_sci[left:right, bottom:top] print("shapes:", full_cut2slice.shape, corr_vals.shape) if full_cut2slice.shape != corr_vals.shape: value = 0 while (((full_cut2slice.shape[0] - corr_vals.shape[0]) != 0) or ((full_cut2slice.shape[1] - corr_vals.shape[1]) != 0)) and \ value < 7: # can delete second criteria once all pass if value == 6: print("WARNING: may be in infinite loop!") x_amount_off = full_cut2slice.shape[0] - corr_vals.shape[0] if x_amount_off >= 1: if x_amount_off % 2 == 0: # need to add two values, so do one on each side right = right - 1 left = left + 1 print("ALTERED SHAPE OF SLICE: V1") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: # just add one value left = left + 1 print("ALTERED SHAPE OF SLICE: V2") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) elif x_amount_off <= -1: if x_amount_off % 2 == 0: right = right + 1 left = left - 1 print("ALTERED SHAPE OF SLICE: V3") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: left = left - 1 print("ALTERED SHAPE OF SLICE: V4") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) y_amount_off = full_cut2slice.shape[1] - corr_vals.shape[1] if y_amount_off >= 1: if y_amount_off % 2 == 0: bottom = bottom - 1 top = top + 1 print("ALTERED SHAPE OF SLICE: V5") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: bottom = bottom + 1 print("ALTERED SHAPE OF SLICE: V6") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) elif y_amount_off <= -1: if y_amount_off % 2 == 0: top = top + 1 bottom = bottom - 1 print("ALTERED SHAPE OF SLICE: V7") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: bottom = bottom - 1 print("ALTERED SHAPE OF SLICE: V8") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) full_cut2slice = previous_sci[left:right, bottom:top] print("final left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) print("NEW SHAPE OF SLICE: {} and corr_vals.shape: {}".format( full_cut2slice.shape, corr_vals.shape)) if full_cut2slice.shape != corr_vals.shape: print("shapes did not match! full_cut2slice: {}, corr_vals {}". format(full_cut2slice.shape, corr_vals.shape)) continue corrected_array = full_cut2slice / corr_vals pipe_correction = pathloss_divided[left:right, bottom:top] if pipe_correction.shape != corr_vals.shape: print("shapes did not match! pipe_correction: {}, corr_vals {}". format(pipe_correction.shape, corr_vals.shape)) continue prev_sci_slit = previous_sci[left:right, bottom:top] if prev_sci_slit.shape != corr_vals.shape: print( "shapes did not match! prev_sci_slit: {}, corr_vals {}".format( prev_sci_slit.shape, corr_vals.shape)) continue comp_sci_slit = comp_sci[left:right, bottom:top] if comp_sci_slit.shape != corr_vals.shape: print( "shapes did not match! comp_sci_slit: {}, corr_vals {}".format( comp_sci_slit.shape, corr_vals.shape)) continue # Plots: # my correction values fig = plt.figure(figsize=(15, 15)) plt.subplot(221) norm = ImageNormalize(corr_vals) plt.imshow(corr_vals, vmin=0.999995, vmax=1.000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Calculated Correction') plt.colorbar() # pipe correction plt.subplot(222) norm = ImageNormalize(pipe_correction) plt.imshow(pipe_correction, vmin=0.999995, vmax=1.000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Pipe Correction') plt.colorbar() # residuals (pipe correction - my correction) if pipe_correction.shape == corr_vals.shape: corr_residuals = pipe_correction - corr_vals plt.subplot(223) norm = ImageNormalize(corr_residuals) plt.imshow(corr_residuals, vmin=-0.000000005, vmax=0.000000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Correction residuals') plt.colorbar() # my science data after pathloss plt.subplot(224) norm = ImageNormalize(corrected_array) plt.imshow(corrected_array, vmin=0, vmax=300, aspect=10.0, origin='lower', cmap='viridis') plt.title('Corrected Data After Pathloss') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.colorbar() fig.suptitle("IFU PS Pathloss Correction Testing" + str(corrected_array.shape)) fig.tight_layout(pad=3.0) if show_figs: plt.show() if save_figs: step_input_filepath = step_input_filename.replace(".fits", "") plt_name = step_input_filepath + "_Pathloss_test_slitlet_IFU_PS_" + str( slit_num) + ".png" plt.savefig(plt_name) print('Figure saved as: ', plt_name) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." if debug: print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." if debug: print(msg) log_msgs.append(msg) plt.clf() # create fits file to hold the calculated pathloss for each slit if writefile: msg = "Saving the fits files with the calculated pathloss for each slit..." print(msg) log_msgs.append(msg) # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(corr_vals, name=str(slit_num)) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values compfile_ext = fits.ImageHDU(corr_residuals, name=str(slit_num)) compfile.append(compfile_ext) if corr_residuals[~np.isnan(corr_residuals)].size == 0: msg1 = " * Unable to calculate statistics because difference array has all values as NaN. " \ "Test will be set to FAILED." print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: msg = "Calculating statistics... " print(msg) log_msgs.append(msg) corr_residuals = corr_residuals[ np.where((corr_residuals != 999.0) & (corr_residuals < 0.1) & (corr_residuals > -0.1))] # ignore outliers if corr_residuals.size == 0: msg1 = """ * Unable to calculate statistics because difference array has all outlier values. Test will be set to FAILED.""" print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: stats_and_strings = auxfunc.print_stats(corr_residuals, "Difference", float(threshold_diff), absolute=True) stats, stats_print_strings = stats_and_strings corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats for msg in stats_print_strings: log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(corr_residuals_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) total_test_result.append(test_result) hdul.close() ifu_pipe_model.close() ifu_input_model.close() if writefile: outfile_name = step_input_filename.replace("srctype", "_calcuated_pathloss") compfile_name = step_input_filename.replace("srctype", "_comparison_pathloss") # create the fits list to hold the calculated pathloss values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values compfile.writeto(compfile_name, overwrite=True) msg = "\nFits file with calculated pathloss values of each slit saved as: " print(msg) log_msgs.append(msg) print(outfile_name) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: " print(msg) log_msgs.append(msg) print(compfile_name) log_msgs.append(compfile_name) # If all tests passed then pytest will be marked as PASSED, else FAILED FINAL_TEST_RESULT = False for t in total_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break else: FINAL_TEST_RESULT = True if FINAL_TEST_RESULT: msg = "\n *** Final pathloss test result reported as PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slits PASSED path_loss test." else: msg = "\n *** Final pathloss test result reported as FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slits FAILED path_loss test." # end the timer pathloss_end_time = time.time() - pathtest_start_time if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60.0 # in minutes pathloss_tot_time = "* Script IFU_PS.py took ", repr( pathloss_end_time) + " minutes to finish." if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60. # in hours pathloss_tot_time = "* Script IFU_PS.py took ", repr( pathloss_end_time) + " hours to finish." else: pathloss_tot_time = "* Script IFU_PS.py took ", repr( pathloss_end_time) + " seconds to finish." print(pathloss_tot_time) log_msgs.append(pathloss_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs