def display_slitlet_histogram(csu_bar_slit_width, n_clusters=2, geometry=None, debugplot=0): """ Find separations between groups of slitlet widths. Parameters ---------- csu_bar_slit_width : numpy array Array containing the csu_bar_slit_center values. n_clusters : int Number of slitlet groups to be sought. geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- TBD """ # protections if n_clusters < 2: raise ValueError("n_clusters must be >= 2") # determine useful slitlets from their widths km = KMeans(n_clusters=n_clusters) fit = km.fit(np.array(csu_bar_slit_width).reshape(-1, 1)) separator_list = [] for i in range(n_clusters - 1): peak1 = fit.cluster_centers_[i][0] peak2 = fit.cluster_centers_[i + 1][0] separator = (peak1 + peak2) / 2 separator_list.append(separator) print('---> separator: {0:7.3f}'.format(separator)) # display histogram if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) ax.hist(csu_bar_slit_width, bins=100) ax.set_xlabel('slit width (mm)') ax.set_ylabel('number of slitlets') for separator in separator_list: ax.axvline(separator, color='C1', linestyle='dashed') pause_debugplot(debugplot, pltshow=True)
def display_slitlet_histogram(csu_bar_slit_width, n_clusters=2, geometry=None, debugplot=0): """ Find separations between groups of slitlet widths. Parameters ---------- csu_bar_slit_width : numpy array Array containing the csu_bar_slit_center values. n_clusters : int Number of slitlet groups to be sought. geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- TBD """ # protections if n_clusters < 2: raise ValueError("n_clusters must be >= 2") # determine useful slitlets from their widths km = KMeans(n_clusters=n_clusters) fit = km.fit(np.array(csu_bar_slit_width).reshape(-1, 1)) separator_list = [] for i in range(n_clusters - 1): peak1 = fit.cluster_centers_[i][0] peak2 = fit.cluster_centers_[i+1][0] separator = (peak1 + peak2) / 2 separator_list.append(separator) print('---> separator: {0:7.3f}'.format(separator)) # display histogram if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) ax.hist(csu_bar_slit_width, bins=100) ax.set_xlabel('slit width (mm)') ax.set_ylabel('number of slitlets') for separator in separator_list: ax.axvline(separator, color='C1', linestyle='dashed') pause_debugplot(debugplot, pltshow=True)
def refine_rectwv_coeff(input_image, rectwv_coeff, refine_wavecalib_mode, minimum_slitlet_width_mm, maximum_slitlet_width_mm, save_intermediate_results=False, debugplot=0): """Refine RectWaveCoeff object using a catalogue of lines One and only one among refine_with_oh_lines_mode and refine_with_arc_lines must be different from zero. Parameters ---------- input_image : HDUList object Input 2D image. rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for the particular CSU configuration. refine_wavecalib_mode : int Integer, indicating the type of refinement: 0 : no refinement 1 : apply the same global offset to all the slitlets (using ARC lines) 2 : apply individual offset to each slitlet (using ARC lines) 11 : apply the same global offset to all the slitlets (using OH lines) 12 : apply individual offset to each slitlet (using OH lines) minimum_slitlet_width_mm : float Minimum slitlet width (mm) for a valid slitlet. maximum_slitlet_width_mm : float Maximum slitlet width (mm) for a valid slitlet. save_intermediate_results : bool If True, save plots in PDF files debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot. Returns ------- refined_rectwv_coeff : RectWaveCoeff instance Refined rectification and wavelength calibration coefficients for the particular CSU configuration. expected_cat_image : HDUList object Output 2D image with the expected catalogued lines. """ logger = logging.getLogger(__name__) if save_intermediate_results: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages('crosscorrelation.pdf') else: pdf = None # image header main_header = input_image[0].header filter_name = main_header['filter'] grism_name = main_header['grism'] # protections if refine_wavecalib_mode not in [1, 2, 11, 12]: logger.error('Wavelength calibration refinemente mode={}'.format( refine_wavecalib_mode)) raise ValueError("Invalid wavelength calibration refinement mode") # read tabulated lines if refine_wavecalib_mode in [1, 2]: # ARC lines if grism_name == 'LR': catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat' else: catlines_file = 'lines_argon_neon_xenon_empirical.dat' dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file) arc_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(arc_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = catlines[:, 0] catlines_all_flux = catlines[:, 1] mode = refine_wavecalib_mode elif refine_wavecalib_mode in [11, 12]: # OH lines dumdata = pkgutil.get_data('emirdrp.instrument.configs', 'Oliva_etal_2013.dat') oh_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(oh_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0])) catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2])) mode = refine_wavecalib_mode - 10 else: raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode)) # initialize output refined_rectwv_coeff = deepcopy(rectwv_coeff) logger.info('Computing median spectrum') # compute median spectrum and normalize it sp_median = median_slitlets_rectified( input_image, mode=2, minimum_slitlet_width_mm=minimum_slitlet_width_mm, maximum_slitlet_width_mm=maximum_slitlet_width_mm)[0].data sp_median /= sp_median.max() # determine minimum and maximum useful wavelength jmin, jmax = find_pix_borders(sp_median, 0) naxis1 = main_header['naxis1'] naxis2 = main_header['naxis2'] crpix1 = main_header['crpix1'] crval1 = main_header['crval1'] cdelt1 = main_header['cdelt1'] xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1 if grism_name == 'LR': wv_parameters = set_wv_parameters(filter_name, grism_name) wave_min = wv_parameters['wvmin_useful'] wave_max = wv_parameters['wvmax_useful'] else: wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1 wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1 logger.info('Setting wave_min to {}'.format(wave_min)) logger.info('Setting wave_max to {}'.format(wave_max)) # extract subset of catalogue lines within current wavelength range lok1 = catlines_all_wave >= wave_min lok2 = catlines_all_wave <= wave_max catlines_reference_wave = catlines_all_wave[lok1 * lok2] catlines_reference_flux = catlines_all_flux[lok1 * lok2] catlines_reference_flux /= catlines_reference_flux.max() # estimate sigma to broaden catalogue lines csu_config = CsuConfiguration.define_from_header(main_header) # segregate slitlets list_useful_slitlets = csu_config.widths_in_range_mm( minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm) list_not_useful_slitlets = [ i for i in list(range(1, EMIR_NBARS + 1)) if i not in list_useful_slitlets ] logger.info('list of useful slitlets: {}'.format(list_useful_slitlets)) logger.info( 'list of not useful slitlets: {}'.format(list_not_useful_slitlets)) tempwidths = np.array([ csu_config.csu_bar_slit_width(islitlet) for islitlet in list_useful_slitlets ]) widths_summary = summary(tempwidths) logger.info('Statistics of useful slitlet widths (mm):') logger.info('- npoints....: {0:d}'.format(widths_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(widths_summary['median'])) logger.info('- std........: {0:7.3f}'.format(widths_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std'])) # empirical transformation of slit width (mm) to pixels sigma_broadening = cdelt1 * widths_summary['median'] # convolve location of catalogue lines to generate expected spectrum xwave_reference, sp_reference = convolve_comb_lines( catlines_reference_wave, catlines_reference_flux, sigma_broadening, crpix1, crval1, cdelt1, naxis1) sp_reference /= sp_reference.max() # generate image2d with expected lines image2d_expected_lines = np.tile(sp_reference, (naxis2, 1)) hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header) expected_cat_image = fits.HDUList([hdu]) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xwave, sp_median, 'C1-', xlabel='Wavelength (Angstroms, in vacuum)', ylabel='Normalized number of counts', title='Median spectrum', label='observed spectrum', show=False) # overplot reference catalogue lines ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-', markerfmt=' ', basefmt='C4-', label='tabulated lines') # overplot convolved reference lines ax.plot(xwave_reference, sp_reference, 'C0-', label='expected spectrum') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) # compute baseline signal in sp_median baseline = np.percentile(sp_median[sp_median > 0], q=10) if (abs(debugplot) % 10 != 0) or (pdf is not None): fig = plt.figure() ax = fig.add_subplot(111) ax.hist(sp_median, bins=1000, log=True) ax.set_xlabel('Normalized number of counts') ax.set_ylabel('Number of pixels') ax.set_title('Median spectrum') ax.axvline(float(baseline), linestyle='--', color='grey') if pdf is not None: pdf.savefig() else: geometry = (0, 0, 640, 480) set_window_geometry(geometry) plt.show() # subtract baseline to sp_median (only pixels with signal above zero) lok = np.where(sp_median > 0) sp_median[lok] -= baseline # compute global offset through periodic correlation logger.info('Computing global offset') global_offset, fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=sp_median, fminmax=None, naround_zero=50, plottitle='Median spectrum (cross-correlation)', pdf=pdf, debugplot=debugplot) logger.info('Global offset: {} pixels'.format(-global_offset)) missing_slitlets = rectwv_coeff.missing_slitlets if mode == 1: # apply computed offset to obtain refined_rectwv_coeff_global for islitlet in range(1, EMIR_NBARS + 1): if islitlet not in missing_slitlets: i = islitlet - 1 dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= global_offset * cdelt1 elif mode == 2: # compute individual offset for each slitlet logger.info('Computing individual offsets') median_55sp = median_slitlets_rectified(input_image, mode=1) offset_array = np.zeros(EMIR_NBARS) xplot = [] yplot = [] xplot_skipped = [] yplot_skipped = [] cout = '0' for islitlet in range(1, EMIR_NBARS + 1): if islitlet in list_useful_slitlets: i = islitlet - 1 sp_median = median_55sp[0].data[i, :] lok = np.where(sp_median > 0) baseline = np.percentile(sp_median[lok], q=10) sp_median[lok] -= baseline sp_median /= sp_median.max() offset_array[i], fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=median_55sp[0].data[i, :], fminmax=None, naround_zero=50, plottitle='slitlet #{0} (cross-correlation)'.format( islitlet), pdf=pdf, debugplot=debugplot) dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= offset_array[i] * cdelt1 xplot.append(islitlet) yplot.append(-offset_array[i]) # second correction wpoly_coeff_refined = check_wlcalib_sp( sp=median_55sp[0].data[i, :], crpix1=crpix1, crval1=crval1 - offset_array[i] * cdelt1, cdelt1=cdelt1, wv_master=catlines_reference_wave, coeff_ini=dumdict['wpoly_coeff'], naxis1_ini=EMIR_NAXIS1, title='slitlet #{0} (after applying offset)'.format( islitlet), ylogscale=False, pdf=pdf, debugplot=debugplot) dumdict['wpoly_coeff'] = wpoly_coeff_refined cout += '.' else: xplot_skipped.append(islitlet) yplot_skipped.append(0) cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) logger.info(cout) # show offsets with opposite sign stat_summary = summary(np.array(yplot)) logger.info('Statistics of individual slitlet offsets (pixels):') logger.info('- npoints....: {0:d}'.format(stat_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(stat_summary['median'])) logger.info('- std........: {0:7.3f}'.format(stat_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format( stat_summary['robust_std'])) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xplot, yplot, linestyle='', marker='o', color='C0', xlabel='slitlet number', ylabel='-offset (pixels) = offset to be applied', title='cross-correlation result', show=False, **{'label': 'individual slitlets'}) if len(xplot_skipped) > 0: ax.plot(xplot_skipped, yplot_skipped, 'mx') ax.axhline(-global_offset, linestyle='--', color='C1', label='global offset') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) else: raise ValueError('Unexpected mode={}'.format(mode)) # close output PDF file if pdf is not None: pdf.close() # return result return refined_rectwv_coeff, expected_cat_image
def display_slitlet_arrangement(fileobj, grism=None, spfilter=None, bbox=None, adjust=None, geometry=None, debugplot=0): """Display slitlet arrangment from CSUP keywords in FITS header. Parameters ---------- fileobj : file object FITS or TXT file object. grism : str Grism. grism : str Filter. bbox : tuple of 4 floats If not None, values for xmin, xmax, ymin and ymax. adjust : bool Adjust X range according to minimum and maximum csu_bar_left and csu_bar_right (note that this option is overriden by 'bbox') geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- csu_bar_left : list of floats Location (mm) of the left bar for each slitlet. csu_bar_right : list of floats Location (mm) of the right bar for each slitlet, using the same origin employed for csu_bar_left (which is not the value stored in the FITS keywords. csu_bar_slit_center : list of floats Middle point (mm) in between the two bars defining a slitlet. csu_bar_slit_width : list of floats Slitlet width (mm), computed as the distance between the two bars defining the slitlet. """ if fileobj.name[-4:] == ".txt": if grism is None: raise ValueError("Undefined grism!") if spfilter is None: raise ValueError("Undefined filter!") # define CsuConfiguration object csu_config = CsuConfiguration() csu_config._csu_bar_left = [] csu_config._csu_bar_right = [] csu_config._csu_bar_slit_center = [] csu_config._csu_bar_slit_width = [] # since the input filename has been opened with argparse in binary # mode, it is necessary to close it and open it in text mode fileobj.close() # read TXT file with open(fileobj.name, mode='rt') as f: file_content = f.read().splitlines() next_id_bar = 1 for line in file_content: if len(line) > 0: if line[0] not in ['#']: line_contents = line.split() id_bar = int(line_contents[0]) position = float(line_contents[1]) if id_bar == next_id_bar: if id_bar <= EMIR_NBARS: csu_config._csu_bar_left.append(position) next_id_bar = id_bar + EMIR_NBARS else: csu_config._csu_bar_right.append(341.5 - position) next_id_bar = id_bar - EMIR_NBARS + 1 else: raise ValueError("Unexpected id_bar:" + str(id_bar)) # compute slit width and center for i in range(EMIR_NBARS): csu_config._csu_bar_slit_center.append( (csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i]) / 2) csu_config._csu_bar_slit_width.append( csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i]) else: # read input FITS file hdulist = fits.open(fileobj.name) image_header = hdulist[0].header hdulist.close() # additional info from header grism = image_header['grism'] spfilter = image_header['filter'] # define slitlet arrangement csu_config = CsuConfiguration.define_from_fits(fileobj) # determine calibration if grism in ["J", "OPEN"] and spfilter == "J": wv_parameters = set_wv_parameters("J", "J") elif grism in ["H", "OPEN"] and spfilter == "H": wv_parameters = set_wv_parameters("H", "H") elif grism in ["K", "OPEN"] and spfilter == "Ksp": wv_parameters = set_wv_parameters("Ksp", "K") elif grism in ["LR", "OPEN"] and spfilter == "YJ": wv_parameters = set_wv_parameters("YJ", "LR") elif grism in ["LR", "OPEN"] and spfilter == "HK": wv_parameters = set_wv_parameters("HK", "LR") else: raise ValueError("Invalid grism + filter configuration") crval1 = wv_parameters['poly_crval1_linear'] cdelt1 = wv_parameters['poly_cdelt1_linear'] wvmin_useful = wv_parameters['wvmin_useful'] wvmax_useful = wv_parameters['wvmax_useful'] # display arrangement if abs(debugplot) >= 10: print("slit left right center width min.wave max.wave") print("==== ======= ======= ======= ===== ======== ========") for i in range(EMIR_NBARS): ibar = i + 1 csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar)) csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar)) csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1 if wvmin_useful is not None: csu_crval1 = np.amax([csu_crval1, wvmin_useful]) if wvmax_useful is not None: csu_crvaln = np.amin([csu_crvaln, wvmax_useful]) print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} " "{5:8.2f} {6:8.2f}".format( ibar, csu_config.csu_bar_left(ibar), csu_config.csu_bar_right(ibar), csu_config.csu_bar_slit_center(ibar), csu_config.csu_bar_slit_width(ibar), csu_crval1, csu_crvaln)) print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format( np.mean(csu_config._csu_bar_left), np.mean(csu_config._csu_bar_right), np.mean(csu_config._csu_bar_slit_center), np.mean(csu_config._csu_bar_slit_width))) print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format( np.mean(csu_config._csu_bar_left[::2]), np.mean(csu_config._csu_bar_right[::2]), np.mean(csu_config._csu_bar_slit_center[::2]), np.mean(csu_config._csu_bar_slit_width[::2]))) print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format( np.mean(csu_config._csu_bar_left[1::2]), np.mean(csu_config._csu_bar_right[1::2]), np.mean(csu_config._csu_bar_slit_center[1::2]), np.mean(csu_config._csu_bar_slit_width[1::2]))) # display slit arrangement if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) if bbox is None: if adjust: xmin = min(csu_config._csu_bar_left) xmax = max(csu_config._csu_bar_right) dx = xmax - xmin if dx == 0: dx = 1 xmin -= dx / 20 xmax += dx / 20 ax.set_xlim(xmin, xmax) else: ax.set_xlim(0., 341.5) ax.set_ylim(0, 56) else: ax.set_xlim(bbox[0], bbox[1]) ax.set_ylim(bbox[2], bbox[3]) ax.set_xlabel('csu_bar_position (mm)') ax.set_ylabel('slit number') for i in range(EMIR_NBARS): ibar = i + 1 ax.add_patch( patches.Rectangle((csu_config.csu_bar_left(ibar), ibar - 0.5), csu_config.csu_bar_slit_width(ibar), 1.0)) ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar], '-', color='gray') ax.plot([csu_config.csu_bar_right(ibar), 341.5], [ibar, ibar], '-', color='gray') plt.title("File: " + fileobj.name + "\ngrism=" + grism + ", filter=" + spfilter) pause_debugplot(debugplot, pltshow=True) # return results return csu_config._csu_bar_left, csu_config._csu_bar_right, \ csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def display_slitlet_arrangement(fileobj, grism=None, spfilter=None, bbox=None, adjust=None, geometry=None, debugplot=0): """Display slitlet arrangment from CSUP keywords in FITS header. Parameters ---------- fileobj : file object FITS or TXT file object. grism : str Grism. grism : str Filter. bbox : tuple of 4 floats If not None, values for xmin, xmax, ymin and ymax. adjust : bool Adjust X range according to minimum and maximum csu_bar_left and csu_bar_right (note that this option is overriden by 'bbox') geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- csu_bar_left : list of floats Location (mm) of the left bar for each slitlet. csu_bar_right : list of floats Location (mm) of the right bar for each slitlet, using the same origin employed for csu_bar_left (which is not the value stored in the FITS keywords. csu_bar_slit_center : list of floats Middle point (mm) in between the two bars defining a slitlet. csu_bar_slit_width : list of floats Slitlet width (mm), computed as the distance between the two bars defining the slitlet. """ if fileobj.name[-4:] == ".txt": if grism is None: raise ValueError("Undefined grism!") if spfilter is None: raise ValueError("Undefined filter!") # define CsuConfiguration object csu_config = CsuConfiguration() csu_config._csu_bar_left = [] csu_config._csu_bar_right = [] csu_config._csu_bar_slit_center = [] csu_config._csu_bar_slit_width = [] # since the input filename has been opened with argparse in binary # mode, it is necessary to close it and open it in text mode fileobj.close() # read TXT file with open(fileobj.name, mode='rt') as f: file_content = f.read().splitlines() next_id_bar = 1 for line in file_content: if len(line) > 0: if line[0] not in ['#']: line_contents = line.split() id_bar = int(line_contents[0]) position = float(line_contents[1]) if id_bar == next_id_bar: if id_bar <= EMIR_NBARS: csu_config._csu_bar_left.append(position) next_id_bar = id_bar + EMIR_NBARS else: csu_config._csu_bar_right.append(341.5 - position) next_id_bar = id_bar - EMIR_NBARS + 1 else: raise ValueError("Unexpected id_bar:" + str(id_bar)) # compute slit width and center for i in range(EMIR_NBARS): csu_config._csu_bar_slit_center.append( (csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i])/2 ) csu_config._csu_bar_slit_width.append( csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i] ) else: # read input FITS file hdulist = fits.open(fileobj.name) image_header = hdulist[0].header hdulist.close() # additional info from header grism = image_header['grism'] spfilter = image_header['filter'] # define slitlet arrangement csu_config = CsuConfiguration.define_from_fits(fileobj) # determine calibration if grism in ["J", "OPEN"] and spfilter == "J": wv_parameters = set_wv_parameters("J", "J") elif grism in ["H", "OPEN"] and spfilter == "H": wv_parameters = set_wv_parameters("H", "H") elif grism in ["K", "OPEN"] and spfilter == "Ksp": wv_parameters = set_wv_parameters("Ksp", "K") elif grism in ["LR", "OPEN"] and spfilter == "YJ": wv_parameters = set_wv_parameters("YJ", "LR") elif grism in ["LR", "OPEN"] and spfilter == "HK": wv_parameters = set_wv_parameters("HK", "LR") else: raise ValueError("Invalid grism + filter configuration") crval1 = wv_parameters['poly_crval1_linear'] cdelt1 = wv_parameters['poly_cdelt1_linear'] wvmin_useful = wv_parameters['wvmin_useful'] wvmax_useful = wv_parameters['wvmax_useful'] # display arrangement if abs(debugplot) >= 10: print("slit left right center width min.wave max.wave") print("==== ======= ======= ======= ===== ======== ========") for i in range(EMIR_NBARS): ibar = i + 1 csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar)) csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar)) csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1 if wvmin_useful is not None: csu_crval1 = np.amax([csu_crval1, wvmin_useful]) if wvmax_useful is not None: csu_crvaln = np.amin([csu_crvaln, wvmax_useful]) print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} " "{5:8.2f} {6:8.2f}".format( ibar, csu_config.csu_bar_left(ibar), csu_config.csu_bar_right(ibar), csu_config.csu_bar_slit_center(ibar), csu_config.csu_bar_slit_width(ibar), csu_crval1, csu_crvaln) ) print( "---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format( np.mean(csu_config._csu_bar_left), np.mean(csu_config._csu_bar_right), np.mean(csu_config._csu_bar_slit_center), np.mean(csu_config._csu_bar_slit_width) ) ) print( "---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format( np.mean(csu_config._csu_bar_left[::2]), np.mean(csu_config._csu_bar_right[::2]), np.mean(csu_config._csu_bar_slit_center[::2]), np.mean(csu_config._csu_bar_slit_width[::2]) ) ) print( "---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format( np.mean(csu_config._csu_bar_left[1::2]), np.mean(csu_config._csu_bar_right[1::2]), np.mean(csu_config._csu_bar_slit_center[1::2]), np.mean(csu_config._csu_bar_slit_width[1::2]) ) ) # display slit arrangement if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) if bbox is None: if adjust: xmin = min(csu_config._csu_bar_left) xmax = max(csu_config._csu_bar_right) dx = xmax - xmin if dx == 0: dx = 1 xmin -= dx/20 xmax += dx/20 ax.set_xlim(xmin, xmax) else: ax.set_xlim(0., 341.5) ax.set_ylim(0, 56) else: ax.set_xlim(bbox[0], bbox[1]) ax.set_ylim(bbox[2], bbox[3]) ax.set_xlabel('csu_bar_position (mm)') ax.set_ylabel('slit number') for i in range(EMIR_NBARS): ibar = i + 1 ax.add_patch(patches.Rectangle( (csu_config.csu_bar_left(ibar), ibar-0.5), csu_config.csu_bar_slit_width(ibar), 1.0)) ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar], '-', color='gray') ax.plot([csu_config.csu_bar_right(ibar), 341.5], [ibar, ibar], '-', color='gray') plt.title("File: " + fileobj.name + "\ngrism=" + grism + ", filter=" + spfilter) pause_debugplot(debugplot, pltshow=True) # return results return csu_config._csu_bar_left, csu_config._csu_bar_right, \ csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def refine_rectwv_coeff(input_image, rectwv_coeff, refine_wavecalib_mode, minimum_slitlet_width_mm, maximum_slitlet_width_mm, save_intermediate_results=False, debugplot=0): """Refine RectWaveCoeff object using a catalogue of lines One and only one among refine_with_oh_lines_mode and refine_with_arc_lines must be different from zero. Parameters ---------- input_image : HDUList object Input 2D image. rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for the particular CSU configuration. refine_wavecalib_mode : int Integer, indicating the type of refinement: 0 : no refinement 1 : apply the same global offset to all the slitlets (using ARC lines) 2 : apply individual offset to each slitlet (using ARC lines) 11 : apply the same global offset to all the slitlets (using OH lines) 12 : apply individual offset to each slitlet (using OH lines) minimum_slitlet_width_mm : float Minimum slitlet width (mm) for a valid slitlet. maximum_slitlet_width_mm : float Maximum slitlet width (mm) for a valid slitlet. save_intermediate_results : bool If True, save plots in PDF files debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot. Returns ------- refined_rectwv_coeff : RectWaveCoeff instance Refined rectification and wavelength calibration coefficients for the particular CSU configuration. expected_cat_image : HDUList object Output 2D image with the expected catalogued lines. """ logger = logging.getLogger(__name__) if save_intermediate_results: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages('crosscorrelation.pdf') else: pdf = None # image header main_header = input_image[0].header filter_name = main_header['filter'] grism_name = main_header['grism'] # protections if refine_wavecalib_mode not in [1, 2, 11, 12]: logger.error('Wavelength calibration refinemente mode={}'. format( refine_wavecalib_mode )) raise ValueError("Invalid wavelength calibration refinement mode") # read tabulated lines if refine_wavecalib_mode in [1, 2]: # ARC lines if grism_name == 'LR': catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat' else: catlines_file = 'lines_argon_neon_xenon_empirical.dat' dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file) arc_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(arc_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = catlines[:, 0] catlines_all_flux = catlines[:, 1] mode = refine_wavecalib_mode elif refine_wavecalib_mode in [11, 12]: # OH lines dumdata = pkgutil.get_data( 'emirdrp.instrument.configs', 'Oliva_etal_2013.dat' ) oh_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(oh_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0])) catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2])) mode = refine_wavecalib_mode - 10 else: raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode)) # initialize output refined_rectwv_coeff = deepcopy(rectwv_coeff) logger.info('Computing median spectrum') # compute median spectrum and normalize it sp_median = median_slitlets_rectified( input_image, mode=2, minimum_slitlet_width_mm=minimum_slitlet_width_mm, maximum_slitlet_width_mm=maximum_slitlet_width_mm )[0].data sp_median /= sp_median.max() # determine minimum and maximum useful wavelength jmin, jmax = find_pix_borders(sp_median, 0) naxis1 = main_header['naxis1'] naxis2 = main_header['naxis2'] crpix1 = main_header['crpix1'] crval1 = main_header['crval1'] cdelt1 = main_header['cdelt1'] xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1 if grism_name == 'LR': wv_parameters = set_wv_parameters(filter_name, grism_name) wave_min = wv_parameters['wvmin_useful'] wave_max = wv_parameters['wvmax_useful'] else: wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1 wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1 logger.info('Setting wave_min to {}'.format(wave_min)) logger.info('Setting wave_max to {}'.format(wave_max)) # extract subset of catalogue lines within current wavelength range lok1 = catlines_all_wave >= wave_min lok2 = catlines_all_wave <= wave_max catlines_reference_wave = catlines_all_wave[lok1*lok2] catlines_reference_flux = catlines_all_flux[lok1*lok2] catlines_reference_flux /= catlines_reference_flux.max() # estimate sigma to broaden catalogue lines csu_config = CsuConfiguration.define_from_header(main_header) # segregate slitlets list_useful_slitlets = csu_config.widths_in_range_mm( minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm ) # remove missing slitlets if len(refined_rectwv_coeff.missing_slitlets) > 0: for iremove in refined_rectwv_coeff.missing_slitlets: if iremove in list_useful_slitlets: list_useful_slitlets.remove(iremove) list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1)) if i not in list_useful_slitlets] logger.info('list of useful slitlets: {}'.format( list_useful_slitlets)) logger.info('list of not useful slitlets: {}'.format( list_not_useful_slitlets)) tempwidths = np.array([csu_config.csu_bar_slit_width(islitlet) for islitlet in list_useful_slitlets]) widths_summary = summary(tempwidths) logger.info('Statistics of useful slitlet widths (mm):') logger.info('- npoints....: {0:d}'.format(widths_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(widths_summary['median'])) logger.info('- std........: {0:7.3f}'.format(widths_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std'])) # empirical transformation of slit width (mm) to pixels sigma_broadening = cdelt1 * widths_summary['median'] # convolve location of catalogue lines to generate expected spectrum xwave_reference, sp_reference = convolve_comb_lines( catlines_reference_wave, catlines_reference_flux, sigma_broadening, crpix1, crval1, cdelt1, naxis1 ) sp_reference /= sp_reference.max() # generate image2d with expected lines image2d_expected_lines = np.tile(sp_reference, (naxis2, 1)) hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header) expected_cat_image = fits.HDUList([hdu]) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xwave, sp_median, 'C1-', xlabel='Wavelength (Angstroms, in vacuum)', ylabel='Normalized number of counts', title='Median spectrum', label='observed spectrum', show=False) # overplot reference catalogue lines ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-', markerfmt=' ', basefmt='C4-', label='tabulated lines') # overplot convolved reference lines ax.plot(xwave_reference, sp_reference, 'C0-', label='expected spectrum') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) # compute baseline signal in sp_median baseline = np.percentile(sp_median[sp_median > 0], q=10) if (abs(debugplot) % 10 != 0) or (pdf is not None): fig = plt.figure() ax = fig.add_subplot(111) ax.hist(sp_median, bins=1000, log=True) ax.set_xlabel('Normalized number of counts') ax.set_ylabel('Number of pixels') ax.set_title('Median spectrum') ax.axvline(float(baseline), linestyle='--', color='grey') if pdf is not None: pdf.savefig() else: geometry = (0, 0, 640, 480) set_window_geometry(geometry) plt.show() # subtract baseline to sp_median (only pixels with signal above zero) lok = np.where(sp_median > 0) sp_median[lok] -= baseline # compute global offset through periodic correlation logger.info('Computing global offset') global_offset, fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=sp_median, fminmax=None, naround_zero=50, plottitle='Median spectrum (cross-correlation)', pdf=pdf, debugplot=debugplot ) logger.info('Global offset: {} pixels'.format(-global_offset)) missing_slitlets = rectwv_coeff.missing_slitlets if mode == 1: # apply computed offset to obtain refined_rectwv_coeff_global for islitlet in range(1, EMIR_NBARS + 1): if islitlet not in missing_slitlets: i = islitlet - 1 dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= global_offset*cdelt1 elif mode == 2: # compute individual offset for each slitlet logger.info('Computing individual offsets') median_55sp = median_slitlets_rectified(input_image, mode=1) offset_array = np.zeros(EMIR_NBARS) xplot = [] yplot = [] xplot_skipped = [] yplot_skipped = [] cout = '0' for islitlet in range(1, EMIR_NBARS + 1): if islitlet in list_useful_slitlets: i = islitlet - 1 sp_median = median_55sp[0].data[i, :] lok = np.where(sp_median > 0) if np.any(lok): baseline = np.percentile(sp_median[lok], q=10) sp_median[lok] -= baseline sp_median /= sp_median.max() offset_array[i], fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=median_55sp[0].data[i, :], fminmax=None, naround_zero=50, plottitle='slitlet #{0} (cross-correlation)'.format( islitlet), pdf=pdf, debugplot=debugplot ) else: offset_array[i] = 0.0 dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= offset_array[i]*cdelt1 xplot.append(islitlet) yplot.append(-offset_array[i]) # second correction wpoly_coeff_refined = check_wlcalib_sp( sp=median_55sp[0].data[i, :], crpix1=crpix1, crval1=crval1-offset_array[i]*cdelt1, cdelt1=cdelt1, wv_master=catlines_reference_wave, coeff_ini=dumdict['wpoly_coeff'], naxis1_ini=EMIR_NAXIS1, title='slitlet #{0} (after applying offset)'.format( islitlet), ylogscale=False, pdf=pdf, debugplot=debugplot ) dumdict['wpoly_coeff'] = wpoly_coeff_refined cout += '.' else: xplot_skipped.append(islitlet) yplot_skipped.append(0) cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) logger.info(cout) # show offsets with opposite sign stat_summary = summary(np.array(yplot)) logger.info('Statistics of individual slitlet offsets (pixels):') logger.info('- npoints....: {0:d}'.format(stat_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(stat_summary['median'])) logger.info('- std........: {0:7.3f}'.format(stat_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format(stat_summary[ 'robust_std'])) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xplot, yplot, linestyle='', marker='o', color='C0', xlabel='slitlet number', ylabel='-offset (pixels) = offset to be applied', title='cross-correlation result', show=False, **{'label': 'individual slitlets'}) if len(xplot_skipped) > 0: ax.plot(xplot_skipped, yplot_skipped, 'mx') ax.axhline(-global_offset, linestyle='--', color='C1', label='global offset') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) else: raise ValueError('Unexpected mode={}'.format(mode)) # close output PDF file if pdf is not None: pdf.close() # return result return refined_rectwv_coeff, expected_cat_image