def ximshow_rectified(self, slitlet2d_rect, subtitle=None): """Display rectified image with spectrails and frontiers. Parameters ---------- slitlet2d_rect : numpy array Array containing the rectified slitlet image. subtitle : string, optional Subtitle for plot. """ title = "Slitlet#" + str(self.islitlet) if subtitle is not None: title += ' (' + subtitle + ')' ax = ximshow(slitlet2d_rect, title=title, first_pixel=(self.bb_nc1_orig, self.bb_ns1_orig), show=False) # grid with fitted transformation: spectrum trails xx = np.arange(0, self.bb_nc2_orig - self.bb_nc1_orig + 1, dtype=np.float) for spectrail in self.list_spectrails: yy0 = self.corr_yrect_a + \ self.corr_yrect_b * spectrail(self.x0_reference) yy = np.tile([yy0 - self.bb_ns1_orig], xx.size) ax.plot(xx + self.bb_nc1_orig, yy + self.bb_ns1_orig, "b") for spectrail in self.list_frontiers: yy0 = self.corr_yrect_a +\ self.corr_yrect_b * spectrail(self.x0_reference) yy = np.tile([yy0 - self.bb_ns1_orig], xx.size) ax.plot(xx + self.bb_nc1_orig, yy + self.bb_ns1_orig, "b:") # show plot pause_debugplot(self.debugplot, pltshow=True)
def ximshow_unrectified(self, slitlet2d): """Display unrectified image with spectrails and frontiers. Parameters ---------- slitlet2d : numpy array Array containing the unrectified slitlet image. """ title = "Slitlet#" + str(self.islitlet) ax = ximshow(slitlet2d, title=title, first_pixel=(self.bb_nc1_orig, self.bb_ns1_orig), show=False) xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1) ylower = self.list_spectrails[0](xdum) ax.plot(xdum, ylower, 'b-') ymiddle = self.list_spectrails[1](xdum) ax.plot(xdum, ymiddle, 'b--') yupper = self.list_spectrails[2](xdum) ax.plot(xdum, yupper, 'b-') ylower_frontier = self.list_frontiers[0](xdum) ax.plot(xdum, ylower_frontier, 'b:') yupper_frontier = self.list_frontiers[1](xdum) ax.plot(xdum, yupper_frontier, 'b:') pause_debugplot(debugplot=self.debugplot, pltshow=True)
def ximshow_unrectified(self, slitlet2d, subtitle=None): """Display unrectified image with spectrails and frontiers. Parameters ---------- slitlet2d : numpy array Array containing the unrectified slitlet image. subtitle : string, optional Subtitle for plot. """ title = "Slitlet#" + str(self.islitlet) if subtitle is not None: title += ' (' + subtitle + ')' ax = ximshow(slitlet2d, title=title, first_pixel=(self.bb_nc1_orig, self.bb_ns1_orig), show=False) xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1) ylower = self.list_spectrails[0](xdum) ax.plot(xdum, ylower, 'b-') ymiddle = self.list_spectrails[1](xdum) ax.plot(xdum, ymiddle, 'b--') yupper = self.list_spectrails[2](xdum) ax.plot(xdum, yupper, 'b-') ylower_frontier = self.list_frontiers[0](xdum) ax.plot(xdum, ylower_frontier, 'b:') yupper_frontier = self.list_frontiers[1](xdum) ax.plot(xdum, yupper_frontier, 'b:') if title is not None: ax.set_title(title) pause_debugplot(debugplot=self.debugplot, pltshow=True)
def ximshow_unrectified(self, slitlet2d): """Display unrectified image with spectrails and frontiers. Parameters ---------- slitlet2d : numpy array Array containing the unrectified slitlet image. """ title = "Slitlet#" + str(self.islitlet) ax = ximshow(slitlet2d, title=title, first_pixel=(self.bb_nc1_orig, self.bb_ns1_orig), show=False) xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1) ylower = self.list_spectrails[0](xdum) ax.plot(xdum, ylower, 'b-') ymiddle = self.list_spectrails[1](xdum) ax.plot(xdum, ymiddle, 'b--') yupper = self.list_spectrails[2](xdum) ax.plot(xdum, yupper, 'b-') ylower_frontier = self.list_frontiers[0](xdum) ax.plot(xdum, ylower_frontier, 'b:') yupper_frontier = self.list_frontiers[1](xdum) ax.plot(xdum, yupper_frontier, 'b:') pause_debugplot(debugplot=self.debugplot, pltshow=True)
def display_slitlet_histogram(csu_bar_slit_width, n_clusters=2, geometry=None, debugplot=0): """ Find separations between groups of slitlet widths. Parameters ---------- csu_bar_slit_width : numpy array Array containing the csu_bar_slit_center values. n_clusters : int Number of slitlet groups to be sought. geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- TBD """ # protections if n_clusters < 2: raise ValueError("n_clusters must be >= 2") # determine useful slitlets from their widths km = KMeans(n_clusters=n_clusters) fit = km.fit(np.array(csu_bar_slit_width).reshape(-1, 1)) separator_list = [] for i in range(n_clusters - 1): peak1 = fit.cluster_centers_[i][0] peak2 = fit.cluster_centers_[i + 1][0] separator = (peak1 + peak2) / 2 separator_list.append(separator) print('---> separator: {0:7.3f}'.format(separator)) # display histogram if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) ax.hist(csu_bar_slit_width, bins=100) ax.set_xlabel('slit width (mm)') ax.set_ylabel('number of slitlets') for separator in separator_list: ax.axvline(separator, color='C1', linestyle='dashed') pause_debugplot(debugplot, pltshow=True)
def display_slitlet_histogram(csu_bar_slit_width, n_clusters=2, geometry=None, debugplot=0): """ Find separations between groups of slitlet widths. Parameters ---------- csu_bar_slit_width : numpy array Array containing the csu_bar_slit_center values. n_clusters : int Number of slitlet groups to be sought. geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- TBD """ # protections if n_clusters < 2: raise ValueError("n_clusters must be >= 2") # determine useful slitlets from their widths km = KMeans(n_clusters=n_clusters) fit = km.fit(np.array(csu_bar_slit_width).reshape(-1, 1)) separator_list = [] for i in range(n_clusters - 1): peak1 = fit.cluster_centers_[i][0] peak2 = fit.cluster_centers_[i+1][0] separator = (peak1 + peak2) / 2 separator_list.append(separator) print('---> separator: {0:7.3f}'.format(separator)) # display histogram if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) ax.hist(csu_bar_slit_width, bins=100) ax.set_xlabel('slit width (mm)') ax.set_ylabel('number of slitlets') for separator in separator_list: ax.axvline(separator, color='C1', linestyle='dashed') pause_debugplot(debugplot, pltshow=True)
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: compute pixel-to-pixel flatfield' ) # required arguments parser.add_argument("fitsfile", help="Input FITS file (flat ON-OFF)", type=argparse.FileType('rb')) parser.add_argument("--rectwv_coeff", required=True, help="Input JSON file with rectification and " "wavelength calibration coefficients", type=argparse.FileType('rt')) parser.add_argument("--minimum_slitlet_width_mm", required=True, help="Minimum slitlet width in mm", type=float) parser.add_argument("--maximum_slitlet_width_mm", required=True, help="Maximum slitlet width in mm", type=float) parser.add_argument("--minimum_fraction", required=True, help="Minimum allowed flatfielding value", type=float, default=0.01) parser.add_argument("--minimum_value_in_output", help="Minimum value allowed in output file: pixels " "below this value are set to 1.0 (default=0.01)", type=float, default=0.01) parser.add_argument("--maximum_value_in_output", help="Maximum value allowed in output file: pixels " "above this value are set to 1.0 (default=10.0)", type=float, default=10.0) # parser.add_argument("--nwindow_median", required=True, # help="Window size to smooth median spectrum in the " # "spectral direction", # type=int) parser.add_argument("--outfile", required=True, help="Output FITS file", type=lambda x: arg_file_is_new(parser, x, mode='wb')) # optional arguments parser.add_argument("--delta_global_integer_offset_x_pix", help="Delta global integer offset in the X direction " "(default=0)", default=0, type=int) parser.add_argument("--delta_global_integer_offset_y_pix", help="Delta global integer offset in the Y direction " "(default=0)", default=0, type=int) parser.add_argument("--resampling", help="Resampling method: 1 -> nearest neighbor, " "2 -> linear interpolation (default)", default=2, type=int, choices=(1, 2)) parser.add_argument("--ignore_DTUconf", help="Ignore DTU configurations differences between " "model and input image", action="store_true") parser.add_argument("--debugplot", help="Integer indicating plotting & debugging options" " (default=0)", default=0, type=int, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args) if args.echo: print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n') # read calibration structure from JSON file rectwv_coeff = RectWaveCoeff._datatype_load(args.rectwv_coeff.name) # modify (when requested) global offsets rectwv_coeff.global_integer_offset_x_pix += \ args.delta_global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix += \ args.delta_global_integer_offset_y_pix # read FITS image and its corresponding header hdulist = fits.open(args.fitsfile) header = hdulist[0].header image2d = hdulist[0].data hdulist.close() # apply global offsets image2d = apply_integer_offsets( image2d=image2d, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix ) # protections naxis2, naxis1 = image2d.shape if naxis1 != header['naxis1'] or naxis2 != header['naxis2']: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) raise ValueError('Something is wrong with NAXIS1 and/or NAXIS2') if abs(args.debugplot) >= 10: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) # check that the input FITS file grism and filter match filter_name = header['filter'] if filter_name != rectwv_coeff.tags['filter']: raise ValueError("Filter name does not match!") grism_name = header['grism'] if grism_name != rectwv_coeff.tags['grism']: raise ValueError("Filter name does not match!") if abs(args.debugplot) >= 10: print('>>> grism.......:', grism_name) print('>>> filter......:', filter_name) # check that the DTU configurations are compatible dtu_conf_fitsfile = DtuConfiguration.define_from_fits(args.fitsfile) dtu_conf_jsonfile = DtuConfiguration.define_from_dictionary( rectwv_coeff.meta_info['dtu_configuration']) if dtu_conf_fitsfile != dtu_conf_jsonfile: print('DTU configuration (FITS file):\n\t', dtu_conf_fitsfile) print('DTU configuration (JSON file):\n\t', dtu_conf_jsonfile) if args.ignore_DTUconf: print('WARNING: DTU configuration differences found!') else: raise ValueError('DTU configurations do not match') else: if abs(args.debugplot) >= 10: print('>>> DTU Configuration match!') print(dtu_conf_fitsfile) # load CSU configuration csu_conf_fitsfile = CsuConfiguration.define_from_fits(args.fitsfile) if abs(args.debugplot) >= 10: print(csu_conf_fitsfile) # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: print('-> Removing slitlet (not defined):', idel) list_valid_islitlets.remove(idel) # filter out slitlets with widths outside valid range list_outside_valid_width = [] for islitlet in list_valid_islitlets: slitwidth = csu_conf_fitsfile.csu_bar_slit_width(islitlet) if (slitwidth < args.minimum_slitlet_width_mm) or \ (slitwidth > args.maximum_slitlet_width_mm): list_outside_valid_width.append(islitlet) print('-> Removing slitlet (invalid width):', islitlet) if len(list_outside_valid_width) > 0: for idel in list_outside_valid_width: list_valid_islitlets.remove(idel) print('>>> valid slitlet numbers:\n', list_valid_islitlets) # --- # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop for islitlet in list(range(1, EMIR_NBARS + 1)): if islitlet in list_valid_islitlets: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS, ignore=False) # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=args.debugplot) if abs(args.debugplot) >= 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d( image_2k2k=image2d, subtitle='original image' ) # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d=slitlet2d, resampling=args.resampling, subtitle='original rectified' ) naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") # get useful slitlet region (use boundaries instead of frontiers; # note that the nscan_minmax_frontiers() works well independently # of using frontiers of boundaries as arguments) nscan_min, nscan_max = nscan_minmax_frontiers( slt.y0_reference_lower, slt.y0_reference_upper, resize=False ) ii1 = nscan_min - slt.bb_ns1_orig ii2 = nscan_max - slt.bb_ns1_orig + 1 # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction # sp_median = ndimage.median_filter( # sp_collapsed, # args.nwindow_median, # mode='nearest' # ) xaxis1 = np.arange(1, naxis1_slitlet2d + 1) nremove = 5 spl = AdaptiveLSQUnivariateSpline( x=xaxis1[nremove:-nremove], y=sp_collapsed[nremove:-nremove], t=11, adaptive=True ) xknots = spl.get_knots() yknots = spl(xknots) sp_median = spl(xaxis1) # compute rms within each knot interval nknots = len(xknots) rms_array = np.zeros(nknots - 1, dtype=float) for iknot in range(nknots - 1): residuals = [] for xdum, ydum, yydum in \ zip(xaxis1, sp_collapsed, sp_median): if xknots[iknot] <= xdum <= xknots[iknot + 1]: residuals.append(abs(ydum - yydum)) if len(residuals) > 5: rms_array[iknot] = np.std(residuals) else: rms_array[iknot] = 0 # determine in which knot interval falls each pixel iknot_array = np.zeros(len(xaxis1), dtype=int) for idum, xdum in enumerate(xaxis1): for iknot in range(nknots - 1): if xknots[iknot] <= xdum <= xknots[iknot + 1]: iknot_array[idum] = iknot # compute new fit removing deviant points (with fixed knots) xnewfit = [] ynewfit = [] for idum in range(len(xaxis1)): delta_sp = abs(sp_collapsed[idum] - sp_median[idum]) rms_tmp = rms_array[iknot_array[idum]] if idum == 0 or idum == (len(xaxis1) - 1): lok = True elif rms_tmp > 0: if delta_sp < 3.0 * rms_tmp: lok = True else: lok = False else: lok = True if lok: xnewfit.append(xaxis1[idum]) ynewfit.append(sp_collapsed[idum]) nremove = 5 splnew = AdaptiveLSQUnivariateSpline( x=xnewfit[nremove:-nremove], y=ynewfit[nremove:-nremove], t=xknots[1:-1], adaptive=False ) sp_median = splnew(xaxis1) ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * args.minimum_fraction sp_median[np.where(sp_median < y_threshold)] = 0.0 if abs(args.debugplot) > 10: title = 'Slitlet#' + str(islitlet) + ' (median spectrum)' ax = ximplotxy(xaxis1, sp_collapsed, title=title, show=False, **{'label' : 'collapsed spectrum'}) ax.plot(xaxis1, sp_median, label='fitted spectrum') ax.plot([1, naxis1_slitlet2d], 2*[y_threshold], label='threshold') ax.plot(xknots, yknots, 'o', label='knots') ax.legend() ax.set_ylim(-0.05*ymax_spmedian, 1.05*ymax_spmedian) pause_debugplot(args.debugplot, pltshow=True, tight_layout=True) # generate rectified slitlet region filled with the median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(args.debugplot) > 10: slt.ximshow_rectified( slitlet2d_rect=slitlet2d_rect_spmedian, subtitle='rectified, filled with median spectrum' ) # unrectified image slitlet2d_unrect_spmedian = slt.rectify( slitlet2d=slitlet2d_rect_spmedian, resampling=args.resampling, inverse=True, subtitle='unrectified, filled with median spectrum' ) # normalize initial slitlet image (avoid division by zero) slitlet2d_norm = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_spmedian[i, j] if den == 0: slitlet2d_norm[i, j] = 1.0 else: slitlet2d_norm[i, j] = slitlet2d[i, j] / den if abs(args.debugplot) > 10: slt.ximshow_unrectified( slitlet2d=slitlet2d_norm, subtitle='unrectified, pixel-to-pixel' ) # check for pseudo-longslit with previous slitlet if islitlet > 1: if (islitlet - 1) in list_valid_islitlets: c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet - 1) w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet - 1) c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet) w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet) if abs(w1-w2)/w1 < 0.25: wmean = (w1 + w2) / 2.0 if abs(c1 - c2) < wmean/4.0: same_slitlet_below = True else: same_slitlet_below = False else: same_slitlet_below = False else: same_slitlet_below = False else: same_slitlet_below = False # check for pseudo-longslit with previous slitlet if islitlet < EMIR_NBARS: if (islitlet + 1) in list_valid_islitlets: c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet) w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet) c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet + 1) w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet + 1) if abs(w1-w2)/w1 < 0.25: wmean = (w1 + w2) / 2.0 if abs(c1 - c2) < wmean/4.0: same_slitlet_above = True else: same_slitlet_above = False else: same_slitlet_above = False else: same_slitlet_above = False else: same_slitlet_above = False for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers if not same_slitlet_below: image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 if not same_slitlet_above: image2d_flatfielded[(n2 - 5):n2, j] = 1 else: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS, ignore=True) if args.debugplot == 0: print('OK!') # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded , offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix ) # set pixels below minimum value to 1.0 filtered = np.where(image2d_flatfielded < args.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # set pixels above maximum value to 1.0 filtered = np.where(image2d_flatfielded > args.maximum_value_in_output) image2d_flatfielded[filtered] = 1.0 # save output file save_ndarray_to_fits( array=image2d_flatfielded, file_name=args.outfile, main_header=header, overwrite=True ) print('>>> Saving file ' + args.outfile.name)
def display_slitlet_arrangement(fileobj, grism=None, spfilter=None, bbox=None, adjust=None, geometry=None, debugplot=0): """Display slitlet arrangment from CSUP keywords in FITS header. Parameters ---------- fileobj : file object FITS or TXT file object. grism : str Grism. grism : str Filter. bbox : tuple of 4 floats If not None, values for xmin, xmax, ymin and ymax. adjust : bool Adjust X range according to minimum and maximum csu_bar_left and csu_bar_right (note that this option is overriden by 'bbox') geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- csu_bar_left : list of floats Location (mm) of the left bar for each slitlet. csu_bar_right : list of floats Location (mm) of the right bar for each slitlet, using the same origin employed for csu_bar_left (which is not the value stored in the FITS keywords. csu_bar_slit_center : list of floats Middle point (mm) in between the two bars defining a slitlet. csu_bar_slit_width : list of floats Slitlet width (mm), computed as the distance between the two bars defining the slitlet. """ if fileobj.name[-4:] == ".txt": if grism is None: raise ValueError("Undefined grism!") if spfilter is None: raise ValueError("Undefined filter!") # define CsuConfiguration object csu_config = CsuConfiguration() csu_config._csu_bar_left = [] csu_config._csu_bar_right = [] csu_config._csu_bar_slit_center = [] csu_config._csu_bar_slit_width = [] # since the input filename has been opened with argparse in binary # mode, it is necessary to close it and open it in text mode fileobj.close() # read TXT file with open(fileobj.name, mode='rt') as f: file_content = f.read().splitlines() next_id_bar = 1 for line in file_content: if len(line) > 0: if line[0] not in ['#']: line_contents = line.split() id_bar = int(line_contents[0]) position = float(line_contents[1]) if id_bar == next_id_bar: if id_bar <= EMIR_NBARS: csu_config._csu_bar_left.append(position) next_id_bar = id_bar + EMIR_NBARS else: csu_config._csu_bar_right.append(341.5 - position) next_id_bar = id_bar - EMIR_NBARS + 1 else: raise ValueError("Unexpected id_bar:" + str(id_bar)) # compute slit width and center for i in range(EMIR_NBARS): csu_config._csu_bar_slit_center.append( (csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i])/2 ) csu_config._csu_bar_slit_width.append( csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i] ) else: # read input FITS file hdulist = fits.open(fileobj.name) image_header = hdulist[0].header hdulist.close() # additional info from header grism = image_header['grism'] spfilter = image_header['filter'] # define slitlet arrangement csu_config = CsuConfiguration.define_from_fits(fileobj) # determine calibration if grism in ["J", "OPEN"] and spfilter == "J": wv_parameters = set_wv_parameters("J", "J") elif grism in ["H", "OPEN"] and spfilter == "H": wv_parameters = set_wv_parameters("H", "H") elif grism in ["K", "OPEN"] and spfilter == "Ksp": wv_parameters = set_wv_parameters("Ksp", "K") elif grism in ["LR", "OPEN"] and spfilter == "YJ": wv_parameters = set_wv_parameters("YJ", "LR") elif grism in ["LR", "OPEN"] and spfilter == "HK": wv_parameters = set_wv_parameters("HK", "LR") else: raise ValueError("Invalid grism + filter configuration") crval1 = wv_parameters['poly_crval1_linear'] cdelt1 = wv_parameters['poly_cdelt1_linear'] wvmin_useful = wv_parameters['wvmin_useful'] wvmax_useful = wv_parameters['wvmax_useful'] # display arrangement if abs(debugplot) >= 10: print("slit left right center width min.wave max.wave") print("==== ======= ======= ======= ===== ======== ========") for i in range(EMIR_NBARS): ibar = i + 1 csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar)) csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar)) csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1 if wvmin_useful is not None: csu_crval1 = np.amax([csu_crval1, wvmin_useful]) if wvmax_useful is not None: csu_crvaln = np.amin([csu_crvaln, wvmax_useful]) print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} " "{5:8.2f} {6:8.2f}".format( ibar, csu_config.csu_bar_left(ibar), csu_config.csu_bar_right(ibar), csu_config.csu_bar_slit_center(ibar), csu_config.csu_bar_slit_width(ibar), csu_crval1, csu_crvaln) ) print( "---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format( np.mean(csu_config._csu_bar_left), np.mean(csu_config._csu_bar_right), np.mean(csu_config._csu_bar_slit_center), np.mean(csu_config._csu_bar_slit_width) ) ) print( "---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format( np.mean(csu_config._csu_bar_left[::2]), np.mean(csu_config._csu_bar_right[::2]), np.mean(csu_config._csu_bar_slit_center[::2]), np.mean(csu_config._csu_bar_slit_width[::2]) ) ) print( "---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format( np.mean(csu_config._csu_bar_left[1::2]), np.mean(csu_config._csu_bar_right[1::2]), np.mean(csu_config._csu_bar_slit_center[1::2]), np.mean(csu_config._csu_bar_slit_width[1::2]) ) ) # display slit arrangement if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) if bbox is None: if adjust: xmin = min(csu_config._csu_bar_left) xmax = max(csu_config._csu_bar_right) dx = xmax - xmin if dx == 0: dx = 1 xmin -= dx/20 xmax += dx/20 ax.set_xlim(xmin, xmax) else: ax.set_xlim(0., 341.5) ax.set_ylim(0, 56) else: ax.set_xlim(bbox[0], bbox[1]) ax.set_ylim(bbox[2], bbox[3]) ax.set_xlabel('csu_bar_position (mm)') ax.set_ylabel('slit number') for i in range(EMIR_NBARS): ibar = i + 1 ax.add_patch(patches.Rectangle( (csu_config.csu_bar_left(ibar), ibar-0.5), csu_config.csu_bar_slit_width(ibar), 1.0)) ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar], '-', color='gray') ax.plot([csu_config.csu_bar_right(ibar), 341.5], [ibar, ibar], '-', color='gray') plt.title("File: " + fileobj.name + "\ngrism=" + grism + ", filter=" + spfilter) pause_debugplot(debugplot, pltshow=True) # return results return csu_config._csu_bar_left, csu_config._csu_bar_right, \ csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: display arrangement of EMIR CSU bars' ) # positional arguments parser.add_argument("filename", help="FITS files (wildcards accepted) or single TXT " "file with CSU configuration from OSP", type=argparse.FileType('rb'), nargs='+') # optional arguments parser.add_argument("--grism", help="Grism (J, H, K, LR)", choices=["J", "H", "K", "LR"]) parser.add_argument("--filter", help="Filter (J, H, Ksp, YJ, HK)", choices=["J", "H", "Ksp", "YJ", "HK"]) parser.add_argument("--n_clusters", help="Display histogram of slitlet widths", default=0, type=int) parser.add_argument("--bbox", help="Bounding box tuple xmin,xmax,ymin,ymax " "indicating plot limits") parser.add_argument("--adjust", help="Adjust X range according to minimum and maximum" " csu_bar_left and csu_bar_right (note that this " "option is overriden by --bbox", action='store_true') parser.add_argument("--geometry", help="Tuple x,y,dx,dy indicating window geometry", default="0,0,640,480") parser.add_argument("--debugplot", help="Integer indicating plotting & debugging options" " (default=12)", default=12, type=int, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args) if args.echo: print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n') # geometry if args.geometry is None: geometry = None else: tmp_str = args.geometry.split(",") x_geom = int(tmp_str[0]) y_geom = int(tmp_str[1]) dx_geom = int(tmp_str[2]) dy_geom = int(tmp_str[3]) geometry = x_geom, y_geom, dx_geom, dy_geom # read bounding box if args.bbox is None: bbox = None else: str_bbox = args.bbox.split(",") xmin, xmax, ymin, ymax = [int(str_bbox[i]) for i in range(4)] bbox = xmin, xmax, ymin, ymax list_fits_file_objects = [] # if input file is a single txt file, assume it is a list of FITS files if len(args.filename) == 1: list_fits_file_objects = [args.filename[0]] else: list_fits_file_objects = args.filename # total number of files to be examined nfiles = len(list_fits_file_objects) # declare arrays to store CSU values csu_bar_left = np.zeros((nfiles, EMIR_NBARS)) csu_bar_right = np.zeros((nfiles, EMIR_NBARS)) csu_bar_slit_center = np.zeros((nfiles, EMIR_NBARS)) csu_bar_slit_width = np.zeros((nfiles, EMIR_NBARS)) # display CSU bar arrangement for ifile, fileobj in enumerate(list_fits_file_objects): print("\nFile " + str(ifile+1) + "/" + str(nfiles) + ": " + fileobj.name) csu_bar_left[ifile, :], csu_bar_right[ifile, :], \ csu_bar_slit_center[ifile, :], csu_bar_slit_width[ifile, :] = \ display_slitlet_arrangement( fileobj, grism=args.grism, spfilter=args.filter, bbox=bbox, adjust=args.adjust, geometry=geometry, debugplot=args.debugplot ) if args.n_clusters >= 2: display_slitlet_histogram( csu_bar_slit_width[ifile, :], n_clusters=args.n_clusters, geometry=geometry, debugplot=args.debugplot ) # print summary of comparison between files if nfiles > 1: std_csu_bar_left = np.zeros(EMIR_NBARS) std_csu_bar_right = np.zeros(EMIR_NBARS) std_csu_bar_slit_center = np.zeros(EMIR_NBARS) std_csu_bar_slit_width = np.zeros(EMIR_NBARS) if args.debugplot >= 10: print("\n STANDARD DEVIATION BETWEEN IMAGES") print("slit left right center width") print("==== ======= ======= ======= =====") for i in range(EMIR_NBARS): ibar = i + 1 std_csu_bar_left[i] = np.std(csu_bar_left[:, i]) std_csu_bar_right[i] = np.std(csu_bar_right[:, i]) std_csu_bar_slit_center[i] = np.std(csu_bar_slit_center[:, i]) std_csu_bar_slit_width[i] = np.std(csu_bar_slit_width[:, i]) print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f}".format( ibar, std_csu_bar_left[i], std_csu_bar_right[i], std_csu_bar_slit_center[i], std_csu_bar_slit_width[i])) print("==== ======= ======= ======= =====") print("MIN: {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f}".format( std_csu_bar_left.min(), std_csu_bar_right.min(), std_csu_bar_slit_center.min(), std_csu_bar_slit_width.min())) print("MAX: {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f}".format( std_csu_bar_left.max(), std_csu_bar_right.max(), std_csu_bar_slit_center.max(), std_csu_bar_slit_width.max())) print("==== ======= ======= ======= =====") print("Total number of files examined:", nfiles) # stop program execution if len(list_fits_file_objects) > 1: pause_debugplot(12, optional_prompt="Press RETURN to STOP")
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: compute pixel-to-pixel flatfield' ) # required arguments parser.add_argument("fitsfile", help="Input FITS file (flat ON-OFF)", type=argparse.FileType('rb')) parser.add_argument("--rectwv_coeff", required=True, help="Input JSON file with rectification and " "wavelength calibration coefficients", type=argparse.FileType('rt')) parser.add_argument("--minimum_slitlet_width_mm", required=True, help="Minimum slitlet width in mm", type=float) parser.add_argument("--maximum_slitlet_width_mm", required=True, help="Maximum slitlet width in mm", type=float) parser.add_argument("--minimum_fraction", required=True, help="Minimum allowed flatfielding value", type=float, default=0.01) parser.add_argument("--minimum_value_in_output", help="Minimum value allowed in output file: pixels " "below this value are set to 1.0 (default=0.01)", type=float, default=0.01) parser.add_argument("--maximum_value_in_output", help="Maximum value allowed in output file: pixels " "above this value are set to 1.0 (default=10.0)", type=float, default=10.0) parser.add_argument("--nwindow_median", help="Window size to smooth median spectrum in the " "spectral direction", type=int) parser.add_argument("--outfile", required=True, help="Output FITS file", type=lambda x: arg_file_is_new(parser, x, mode='wb')) # optional arguments parser.add_argument("--delta_global_integer_offset_x_pix", help="Delta global integer offset in the X direction " "(default=0)", default=0, type=int) parser.add_argument("--delta_global_integer_offset_y_pix", help="Delta global integer offset in the Y direction " "(default=0)", default=0, type=int) parser.add_argument("--resampling", help="Resampling method: 1 -> nearest neighbor, " "2 -> linear interpolation (default)", default=2, type=int, choices=(1, 2)) parser.add_argument("--ignore_DTUconf", help="Ignore DTU configurations differences between " "model and input image", action="store_true") parser.add_argument("--debugplot", help="Integer indicating plotting & debugging options" " (default=0)", default=0, type=int, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args) if args.echo: print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n') # This code is obsolete raise ValueError('This code is obsolete: use recipe in ' 'emirdrp/recipes/spec/flatpix2pix.py') # read calibration structure from JSON file rectwv_coeff = RectWaveCoeff._datatype_load(args.rectwv_coeff.name) # modify (when requested) global offsets rectwv_coeff.global_integer_offset_x_pix += \ args.delta_global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix += \ args.delta_global_integer_offset_y_pix # read FITS image and its corresponding header hdulist = fits.open(args.fitsfile) header = hdulist[0].header image2d = hdulist[0].data hdulist.close() # apply global offsets image2d = apply_integer_offsets( image2d=image2d, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix ) # protections naxis2, naxis1 = image2d.shape if naxis1 != header['naxis1'] or naxis2 != header['naxis2']: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) raise ValueError('Something is wrong with NAXIS1 and/or NAXIS2') if abs(args.debugplot) >= 10: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) # check that the input FITS file grism and filter match filter_name = header['filter'] if filter_name != rectwv_coeff.tags['filter']: raise ValueError("Filter name does not match!") grism_name = header['grism'] if grism_name != rectwv_coeff.tags['grism']: raise ValueError("Filter name does not match!") if abs(args.debugplot) >= 10: print('>>> grism.......:', grism_name) print('>>> filter......:', filter_name) # check that the DTU configurations are compatible dtu_conf_fitsfile = DtuConfiguration.define_from_fits(args.fitsfile) dtu_conf_jsonfile = DtuConfiguration.define_from_dictionary( rectwv_coeff.meta_info['dtu_configuration']) if dtu_conf_fitsfile != dtu_conf_jsonfile: print('DTU configuration (FITS file):\n\t', dtu_conf_fitsfile) print('DTU configuration (JSON file):\n\t', dtu_conf_jsonfile) if args.ignore_DTUconf: print('WARNING: DTU configuration differences found!') else: raise ValueError('DTU configurations do not match') else: if abs(args.debugplot) >= 10: print('>>> DTU Configuration match!') print(dtu_conf_fitsfile) # load CSU configuration csu_conf_fitsfile = CsuConfiguration.define_from_fits(args.fitsfile) if abs(args.debugplot) >= 10: print(csu_conf_fitsfile) # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: print('-> Removing slitlet (not defined):', idel) list_valid_islitlets.remove(idel) # filter out slitlets with widths outside valid range list_outside_valid_width = [] for islitlet in list_valid_islitlets: slitwidth = csu_conf_fitsfile.csu_bar_slit_width(islitlet) if (slitwidth < args.minimum_slitlet_width_mm) or \ (slitwidth > args.maximum_slitlet_width_mm): list_outside_valid_width.append(islitlet) print('-> Removing slitlet (invalid width):', islitlet) if len(list_outside_valid_width) > 0: for idel in list_outside_valid_width: list_valid_islitlets.remove(idel) print('>>> valid slitlet numbers:\n', list_valid_islitlets) # --- # compute and store median spectrum (and masked region) for each # individual slitlet image2d_sp_median = np.zeros((EMIR_NBARS, EMIR_NAXIS1)) image2d_sp_mask = np.zeros((EMIR_NBARS, EMIR_NAXIS1), dtype=bool) for islitlet in list(range(1, EMIR_NBARS + 1)): if islitlet in list_valid_islitlets: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS, ignore=False) # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=args.debugplot) if abs(args.debugplot) >= 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d( image_2k2k=image2d, subtitle='original image' ) # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d=slitlet2d, resampling=args.resampling, subtitle='original rectified' ) naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") sp_mask = np.zeros(naxis1_slitlet2d, dtype=bool) # for grism LR set to zero data beyond useful wavelength range if grism_name == 'LR': wv_parameters = set_wv_parameters(filter_name, grism_name) x_pix = np.arange(1, naxis1_slitlet2d + 1) wl_pix = polyval(x_pix, slt.wpoly) lremove = wl_pix < wv_parameters['wvmin_useful'] sp_mask[lremove] = True slitlet2d_rect[:, lremove] = 0.0 lremove = wl_pix > wv_parameters['wvmax_useful'] slitlet2d_rect[:, lremove] = 0.0 sp_mask[lremove] = True # get useful slitlet region (use boundaries instead of frontiers; # note that the nscan_minmax_frontiers() works well independently # of using frontiers of boundaries as arguments) nscan_min, nscan_max = nscan_minmax_frontiers( slt.y0_reference_lower, slt.y0_reference_upper, resize=False ) ii1 = nscan_min - slt.bb_ns1_orig ii2 = nscan_max - slt.bb_ns1_orig + 1 # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction sp_median = ndimage.median_filter( sp_collapsed, args.nwindow_median, mode='nearest' ) """ nremove = 5 spl = AdaptiveLSQUnivariateSpline( x=xaxis1[nremove:-nremove], y=sp_collapsed[nremove:-nremove], t=11, adaptive=True ) xknots = spl.get_knots() yknots = spl(xknots) sp_median = spl(xaxis1) # compute rms within each knot interval nknots = len(xknots) rms_array = np.zeros(nknots - 1, dtype=float) for iknot in range(nknots - 1): residuals = [] for xdum, ydum, yydum in \ zip(xaxis1, sp_collapsed, sp_median): if xknots[iknot] <= xdum <= xknots[iknot + 1]: residuals.append(abs(ydum - yydum)) if len(residuals) > 5: rms_array[iknot] = np.std(residuals) else: rms_array[iknot] = 0 # determine in which knot interval falls each pixel iknot_array = np.zeros(len(xaxis1), dtype=int) for idum, xdum in enumerate(xaxis1): for iknot in range(nknots - 1): if xknots[iknot] <= xdum <= xknots[iknot + 1]: iknot_array[idum] = iknot # compute new fit removing deviant points (with fixed knots) xnewfit = [] ynewfit = [] for idum in range(len(xaxis1)): delta_sp = abs(sp_collapsed[idum] - sp_median[idum]) rms_tmp = rms_array[iknot_array[idum]] if idum == 0 or idum == (len(xaxis1) - 1): lok = True elif rms_tmp > 0: if delta_sp < 3.0 * rms_tmp: lok = True else: lok = False else: lok = True if lok: xnewfit.append(xaxis1[idum]) ynewfit.append(sp_collapsed[idum]) nremove = 5 splnew = AdaptiveLSQUnivariateSpline( x=xnewfit[nremove:-nremove], y=ynewfit[nremove:-nremove], t=xknots[1:-1], adaptive=False ) sp_median = splnew(xaxis1) """ ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * args.minimum_fraction lremove = np.where(sp_median < y_threshold) sp_median[lremove] = 0.0 sp_mask[lremove] = True image2d_sp_median[islitlet - 1, :] = sp_median image2d_sp_mask[islitlet - 1, :] = sp_mask if abs(args.debugplot) % 10 != 0: xaxis1 = np.arange(1, naxis1_slitlet2d + 1) title = 'Slitlet#' + str(islitlet) + ' (median spectrum)' ax = ximplotxy(xaxis1, sp_collapsed, title=title, show=False, **{'label' : 'collapsed spectrum'}) ax.plot(xaxis1, sp_median, label='fitted spectrum') ax.plot([1, naxis1_slitlet2d], 2*[y_threshold], label='threshold') # ax.plot(xknots, yknots, 'o', label='knots') ax.legend() ax.set_ylim(-0.05*ymax_spmedian, 1.05*ymax_spmedian) pause_debugplot(args.debugplot, pltshow=True, tight_layout=True) else: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS, ignore=True) # ToDo: compute "average" spectrum for each pseudo-longslit, scaling # with the median signal in each slitlet; derive a particular # spectrum for each slitlet (scaling properly) image2d_sp_median_masked = np.ma.masked_array( image2d_sp_median, mask=image2d_sp_mask ) ycut_median = np.ma.median(image2d_sp_median_masked, axis=1).data ycut_median_2d = np.repeat(ycut_median, EMIR_NAXIS1).reshape( EMIR_NBARS, EMIR_NAXIS1) image2d_sp_median_eq = image2d_sp_median_masked / ycut_median_2d image2d_sp_median_eq = image2d_sp_median_eq.data if True: ximshow(image2d_sp_median, title='sp_median', debugplot=12) ximplotxy(np.arange(1, EMIR_NBARS + 1), ycut_median, 'ro', title='median value of each spectrum', debugplot=12) ximshow(image2d_sp_median_eq, title='sp_median_eq', debugplot=12) csu_conf_fitsfile.display_pseudo_longslits( list_valid_slitlets=list_valid_islitlets) dict_longslits = csu_conf_fitsfile.pseudo_longslits() # compute median spectrum for each longslit and insert (properly # scaled) that spectrum in each slitlet belonging to that longslit image2d_sp_median_longslit = np.zeros((EMIR_NBARS, EMIR_NAXIS1)) islitlet = 1 loop = True while loop: if islitlet in list_valid_islitlets: imin = dict_longslits[islitlet].imin() imax = dict_longslits[islitlet].imax() print('--> imin, imax: ', imin, imax) sp_median_longslit = np.median( image2d_sp_median_eq[(imin - 1):imax, :], axis=0) for i in range(imin, imax+1): print('----> i: ', i) image2d_sp_median_longslit[(i - 1), :] = \ sp_median_longslit * ycut_median[i - 1] islitlet = imax else: print('--> ignoring: ', islitlet) if islitlet == EMIR_NBARS: loop = False else: islitlet += 1 if True: ximshow(image2d_sp_median_longslit, debugplot=12) # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop for islitlet in list(range(1, EMIR_NBARS + 1)): if islitlet in list_valid_islitlets: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS, ignore=False) # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=args.debugplot) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d( image_2k2k=image2d, subtitle='original image' ) # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d=slitlet2d, resampling=args.resampling, subtitle='original rectified' ) naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape sp_median = image2d_sp_median_longslit[islitlet - 1, :] # generate rectified slitlet region filled with the median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(args.debugplot) > 10: slt.ximshow_rectified( slitlet2d_rect=slitlet2d_rect_spmedian, subtitle='rectified, filled with median spectrum' ) # unrectified image slitlet2d_unrect_spmedian = slt.rectify( slitlet2d=slitlet2d_rect_spmedian, resampling=args.resampling, inverse=True, subtitle='unrectified, filled with median spectrum' ) # normalize initial slitlet image (avoid division by zero) slitlet2d_norm = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_spmedian[i, j] if den == 0: slitlet2d_norm[i, j] = 1.0 else: slitlet2d_norm[i, j] = slitlet2d[i, j] / den if abs(args.debugplot) > 10: slt.ximshow_unrectified( slitlet2d=slitlet2d_norm, subtitle='unrectified, pixel-to-pixel' ) # check for pseudo-longslit with previous slitlet if islitlet > 1: if (islitlet - 1) in list_valid_islitlets: c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet - 1) w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet - 1) c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet) w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet) if abs(w1-w2)/w1 < 0.25: wmean = (w1 + w2) / 2.0 if abs(c1 - c2) < wmean/4.0: same_slitlet_below = True else: same_slitlet_below = False else: same_slitlet_below = False else: same_slitlet_below = False else: same_slitlet_below = False # check for pseudo-longslit with next slitlet if islitlet < EMIR_NBARS: if (islitlet + 1) in list_valid_islitlets: c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet) w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet) c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet + 1) w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet + 1) if abs(w1-w2)/w1 < 0.25: wmean = (w1 + w2) / 2.0 if abs(c1 - c2) < wmean/4.0: same_slitlet_above = True else: same_slitlet_above = False else: same_slitlet_above = False else: same_slitlet_above = False else: same_slitlet_above = False for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers if not same_slitlet_below: image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 if not same_slitlet_above: image2d_flatfielded[(n2 - 5):n2, j] = 1 else: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS, ignore=True) if args.debugplot == 0: print('OK!') # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded , offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix ) # set pixels below minimum value to 1.0 filtered = np.where(image2d_flatfielded < args.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # set pixels above maximum value to 1.0 filtered = np.where(image2d_flatfielded > args.maximum_value_in_output) image2d_flatfielded[filtered] = 1.0 # save output file save_ndarray_to_fits( array=image2d_flatfielded, file_name=args.outfile, main_header=header, overwrite=True ) print('>>> Saving file ' + args.outfile.name)
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: compute pixel-to-pixel flatfield') # required arguments parser.add_argument("fitsfile", help="Input FITS file (flat ON-OFF)", type=argparse.FileType('rb')) parser.add_argument("--rectwv_coeff", required=True, help="Input JSON file with rectification and " "wavelength calibration coefficients", type=argparse.FileType('rt')) parser.add_argument("--minimum_fraction", required=True, help="Minimum allowed flatfielding value", type=float, default=0.01) parser.add_argument("--minimum_value_in_output", help="Minimum value allowed in output file: pixels " "below this value are set to 1.0 (default=0.01)", type=float, default=0.01) parser.add_argument("--nwindow_median", required=True, help="Window size to smooth median spectrum in the " "spectral direction", type=int) parser.add_argument("--outfile", required=True, help="Output FITS file", type=lambda x: arg_file_is_new(parser, x, mode='wb')) # optional arguments parser.add_argument("--delta_global_integer_offset_x_pix", help="Delta global integer offset in the X direction " "(default=0)", default=0, type=int) parser.add_argument("--delta_global_integer_offset_y_pix", help="Delta global integer offset in the Y direction " "(default=0)", default=0, type=int) parser.add_argument("--resampling", help="Resampling method: 1 -> nearest neighbor, " "2 -> linear interpolation (default)", default=2, type=int, choices=(1, 2)) parser.add_argument("--ignore_DTUconf", help="Ignore DTU configurations differences between " "model and input image", action="store_true") parser.add_argument("--debugplot", help="Integer indicating plotting & debugging options" " (default=0)", default=0, type=int, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args) if args.echo: print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n') # read calibration structure from JSON file rectwv_coeff = RectWaveCoeff._datatype_load(args.rectwv_coeff.name) # modify (when requested) global offsets rectwv_coeff.global_integer_offset_x_pix += \ args.delta_global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix += \ args.delta_global_integer_offset_y_pix # read FITS image and its corresponding header hdulist = fits.open(args.fitsfile) header = hdulist[0].header image2d = hdulist[0].data hdulist.close() # apply global offsets image2d = apply_integer_offsets( image2d=image2d, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix) # protections naxis2, naxis1 = image2d.shape if naxis1 != header['naxis1'] or naxis2 != header['naxis2']: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) raise ValueError('Something is wrong with NAXIS1 and/or NAXIS2') if abs(args.debugplot) >= 10: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) # check that the input FITS file grism and filter match filter_name = header['filter'] if filter_name != rectwv_coeff.tags['filter']: raise ValueError("Filter name does not match!") grism_name = header['grism'] if grism_name != rectwv_coeff.tags['grism']: raise ValueError("Filter name does not match!") if abs(args.debugplot) >= 10: print('>>> grism.......:', grism_name) print('>>> filter......:', filter_name) # check that the DTU configurations are compatible dtu_conf_fitsfile = DtuConfiguration.define_from_fits(args.fitsfile) dtu_conf_jsonfile = DtuConfiguration.define_from_dictionary( rectwv_coeff.meta_info['dtu_configuration']) if dtu_conf_fitsfile != dtu_conf_jsonfile: print('DTU configuration (FITS file):\n\t', dtu_conf_fitsfile) print('DTU configuration (JSON file):\n\t', dtu_conf_jsonfile) if args.ignore_DTUconf: print('WARNING: DTU configuration differences found!') else: raise ValueError('DTU configurations do not match') else: if abs(args.debugplot) >= 10: print('>>> DTU Configuration match!') print(dtu_conf_fitsfile) # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: list_valid_islitlets.remove(idel) if abs(args.debugplot) >= 10: print('>>> valid slitlet numbers:\n', list_valid_islitlets) # --- # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop for islitlet in list_valid_islitlets: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS) # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=args.debugplot) if abs(args.debugplot) >= 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d(image2d) # rectify slitlet slitlet2d_rect = slt.rectify(slitlet2d, resampling=args.resampling) naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") # get useful slitlet region (use boundaires instead of frontiers; # note that the nscan_minmax_frontiers() works well independently # of using frontiers of boundaries as arguments) nscan_min, nscan_max = nscan_minmax_frontiers(slt.y0_reference_lower, slt.y0_reference_upper, resize=False) ii1 = nscan_min - slt.bb_ns1_orig ii2 = nscan_max - slt.bb_ns1_orig + 1 # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction sp_median = ndimage.median_filter(sp_collapsed, args.nwindow_median, mode='nearest') ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * args.minimum_fraction sp_median[np.where(sp_median < y_threshold)] = 0.0 if abs(args.debugplot) > 10: title = 'Slitlet#' + str(islitlet) + '(median spectrum)' xdum = np.arange(1, naxis1_slitlet2d + 1) ax = ximplotxy(xdum, sp_collapsed, title=title, show=False, **{'label': 'collapsed spectrum'}) ax.plot(xdum, sp_median, label='filtered spectrum') ax.plot([1, naxis1_slitlet2d], 2 * [y_threshold], label='threshold') ax.legend() ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian) pause_debugplot(args.debugplot, pltshow=True, tight_layout=True) # generate rectified slitlet region filled with the median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(args.debugplot) > 10: slt.ximshow_rectified(slitlet2d_rect_spmedian) # unrectified image slitlet2d_unrect_spmedian = slt.rectify(slitlet2d_rect_spmedian, resampling=args.resampling, inverse=True) # normalize initial slitlet image (avoid division by zero) slitlet2d_norm = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_spmedian[i, j] if den == 0: slitlet2d_norm[i, j] = 1.0 else: slitlet2d_norm[i, j] = slitlet2d[i, j] / den if abs(args.debugplot) > 10: slt.ximshow_unrectified(slitlet2d_norm) for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 image2d_flatfielded[(n2 - 5):n2, j] = 1 if args.debugplot == 0: print('OK!') # set pixels below minimum value to 1.0 filtered = np.where(image2d_flatfielded < args.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded, offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix) # save output file save_ndarray_to_fits(array=image2d_flatfielded, file_name=args.outfile, main_header=header, overwrite=True) print('>>> Saving file ' + args.outfile.name)
def compute_slitlet_boundaries(filename, grism, spfilter, list_slitlets, size_x_medfilt, size_y_savgol, times_sigma_threshold, bounddict, debugplot=0): """Compute slitlet boundaries using continuum lamp images. Parameters ---------- filename : string Input continumm lamp image. grism : string Grism name. It must be one in EMIR_VALID_GRISMS. spfilter : string Filter name. It must be one in EMIR_VALID_FILTERS. list_slitlets : list of integers Number of slitlets to be updated. size_x_medfilt : int Window in the X (spectral) direction, in pixels, to apply the 1d median filter in order to remove bad pixels. size_y_savgol : int Window in the Y (spatial) direction to be used when using the 1d Savitzky-Golay filter. times_sigma_threshold : float Times sigma to detect peaks in derivatives. bounddict : dictionary of dictionaries Structure to store the boundaries. debugplot : int Determines whether intermediate computations and/or plots are displayed. """ # read 2D image hdulist = fits.open(filename) image_header = hdulist[0].header image2d = hdulist[0].data naxis2, naxis1 = image2d.shape hdulist.close() if debugplot >= 10: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) # ToDo: replace this by application of cosmetic defect mask! for j in range(1024): image2d[1024, j] = (image2d[1023, j] + image2d[1025, j]) / 2 image2d[1023, j + 1024] = (image2d[1022, j + 1024] + image2d[1024, j + 1024]) / 2 # remove path from filename sfilename = os.path.basename(filename) # check that the FITS file has been obtained with EMIR instrument = image_header['instrume'] if instrument != 'EMIR': raise ValueError("INSTRUME keyword is not 'EMIR'!") # read CSU configuration from FITS header csu_config = CsuConfiguration.define_from_fits(filename) # read DTU configuration from FITS header dtu_config = DtuConfiguration.define_from_fits(filename) # read grism grism_in_header = image_header['grism'] if grism != grism_in_header: raise ValueError("GRISM keyword=" + grism_in_header + " is not the expected value=" + grism) # read filter spfilter_in_header = image_header['filter'] if spfilter != spfilter_in_header: raise ValueError("FILTER keyword=" + spfilter_in_header + " is not the expected value=" + spfilter) # read rotator position angle rotang = image_header['rotang'] # read date-obs date_obs = image_header['date-obs'] for islitlet in list_slitlets: if debugplot < 10: sys.stdout.write('.') sys.stdout.flush() sltlim = SlitletLimits(grism, spfilter, islitlet) # extract slitlet2d slitlet2d = extract_slitlet2d(image2d, sltlim) if debugplot % 10 != 0: ximshow(slitlet2d, title=sfilename + " [original]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), debugplot=debugplot) # apply 1d median filtering (along the spectral direction) # to remove bad pixels size_x = size_x_medfilt size_y = 1 slitlet2d_smooth = ndimage.filters.median_filter(slitlet2d, size=(size_y, size_x)) if debugplot % 10 != 0: ximshow(slitlet2d_smooth, title=sfilename + " [smoothed]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), debugplot=debugplot) # apply 1d Savitzky-Golay filter (along the spatial direction) # to compute first derivative slitlet2d_savgol = savgol_filter(slitlet2d_smooth, window_length=size_y_savgol, polyorder=2, deriv=1, axis=0) # compute basic statistics q25, q50, q75 = np.percentile(slitlet2d_savgol, q=[25.0, 50.0, 75.0]) sigmag = 0.7413 * (q75 - q25) # robust standard deviation if debugplot >= 10: print("q50, sigmag:", q50, sigmag) if debugplot % 10 != 0: ximshow(slitlet2d_savgol, title=sfilename + " [S.-G.filt.]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), z1z2=(q50 - times_sigma_threshold * sigmag, q50 + times_sigma_threshold * sigmag), debugplot=debugplot) # identify objects in slitlet2d_savgol: pixels with positive # derivatives are identify independently from pixels with # negative derivaties; then the two set of potential features # are merged; this approach avoids some problems when, in # nearby regions, there are pixels with positive and negative # derivatives (in those circumstances a single search as # np.logical_or( # slitlet2d_savgol < q50 - times_sigma_threshold * sigmag, # slitlet2d_savgol > q50 + times_sigma_threshold * sigmag) # led to erroneous detections!) # # search for positive derivatives labels2d_objects_pos, no_objects_pos = ndimage.label( slitlet2d_savgol > q50 + times_sigma_threshold * sigmag) # search for negative derivatives labels2d_objects_neg, no_objects_neg = ndimage.label( slitlet2d_savgol < q50 - times_sigma_threshold * sigmag) # merge both sets non_zero_neg = np.where(labels2d_objects_neg > 0) labels2d_objects = np.copy(labels2d_objects_pos) labels2d_objects[non_zero_neg] += \ labels2d_objects_neg[non_zero_neg] + no_objects_pos no_objects = no_objects_pos + no_objects_neg if debugplot >= 10: print("Number of objects with positive derivative:", no_objects_pos) print("Number of objects with negative derivative:", no_objects_neg) print("Total number of objects initially found...:", no_objects) if debugplot % 10 != 0: ximshow(labels2d_objects, z1z2=(0, no_objects), title=sfilename + " [objects]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), cbar_label="Object number", debugplot=debugplot) # select boundaries as the largest objects found with # positive and negative derivatives n_der_pos = 0 # number of pixels covered by the object with deriv > 0 i_der_pos = 0 # id of the object with deriv > 0 n_der_neg = 0 # number of pixels covered by the object with deriv < 0 i_der_neg = 0 # id of the object with deriv < 0 for i in range(1, no_objects + 1): xy_tmp = np.where(labels2d_objects == i) n_pix = len(xy_tmp[0]) if i <= no_objects_pos: if n_pix > n_der_pos: i_der_pos = i n_der_pos = n_pix else: if n_pix > n_der_neg: i_der_neg = i n_der_neg = n_pix # determine which boundary is lower and which is upper y_center_mass_der_pos = ndimage.center_of_mass(slitlet2d_savgol, labels2d_objects, [i_der_pos])[0][0] y_center_mass_der_neg = ndimage.center_of_mass(slitlet2d_savgol, labels2d_objects, [i_der_neg])[0][0] if y_center_mass_der_pos < y_center_mass_der_neg: i_lower = i_der_pos i_upper = i_der_neg if debugplot >= 10: print("-> lower boundary has positive derivatives") else: i_lower = i_der_neg i_upper = i_der_pos if debugplot >= 10: print("-> lower boundary has negative derivatives") list_slices_ok = [i_lower, i_upper] # adjust individual boundaries passing the selection: # - select points in the image belonging to a given boundary # - compute weighted mean of the pixels of the boundary, column # by column (this reduces dramatically the number of points # to be fitted to determine the boundary) list_boundaries = [] for k in range(2): # k=0 lower boundary, k=1 upper boundary # select points to be fitted for a particular boundary # (note: be careful with array indices and pixel # coordinates) xy_tmp = np.where(labels2d_objects == list_slices_ok[k]) xmin = xy_tmp[1].min() # array indices (integers) xmax = xy_tmp[1].max() # array indices (integers) xfit = [] yfit = [] # fix range for fit if k == 0: xmineff = max(sltlim.xmin_lower_boundary_fit, xmin) xmaxeff = min(sltlim.xmax_lower_boundary_fit, xmax) else: xmineff = max(sltlim.xmin_upper_boundary_fit, xmin) xmaxeff = min(sltlim.xmax_upper_boundary_fit, xmax) # loop in columns of the image belonging to the boundary for xdum in range(xmineff, xmaxeff + 1): # array indices (integer) iok = np.where(xy_tmp[1] == xdum) y_tmp = xy_tmp[0][iok] + sltlim.bb_ns1_orig # image pixel weight = slitlet2d_savgol[xy_tmp[0][iok], xy_tmp[1][iok]] y_wmean = sum(y_tmp * weight) / sum(weight) xfit.append(xdum + sltlim.bb_nc1_orig) yfit.append(y_wmean) xfit = np.array(xfit) yfit = np.array(yfit) # declare new SpectrumTrail instance boundary = SpectrumTrail() # define new boundary boundary.fit(x=xfit, y=yfit, deg=sltlim.deg_boundary, times_sigma_reject=10, title="slit:" + str(sltlim.islitlet) + ", deg=" + str(sltlim.deg_boundary), debugplot=0) list_boundaries.append(boundary) if debugplot % 10 != 0: for tmp_img, tmp_label in zip([slitlet2d_savgol, slitlet2d], [' [S.-G.filt.]', ' [original]']): ax = ximshow(tmp_img, title=sfilename + tmp_label + "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), show=False, debugplot=debugplot) for k in range(2): xpol, ypol = list_boundaries[k].linspace_pix( start=1, stop=EMIR_NAXIS1) ax.plot(xpol, ypol, 'b--', linewidth=1) for k in range(2): xpol, ypol = list_boundaries[k].linspace_pix() ax.plot(xpol, ypol, 'g--', linewidth=4) # show plot pause_debugplot(debugplot, pltshow=True) # update bounddict tmp_dict = { 'boundary_coef_lower': list_boundaries[0].poly_funct.coef.tolist(), 'boundary_xmin_lower': list_boundaries[0].xlower_line, 'boundary_xmax_lower': list_boundaries[0].xupper_line, 'boundary_coef_upper': list_boundaries[1].poly_funct.coef.tolist(), 'boundary_xmin_upper': list_boundaries[1].xlower_line, 'boundary_xmax_upper': list_boundaries[1].xupper_line, 'csu_bar_left': csu_config.csu_bar_left(islitlet), 'csu_bar_right': csu_config.csu_bar_right(islitlet), 'csu_bar_slit_center': csu_config.csu_bar_slit_center(islitlet), 'csu_bar_slit_width': csu_config.csu_bar_slit_width(islitlet), 'rotang': rotang, 'xdtu': dtu_config.xdtu, 'ydtu': dtu_config.ydtu, 'zdtu': dtu_config.zdtu, 'xdtu_0': dtu_config.xdtu_0, 'ydtu_0': dtu_config.ydtu_0, 'zdtu_0': dtu_config.zdtu_0, 'zzz_info1': os.getlogin() + '@' + socket.gethostname(), 'zzz_info2': datetime.now().isoformat() } slitlet_label = "slitlet" + str(islitlet).zfill(2) if slitlet_label not in bounddict['contents']: bounddict['contents'][slitlet_label] = {} bounddict['contents'][slitlet_label][date_obs] = tmp_dict if debugplot < 10: print("")
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: display arrangement of EMIR CSU bars') # positional arguments parser.add_argument("filename", help="FITS files (wildcards accepted) or single TXT " "file with CSU configuration from OSP", type=argparse.FileType('rb'), nargs='+') # optional arguments parser.add_argument("--grism", help="Grism (J, H, K, LR)", choices=["J", "H", "K", "LR"]) parser.add_argument("--filter", help="Filter (J, H, Ksp, YJ, HK)", choices=["J", "H", "Ksp", "YJ", "HK"]) parser.add_argument("--n_clusters", help="Display histogram of slitlet widths", default=0, type=int) parser.add_argument("--bbox", help="Bounding box tuple xmin,xmax,ymin,ymax " "indicating plot limits") parser.add_argument("--adjust", help="Adjust X range according to minimum and maximum" " csu_bar_left and csu_bar_right (note that this " "option is overriden by --bbox", action='store_true') parser.add_argument("--geometry", help="Tuple x,y,dx,dy indicating window geometry", default="0,0,640,480") parser.add_argument("--debugplot", help="Integer indicating plotting & debugging options" " (default=12)", default=12, type=int, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args) if args.echo: print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n') # geometry if args.geometry is None: geometry = None else: tmp_str = args.geometry.split(",") x_geom = int(tmp_str[0]) y_geom = int(tmp_str[1]) dx_geom = int(tmp_str[2]) dy_geom = int(tmp_str[3]) geometry = x_geom, y_geom, dx_geom, dy_geom # read bounding box if args.bbox is None: bbox = None else: str_bbox = args.bbox.split(",") xmin, xmax, ymin, ymax = [int(str_bbox[i]) for i in range(4)] bbox = xmin, xmax, ymin, ymax list_fits_file_objects = [] # if input file is a single txt file, assume it is a list of FITS files if len(args.filename) == 1: list_fits_file_objects = [args.filename[0]] else: list_fits_file_objects = args.filename # total number of files to be examined nfiles = len(list_fits_file_objects) # declare arrays to store CSU values csu_bar_left = np.zeros((nfiles, EMIR_NBARS)) csu_bar_right = np.zeros((nfiles, EMIR_NBARS)) csu_bar_slit_center = np.zeros((nfiles, EMIR_NBARS)) csu_bar_slit_width = np.zeros((nfiles, EMIR_NBARS)) # display CSU bar arrangement for ifile, fileobj in enumerate(list_fits_file_objects): print("\nFile " + str(ifile + 1) + "/" + str(nfiles) + ": " + fileobj.name) csu_bar_left[ifile, :], csu_bar_right[ifile, :], \ csu_bar_slit_center[ifile, :], csu_bar_slit_width[ifile, :] = \ display_slitlet_arrangement( fileobj, grism=args.grism, spfilter=args.filter, bbox=bbox, adjust=args.adjust, geometry=geometry, debugplot=args.debugplot ) if args.n_clusters >= 2: display_slitlet_histogram(csu_bar_slit_width[ifile, :], n_clusters=args.n_clusters, geometry=geometry, debugplot=args.debugplot) # print summary of comparison between files if nfiles > 1: std_csu_bar_left = np.zeros(EMIR_NBARS) std_csu_bar_right = np.zeros(EMIR_NBARS) std_csu_bar_slit_center = np.zeros(EMIR_NBARS) std_csu_bar_slit_width = np.zeros(EMIR_NBARS) if args.debugplot >= 10: print("\n STANDARD DEVIATION BETWEEN IMAGES") print("slit left right center width") print("==== ======= ======= ======= =====") for i in range(EMIR_NBARS): ibar = i + 1 std_csu_bar_left[i] = np.std(csu_bar_left[:, i]) std_csu_bar_right[i] = np.std(csu_bar_right[:, i]) std_csu_bar_slit_center[i] = np.std(csu_bar_slit_center[:, i]) std_csu_bar_slit_width[i] = np.std(csu_bar_slit_width[:, i]) print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f}".format( ibar, std_csu_bar_left[i], std_csu_bar_right[i], std_csu_bar_slit_center[i], std_csu_bar_slit_width[i])) print("==== ======= ======= ======= =====") print("MIN: {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f}".format( std_csu_bar_left.min(), std_csu_bar_right.min(), std_csu_bar_slit_center.min(), std_csu_bar_slit_width.min())) print("MAX: {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f}".format( std_csu_bar_left.max(), std_csu_bar_right.max(), std_csu_bar_slit_center.max(), std_csu_bar_slit_width.max())) print("==== ======= ======= ======= =====") print("Total number of files examined:", nfiles) # stop program execution if len(list_fits_file_objects) > 1: pause_debugplot(12, optional_prompt="Press RETURN to STOP")
def refine_rectwv_coeff(input_image, rectwv_coeff, refine_wavecalib_mode, minimum_slitlet_width_mm, maximum_slitlet_width_mm, save_intermediate_results=False, debugplot=0): """Refine RectWaveCoeff object using a catalogue of lines One and only one among refine_with_oh_lines_mode and refine_with_arc_lines must be different from zero. Parameters ---------- input_image : HDUList object Input 2D image. rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for the particular CSU configuration. refine_wavecalib_mode : int Integer, indicating the type of refinement: 0 : no refinement 1 : apply the same global offset to all the slitlets (using ARC lines) 2 : apply individual offset to each slitlet (using ARC lines) 11 : apply the same global offset to all the slitlets (using OH lines) 12 : apply individual offset to each slitlet (using OH lines) minimum_slitlet_width_mm : float Minimum slitlet width (mm) for a valid slitlet. maximum_slitlet_width_mm : float Maximum slitlet width (mm) for a valid slitlet. save_intermediate_results : bool If True, save plots in PDF files debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot. Returns ------- refined_rectwv_coeff : RectWaveCoeff instance Refined rectification and wavelength calibration coefficients for the particular CSU configuration. expected_cat_image : HDUList object Output 2D image with the expected catalogued lines. """ logger = logging.getLogger(__name__) if save_intermediate_results: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages('crosscorrelation.pdf') else: pdf = None # image header main_header = input_image[0].header filter_name = main_header['filter'] grism_name = main_header['grism'] # protections if refine_wavecalib_mode not in [1, 2, 11, 12]: logger.error('Wavelength calibration refinemente mode={}'.format( refine_wavecalib_mode)) raise ValueError("Invalid wavelength calibration refinement mode") # read tabulated lines if refine_wavecalib_mode in [1, 2]: # ARC lines if grism_name == 'LR': catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat' else: catlines_file = 'lines_argon_neon_xenon_empirical.dat' dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file) arc_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(arc_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = catlines[:, 0] catlines_all_flux = catlines[:, 1] mode = refine_wavecalib_mode elif refine_wavecalib_mode in [11, 12]: # OH lines dumdata = pkgutil.get_data('emirdrp.instrument.configs', 'Oliva_etal_2013.dat') oh_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(oh_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0])) catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2])) mode = refine_wavecalib_mode - 10 else: raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode)) # initialize output refined_rectwv_coeff = deepcopy(rectwv_coeff) logger.info('Computing median spectrum') # compute median spectrum and normalize it sp_median = median_slitlets_rectified( input_image, mode=2, minimum_slitlet_width_mm=minimum_slitlet_width_mm, maximum_slitlet_width_mm=maximum_slitlet_width_mm)[0].data sp_median /= sp_median.max() # determine minimum and maximum useful wavelength jmin, jmax = find_pix_borders(sp_median, 0) naxis1 = main_header['naxis1'] naxis2 = main_header['naxis2'] crpix1 = main_header['crpix1'] crval1 = main_header['crval1'] cdelt1 = main_header['cdelt1'] xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1 if grism_name == 'LR': wv_parameters = set_wv_parameters(filter_name, grism_name) wave_min = wv_parameters['wvmin_useful'] wave_max = wv_parameters['wvmax_useful'] else: wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1 wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1 logger.info('Setting wave_min to {}'.format(wave_min)) logger.info('Setting wave_max to {}'.format(wave_max)) # extract subset of catalogue lines within current wavelength range lok1 = catlines_all_wave >= wave_min lok2 = catlines_all_wave <= wave_max catlines_reference_wave = catlines_all_wave[lok1 * lok2] catlines_reference_flux = catlines_all_flux[lok1 * lok2] catlines_reference_flux /= catlines_reference_flux.max() # estimate sigma to broaden catalogue lines csu_config = CsuConfiguration.define_from_header(main_header) # segregate slitlets list_useful_slitlets = csu_config.widths_in_range_mm( minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm) list_not_useful_slitlets = [ i for i in list(range(1, EMIR_NBARS + 1)) if i not in list_useful_slitlets ] logger.info('list of useful slitlets: {}'.format(list_useful_slitlets)) logger.info( 'list of not useful slitlets: {}'.format(list_not_useful_slitlets)) tempwidths = np.array([ csu_config.csu_bar_slit_width(islitlet) for islitlet in list_useful_slitlets ]) widths_summary = summary(tempwidths) logger.info('Statistics of useful slitlet widths (mm):') logger.info('- npoints....: {0:d}'.format(widths_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(widths_summary['median'])) logger.info('- std........: {0:7.3f}'.format(widths_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std'])) # empirical transformation of slit width (mm) to pixels sigma_broadening = cdelt1 * widths_summary['median'] # convolve location of catalogue lines to generate expected spectrum xwave_reference, sp_reference = convolve_comb_lines( catlines_reference_wave, catlines_reference_flux, sigma_broadening, crpix1, crval1, cdelt1, naxis1) sp_reference /= sp_reference.max() # generate image2d with expected lines image2d_expected_lines = np.tile(sp_reference, (naxis2, 1)) hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header) expected_cat_image = fits.HDUList([hdu]) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xwave, sp_median, 'C1-', xlabel='Wavelength (Angstroms, in vacuum)', ylabel='Normalized number of counts', title='Median spectrum', label='observed spectrum', show=False) # overplot reference catalogue lines ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-', markerfmt=' ', basefmt='C4-', label='tabulated lines') # overplot convolved reference lines ax.plot(xwave_reference, sp_reference, 'C0-', label='expected spectrum') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) # compute baseline signal in sp_median baseline = np.percentile(sp_median[sp_median > 0], q=10) if (abs(debugplot) % 10 != 0) or (pdf is not None): fig = plt.figure() ax = fig.add_subplot(111) ax.hist(sp_median, bins=1000, log=True) ax.set_xlabel('Normalized number of counts') ax.set_ylabel('Number of pixels') ax.set_title('Median spectrum') ax.axvline(float(baseline), linestyle='--', color='grey') if pdf is not None: pdf.savefig() else: geometry = (0, 0, 640, 480) set_window_geometry(geometry) plt.show() # subtract baseline to sp_median (only pixels with signal above zero) lok = np.where(sp_median > 0) sp_median[lok] -= baseline # compute global offset through periodic correlation logger.info('Computing global offset') global_offset, fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=sp_median, fminmax=None, naround_zero=50, plottitle='Median spectrum (cross-correlation)', pdf=pdf, debugplot=debugplot) logger.info('Global offset: {} pixels'.format(-global_offset)) missing_slitlets = rectwv_coeff.missing_slitlets if mode == 1: # apply computed offset to obtain refined_rectwv_coeff_global for islitlet in range(1, EMIR_NBARS + 1): if islitlet not in missing_slitlets: i = islitlet - 1 dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= global_offset * cdelt1 elif mode == 2: # compute individual offset for each slitlet logger.info('Computing individual offsets') median_55sp = median_slitlets_rectified(input_image, mode=1) offset_array = np.zeros(EMIR_NBARS) xplot = [] yplot = [] xplot_skipped = [] yplot_skipped = [] cout = '0' for islitlet in range(1, EMIR_NBARS + 1): if islitlet in list_useful_slitlets: i = islitlet - 1 sp_median = median_55sp[0].data[i, :] lok = np.where(sp_median > 0) baseline = np.percentile(sp_median[lok], q=10) sp_median[lok] -= baseline sp_median /= sp_median.max() offset_array[i], fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=median_55sp[0].data[i, :], fminmax=None, naround_zero=50, plottitle='slitlet #{0} (cross-correlation)'.format( islitlet), pdf=pdf, debugplot=debugplot) dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= offset_array[i] * cdelt1 xplot.append(islitlet) yplot.append(-offset_array[i]) # second correction wpoly_coeff_refined = check_wlcalib_sp( sp=median_55sp[0].data[i, :], crpix1=crpix1, crval1=crval1 - offset_array[i] * cdelt1, cdelt1=cdelt1, wv_master=catlines_reference_wave, coeff_ini=dumdict['wpoly_coeff'], naxis1_ini=EMIR_NAXIS1, title='slitlet #{0} (after applying offset)'.format( islitlet), ylogscale=False, pdf=pdf, debugplot=debugplot) dumdict['wpoly_coeff'] = wpoly_coeff_refined cout += '.' else: xplot_skipped.append(islitlet) yplot_skipped.append(0) cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) logger.info(cout) # show offsets with opposite sign stat_summary = summary(np.array(yplot)) logger.info('Statistics of individual slitlet offsets (pixels):') logger.info('- npoints....: {0:d}'.format(stat_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(stat_summary['median'])) logger.info('- std........: {0:7.3f}'.format(stat_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format( stat_summary['robust_std'])) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xplot, yplot, linestyle='', marker='o', color='C0', xlabel='slitlet number', ylabel='-offset (pixels) = offset to be applied', title='cross-correlation result', show=False, **{'label': 'individual slitlets'}) if len(xplot_skipped) > 0: ax.plot(xplot_skipped, yplot_skipped, 'mx') ax.axhline(-global_offset, linestyle='--', color='C1', label='global offset') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) else: raise ValueError('Unexpected mode={}'.format(mode)) # close output PDF file if pdf is not None: pdf.close() # return result return refined_rectwv_coeff, expected_cat_image
def main(args=None): # parse command-line options parser = argparse.ArgumentParser() # positional arguments parser.add_argument("fitsfile", help="FITS file name to be displayed", type=argparse.FileType('rb')) parser.add_argument("--bounddict", required=True, help="bounddict file name", type=argparse.FileType('rt')) parser.add_argument("--tuple_slit_numbers", required=True, help="Tuple n1[,n2[,step]] to define slitlet numbers") # optional arguments parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args() if args.echo: print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n') # read slitlet numbers to be computed tmp_str = args.tuple_slit_numbers.split(",") if len(tmp_str) == 3: if int(tmp_str[0]) < 1: raise ValueError("Invalid slitlet number < 1") if int(tmp_str[1]) > EMIR_NBARS: raise ValueError("Invalid slitlet number > EMIR_NBARS") list_slitlets = range(int(tmp_str[0]), int(tmp_str[1])+1, int(tmp_str[2])) elif len(tmp_str) == 2: if int(tmp_str[0]) < 1: raise ValueError("Invalid slitlet number < 1") if int(tmp_str[1]) > EMIR_NBARS: raise ValueError("Invalid slitlet number > EMIR_NBARS") list_slitlets = range(int(tmp_str[0]), int(tmp_str[1])+1, 1) elif len(tmp_str) == 1: if int(tmp_str[0]) < 1: raise ValueError("Invalid slitlet number < 1") if int(tmp_str[0]) > EMIR_NBARS: raise ValueError("Invalid slitlet number > EMIR_NBARS") list_slitlets = [int(tmp_str[0])] else: raise ValueError("Invalid tuple for slitlet numbers") # read input FITS file hdulist = fits.open(args.fitsfile.name) image_header = hdulist[0].header image2d = hdulist[0].data hdulist.close() naxis1 = image_header['naxis1'] naxis2 = image_header['naxis2'] if image2d.shape != (naxis2, naxis1): raise ValueError("Unexpected error with NAXIS1, NAXIS2") # remove path from fitsfile sfitsfile = os.path.basename(args.fitsfile.name) # check that the FITS file has been obtained with EMIR instrument = image_header['instrume'] if instrument != 'EMIR': raise ValueError("INSTRUME keyword is not 'EMIR'!") # read GRISM, FILTER and ROTANG from FITS header grism = image_header['grism'] spfilter = image_header['filter'] rotang = image_header['rotang'] # display full image ax = ximshow(image2d=image2d, title=sfitsfile + "\ngrism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), image_bbox=(1, naxis1, 1, naxis2), show=False) # overplot boundaries for each slitlet for slitlet_number in list_slitlets: pol_lower_boundary, pol_upper_boundary, \ xmin_lower, xmax_lower, xmin_upper, xmax_upper, \ csu_bar_slit_center = \ get_boundaries(args.bounddict, slitlet_number) if (pol_lower_boundary is not None) and \ (pol_upper_boundary is not None): xp = np.linspace(start=xmin_lower, stop=xmax_lower, num=1000) yp = pol_lower_boundary(xp) ax.plot(xp, yp, 'g-') xp = np.linspace(start=xmin_upper, stop=xmax_upper, num=1000) yp = pol_upper_boundary(xp) ax.plot(xp, yp, 'b-') # slitlet label yc_lower = pol_lower_boundary(EMIR_NAXIS1 / 2 + 0.5) yc_upper = pol_upper_boundary(EMIR_NAXIS1 / 2 + 0.5) tmpcolor = ['r', 'b'][slitlet_number % 2] xcsu = EMIR_NAXIS1 * csu_bar_slit_center/341.5 ax.text(xcsu, (yc_lower + yc_upper) / 2, str(slitlet_number), fontsize=10, va='center', ha='center', bbox=dict(boxstyle="round,pad=0.1", fc="white", ec="grey"), color=tmpcolor, fontweight='bold', backgroundcolor='white') # show plot pause_debugplot(12, pltshow=True)
def rectwv_coeff_from_arc_image(reduced_image, bound_param, lines_catalog, args_nbrightlines=None, args_ymargin_bb=2, args_remove_sp_background=True, args_times_sigma_threshold=10, args_order_fmap=2, args_sigma_gaussian_filtering=2, args_margin_npix=50, args_poldeg_initial=3, args_poldeg_refined=5, args_interactive=False, args_threshold_wv=0, args_ylogscale=False, args_pdf=None, args_geometry=(0,0,640,480), debugplot=0): """Evaluate rect.+wavecal. coefficients from arc image Parameters ---------- reduced_image : HDUList object Image with preliminary basic reduction: bpm, bias, dark and flatfield. bound_param : RefinedBoundaryModelParam instance Refined boundary model. lines_catalog : Numpy array 2D numpy array with the contents of the master file with the expected arc line wavelengths. args_nbrightlines : int TBD args_ymargin_bb : int TBD args_remove_sp_background : bool TBD args_times_sigma_threshold : float TBD args_order_fmap : int TBD args_sigma_gaussian_filtering : float TBD args_margin_npix : int TBD args_poldeg_initial : int TBD args_poldeg_refined : int TBD args_interactive : bool TBD args_threshold_wv : float TBD args_ylogscale : bool TBD args_pdf : TBD args_geometry : TBD debugplot : int Debugging level for messages and plots. For details see 'numina.array.display.pause_debugplot.py'. Returns ------- rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for the particular CSU configuration of the input arc image. reduced_55sp : HDUList object Image with 55 spectra corresponding to the median spectrum for each slitlet, employed to derived the wavelength calibration polynomial. """ logger = logging.getLogger(__name__) # protections if args_interactive and args_pdf is not None: logger.error('--interactive and --pdf are incompatible options') raise ValueError('--interactive and --pdf are incompatible options') # header and data array header = reduced_image[0].header image2d = reduced_image[0].data # check grism and filter filter_name = header['filter'] logger.info('Filter: ' + filter_name) if filter_name != bound_param.tags['filter']: raise ValueError('Filter name does not match!') grism_name = header['grism'] logger.info('Grism: ' + grism_name) if grism_name != bound_param.tags['grism']: raise ValueError('Grism name does not match!') # read the CSU configuration from the image header csu_conf = CsuConfiguration.define_from_header(header) logger.debug(csu_conf) # read the DTU configuration from the image header dtu_conf = DtuConfiguration.define_from_header(header) logger.debug(dtu_conf) # set boundary parameters parmodel = bound_param.meta_info['parmodel'] params = bound_params_from_dict(bound_param.__getstate__()) if abs(debugplot) >= 10: print('-' * 83) print('* FITTED BOUND PARAMETERS') params.pretty_print() pause_debugplot(debugplot) # determine parameters according to grism+filter combination wv_parameters = set_wv_parameters(filter_name, grism_name) islitlet_min = wv_parameters['islitlet_min'] islitlet_max = wv_parameters['islitlet_max'] if args_nbrightlines is None: nbrightlines = wv_parameters['nbrightlines'] else: nbrightlines = [int(idum) for idum in args_nbrightlines.split(',')] poly_crval1_linear = wv_parameters['poly_crval1_linear'] poly_cdelt1_linear = wv_parameters['poly_cdelt1_linear'] wvmin_expected = wv_parameters['wvmin_expected'] wvmax_expected = wv_parameters['wvmax_expected'] wvmin_useful = wv_parameters['wvmin_useful'] wvmax_useful = wv_parameters['wvmax_useful'] # list of slitlets to be computed logger.info('list_slitlets: [' + str(islitlet_min) + ',... ' + str(islitlet_max) + ']') # read master arc line wavelengths (only brightest lines) wv_master = read_wv_master_from_array( master_table=lines_catalog, lines='brightest', debugplot=debugplot ) # read master arc line wavelengths (whole data set) wv_master_all = read_wv_master_from_array( master_table=lines_catalog, lines='all', debugplot=debugplot ) # check that the arc lines in the master file are properly sorted # in ascending order for i in range(len(wv_master_all) - 1): if wv_master_all[i] >= wv_master_all[i + 1]: logger.error('>>> wavelengths: ' + str(wv_master_all[i]) + ' ' + str(wv_master_all[i+1])) raise ValueError('Arc lines are not sorted in master file') # --- image2d_55sp = np.zeros((EMIR_NBARS, EMIR_NAXIS1)) # compute rectification transformation and wavelength calibration # polynomials measured_slitlets = [] cout = '0' for islitlet in range(1, EMIR_NBARS + 1): if islitlet_min <= islitlet <= islitlet_max: # define Slitlet2dArc object slt = Slitlet2dArc( islitlet=islitlet, csu_conf=csu_conf, ymargin_bb=args_ymargin_bb, params=params, parmodel=parmodel, debugplot=debugplot ) # extract 2D image corresponding to the selected slitlet, clipping # the image beyond the unrectified slitlet (in order to isolate # the arc lines of the current slitlet; otherwise there are # problems with arc lines from neighbour slitlets) image2d_tmp = select_unrectified_slitlet( image2d=image2d, islitlet=islitlet, csu_bar_slit_center=csu_conf.csu_bar_slit_center(islitlet), params=params, parmodel=parmodel, maskonly=False ) slitlet2d = slt.extract_slitlet2d(image2d_tmp) # subtract smooth background computed as follows: # - median collapsed spectrum of the whole slitlet2d # - independent median filtering of the previous spectrum in the # two halves in the spectral direction if args_remove_sp_background: spmedian = np.median(slitlet2d, axis=0) naxis1_tmp = spmedian.shape[0] jmidpoint = naxis1_tmp // 2 sp1 = medfilt(spmedian[:jmidpoint], [201]) sp2 = medfilt(spmedian[jmidpoint:], [201]) spbackground = np.concatenate((sp1, sp2)) slitlet2d -= spbackground # locate unknown arc lines slt.locate_unknown_arc_lines( slitlet2d=slitlet2d, times_sigma_threshold=args_times_sigma_threshold) # continue working with current slitlet only if arc lines have # been detected if slt.list_arc_lines is not None: # compute intersections between spectrum trails and arc lines slt.xy_spectrail_arc_intersections(slitlet2d=slitlet2d) # compute rectification transformation slt.estimate_tt_to_rectify(order=args_order_fmap, slitlet2d=slitlet2d) # rectify image slitlet2d_rect = slt.rectify(slitlet2d, resampling=2, transformation=1) # median spectrum and line peaks from rectified image sp_median, fxpeaks = slt.median_spectrum_from_rectified_image( slitlet2d_rect, sigma_gaussian_filtering=args_sigma_gaussian_filtering, nwinwidth_initial=5, nwinwidth_refined=5, times_sigma_threshold=5, npix_avoid_border=6, nbrightlines=nbrightlines ) image2d_55sp[islitlet - 1, :] = sp_median # determine expected wavelength limits prior to the wavelength # calibration csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet) crval1_linear = poly_crval1_linear(csu_bar_slit_center) cdelt1_linear = poly_cdelt1_linear(csu_bar_slit_center) expected_wvmin = crval1_linear - \ args_margin_npix * cdelt1_linear naxis1_linear = sp_median.shape[0] crvaln_linear = crval1_linear + \ (naxis1_linear - 1) * cdelt1_linear expected_wvmax = crvaln_linear + \ args_margin_npix * cdelt1_linear # override previous estimates when necessary if wvmin_expected is not None: expected_wvmin = wvmin_expected if wvmax_expected is not None: expected_wvmax = wvmax_expected # clip initial master arc line list with bright lines to # the expected wavelength range lok1 = expected_wvmin <= wv_master lok2 = wv_master <= expected_wvmax lok = lok1 * lok2 wv_master_eff = wv_master[lok] # perform initial wavelength calibration solution_wv = wvcal_spectrum( sp=sp_median, fxpeaks=fxpeaks, poly_degree_wfit=args_poldeg_initial, wv_master=wv_master_eff, wv_ini_search=expected_wvmin, wv_end_search=expected_wvmax, wvmin_useful=wvmin_useful, wvmax_useful=wvmax_useful, geometry=args_geometry, debugplot=slt.debugplot ) # store initial wavelength calibration polynomial in current # slitlet instance slt.wpoly = np.polynomial.Polynomial(solution_wv.coeff) pause_debugplot(debugplot) # clip initial master arc line list with all the lines to # the expected wavelength range lok1 = expected_wvmin <= wv_master_all lok2 = wv_master_all <= expected_wvmax lok = lok1 * lok2 wv_master_all_eff = wv_master_all[lok] # clip master arc line list to useful region if wvmin_useful is not None: lok = wvmin_useful <= wv_master_all_eff wv_master_all_eff = wv_master_all_eff[lok] if wvmax_useful is not None: lok = wv_master_all_eff <= wvmax_useful wv_master_all_eff = wv_master_all_eff[lok] # refine wavelength calibration if args_poldeg_refined > 0: plottitle = '[slitlet#{}, refined]'.format(islitlet) poly_refined, yres_summary = refine_arccalibration( sp=sp_median, poly_initial=slt.wpoly, wv_master=wv_master_all_eff, poldeg=args_poldeg_refined, ntimes_match_wv=1, interactive=args_interactive, threshold=args_threshold_wv, plottitle=plottitle, ylogscale=args_ylogscale, geometry=args_geometry, pdf=args_pdf, debugplot=slt.debugplot ) # store refined wavelength calibration polynomial in # current slitlet instance slt.wpoly = poly_refined # compute approximate linear values for CRVAL1 and CDELT1 naxis1_linear = sp_median.shape[0] crmin1_linear = slt.wpoly(1) crmax1_linear = slt.wpoly(naxis1_linear) slt.crval1_linear = crmin1_linear slt.cdelt1_linear = \ (crmax1_linear - crmin1_linear) / (naxis1_linear - 1) # check that the trimming of wv_master and wv_master_all has # preserved the wavelength range [crmin1_linear, crmax1_linear] if crmin1_linear < expected_wvmin: logger.warning(">>> islitlet: " +str(islitlet)) logger.warning("expected_wvmin: " + str(expected_wvmin)) logger.warning("crmin1_linear.: " + str(crmin1_linear)) logger.warning("WARNING: Unexpected crmin1_linear < " "expected_wvmin") if crmax1_linear > expected_wvmax: logger.warning(">>> islitlet: " +str(islitlet)) logger.warning("expected_wvmax: " + str(expected_wvmax)) logger.warning("crmax1_linear.: " + str(crmax1_linear)) logger.warning("WARNING: Unexpected crmax1_linear > " "expected_wvmax") cout += '.' else: cout += 'x' if islitlet % 10 == 0: if cout != 'x': cout = str(islitlet // 10) if debugplot != 0: pause_debugplot(debugplot) else: # define Slitlet2dArc object slt = Slitlet2dArc( islitlet=islitlet, csu_conf=csu_conf, ymargin_bb=args_ymargin_bb, params=None, parmodel=None, debugplot=debugplot ) cout += 'i' # store current slitlet in list of measured slitlets measured_slitlets.append(slt) logger.info(cout) # --- # generate FITS file structure with 55 spectra corresponding to the # median spectrum for each slitlet reduced_55sp = fits.PrimaryHDU(data=image2d_55sp) reduced_55sp.header['crpix1'] = (0.0, 'reference pixel') reduced_55sp.header['crval1'] = (0.0, 'central value at crpix2') reduced_55sp.header['cdelt1'] = (1.0, 'increment') reduced_55sp.header['ctype1'] = 'PIXEL' reduced_55sp.header['cunit1'] = ('Pixel', 'units along axis2') reduced_55sp.header['crpix2'] = (0.0, 'reference pixel') reduced_55sp.header['crval2'] = (0.0, 'central value at crpix2') reduced_55sp.header['cdelt2'] = (1.0, 'increment') reduced_55sp.header['ctype2'] = 'PIXEL' reduced_55sp.header['cunit2'] = ('Pixel', 'units along axis2') # --- # Generate structure to store intermediate results outdict = {} outdict['instrument'] = 'EMIR' outdict['meta_info'] = {} outdict['meta_info']['creation_date'] = datetime.now().isoformat() outdict['meta_info']['description'] = \ 'computation of rectification and wavelength calibration polynomial ' \ 'coefficients for a particular CSU configuration' outdict['meta_info']['recipe_name'] = 'undefined' outdict['meta_info']['origin'] = {} outdict['meta_info']['origin']['bound_param_uuid'] = \ bound_param.uuid outdict['meta_info']['origin']['arc_image_uuid'] = 'undefined' outdict['tags'] = {} outdict['tags']['grism'] = grism_name outdict['tags']['filter'] = filter_name outdict['tags']['islitlet_min'] = islitlet_min outdict['tags']['islitlet_max'] = islitlet_max outdict['dtu_configuration'] = dtu_conf.outdict() outdict['uuid'] = str(uuid4()) outdict['contents'] = {} missing_slitlets = [] for slt in measured_slitlets: islitlet = slt.islitlet if islitlet_min <= islitlet <= islitlet_max: # avoid error when creating a python list of coefficients from # numpy polynomials when the polynomials do not exist (note that # the JSON format doesn't handle numpy arrays and such arrays must # be transformed into native python lists) if slt.wpoly is None: wpoly_coeff = None else: wpoly_coeff = slt.wpoly.coef.tolist() if slt.wpoly_longslit_model is None: wpoly_coeff_longslit_model = None else: wpoly_coeff_longslit_model = \ slt.wpoly_longslit_model.coef.tolist() # avoid similar error when creating a python list of coefficients # when the numpy array does not exist; note that this problem # does not happen with tt?_aij_longslit_model and # tt?_bij_longslit_model because the latter have already been # created as native python lists if slt.ttd_aij is None: ttd_aij = None else: ttd_aij = slt.ttd_aij.tolist() if slt.ttd_bij is None: ttd_bij = None else: ttd_bij = slt.ttd_bij.tolist() if slt.tti_aij is None: tti_aij = None else: tti_aij = slt.tti_aij.tolist() if slt.tti_bij is None: tti_bij = None else: tti_bij = slt.tti_bij.tolist() # creating temporary dictionary with the information corresponding # to the current slitlett that will be saved in the JSON file tmp_dict = { 'csu_bar_left': slt.csu_bar_left, 'csu_bar_right': slt.csu_bar_right, 'csu_bar_slit_center': slt.csu_bar_slit_center, 'csu_bar_slit_width': slt.csu_bar_slit_width, 'x0_reference': slt.x0_reference, 'y0_reference_lower': slt.y0_reference_lower, 'y0_reference_middle': slt.y0_reference_middle, 'y0_reference_upper': slt.y0_reference_upper, 'y0_reference_lower_expected': slt.y0_reference_lower_expected, 'y0_reference_middle_expected': slt.y0_reference_middle_expected, 'y0_reference_upper_expected': slt.y0_reference_upper_expected, 'y0_frontier_lower': slt.y0_frontier_lower, 'y0_frontier_upper': slt.y0_frontier_upper, 'y0_frontier_lower_expected': slt.y0_frontier_lower_expected, 'y0_frontier_upper_expected': slt.y0_frontier_upper_expected, 'corr_yrect_a': slt.corr_yrect_a, 'corr_yrect_b': slt.corr_yrect_b, 'min_row_rectified': slt.min_row_rectified, 'max_row_rectified': slt.max_row_rectified, 'ymargin_bb': slt.ymargin_bb, 'bb_nc1_orig': slt.bb_nc1_orig, 'bb_nc2_orig': slt.bb_nc2_orig, 'bb_ns1_orig': slt.bb_ns1_orig, 'bb_ns2_orig': slt.bb_ns2_orig, 'spectrail': { 'poly_coef_lower': slt.list_spectrails[ slt.i_lower_spectrail].poly_funct.coef.tolist(), 'poly_coef_middle': slt.list_spectrails[ slt.i_middle_spectrail].poly_funct.coef.tolist(), 'poly_coef_upper': slt.list_spectrails[ slt.i_upper_spectrail].poly_funct.coef.tolist(), }, 'frontier': { 'poly_coef_lower': slt.list_frontiers[0].poly_funct.coef.tolist(), 'poly_coef_upper': slt.list_frontiers[1].poly_funct.coef.tolist(), }, 'ttd_order': slt.ttd_order, 'ttd_aij': ttd_aij, 'ttd_bij': ttd_bij, 'tti_aij': tti_aij, 'tti_bij': tti_bij, 'ttd_order_longslit_model': slt.ttd_order_longslit_model, 'ttd_aij_longslit_model': slt.ttd_aij_longslit_model, 'ttd_bij_longslit_model': slt.ttd_bij_longslit_model, 'tti_aij_longslit_model': slt.tti_aij_longslit_model, 'tti_bij_longslit_model': slt.tti_bij_longslit_model, 'wpoly_coeff': wpoly_coeff, 'wpoly_coeff_longslit_model': wpoly_coeff_longslit_model, 'crval1_linear': slt.crval1_linear, 'cdelt1_linear': slt.cdelt1_linear } else: missing_slitlets.append(islitlet) tmp_dict = { 'csu_bar_left': slt.csu_bar_left, 'csu_bar_right': slt.csu_bar_right, 'csu_bar_slit_center': slt.csu_bar_slit_center, 'csu_bar_slit_width': slt.csu_bar_slit_width, 'x0_reference': slt.x0_reference, 'y0_frontier_lower_expected': slt.y0_frontier_lower_expected, 'y0_frontier_upper_expected': slt.y0_frontier_upper_expected } slitlet_label = "slitlet" + str(islitlet).zfill(2) outdict['contents'][slitlet_label] = tmp_dict # --- # OBSOLETE ''' # save JSON file needed to compute the MOS model with open(args.out_json.name, 'w') as fstream: json.dump(outdict, fstream, indent=2, sort_keys=True) print('>>> Saving file ' + args.out_json.name) ''' # --- # Create object of type RectWaveCoeff with coefficients for # rectification and wavelength calibration rectwv_coeff = RectWaveCoeff(instrument='EMIR') rectwv_coeff.quality_control = numina.types.qc.QC.GOOD rectwv_coeff.tags['grism'] = grism_name rectwv_coeff.tags['filter'] = filter_name rectwv_coeff.meta_info['origin']['bound_param'] = \ 'uuid' + bound_param.uuid rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration'] rectwv_coeff.total_slitlets = EMIR_NBARS rectwv_coeff.missing_slitlets = missing_slitlets for i in range(EMIR_NBARS): islitlet = i + 1 dumdict = {'islitlet': islitlet} cslitlet = 'slitlet' + str(islitlet).zfill(2) if cslitlet in outdict['contents']: dumdict.update(outdict['contents'][cslitlet]) else: raise ValueError("Unexpected error") rectwv_coeff.contents.append(dumdict) # debugging __getstate__ and __setstate__ # rectwv_coeff.writeto(args.out_json.name) # print('>>> Saving file ' + args.out_json.name) # check_setstate_getstate(rectwv_coeff, args.out_json.name) logger.info('Generating RectWaveCoeff object with uuid=' + rectwv_coeff.uuid) return rectwv_coeff, reduced_55sp
def refine_rectwv_coeff(input_image, rectwv_coeff, refine_wavecalib_mode, minimum_slitlet_width_mm, maximum_slitlet_width_mm, save_intermediate_results=False, debugplot=0): """Refine RectWaveCoeff object using a catalogue of lines One and only one among refine_with_oh_lines_mode and refine_with_arc_lines must be different from zero. Parameters ---------- input_image : HDUList object Input 2D image. rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for the particular CSU configuration. refine_wavecalib_mode : int Integer, indicating the type of refinement: 0 : no refinement 1 : apply the same global offset to all the slitlets (using ARC lines) 2 : apply individual offset to each slitlet (using ARC lines) 11 : apply the same global offset to all the slitlets (using OH lines) 12 : apply individual offset to each slitlet (using OH lines) minimum_slitlet_width_mm : float Minimum slitlet width (mm) for a valid slitlet. maximum_slitlet_width_mm : float Maximum slitlet width (mm) for a valid slitlet. save_intermediate_results : bool If True, save plots in PDF files debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot. Returns ------- refined_rectwv_coeff : RectWaveCoeff instance Refined rectification and wavelength calibration coefficients for the particular CSU configuration. expected_cat_image : HDUList object Output 2D image with the expected catalogued lines. """ logger = logging.getLogger(__name__) if save_intermediate_results: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages('crosscorrelation.pdf') else: pdf = None # image header main_header = input_image[0].header filter_name = main_header['filter'] grism_name = main_header['grism'] # protections if refine_wavecalib_mode not in [1, 2, 11, 12]: logger.error('Wavelength calibration refinemente mode={}'. format( refine_wavecalib_mode )) raise ValueError("Invalid wavelength calibration refinement mode") # read tabulated lines if refine_wavecalib_mode in [1, 2]: # ARC lines if grism_name == 'LR': catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat' else: catlines_file = 'lines_argon_neon_xenon_empirical.dat' dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file) arc_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(arc_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = catlines[:, 0] catlines_all_flux = catlines[:, 1] mode = refine_wavecalib_mode elif refine_wavecalib_mode in [11, 12]: # OH lines dumdata = pkgutil.get_data( 'emirdrp.instrument.configs', 'Oliva_etal_2013.dat' ) oh_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(oh_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0])) catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2])) mode = refine_wavecalib_mode - 10 else: raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode)) # initialize output refined_rectwv_coeff = deepcopy(rectwv_coeff) logger.info('Computing median spectrum') # compute median spectrum and normalize it sp_median = median_slitlets_rectified( input_image, mode=2, minimum_slitlet_width_mm=minimum_slitlet_width_mm, maximum_slitlet_width_mm=maximum_slitlet_width_mm )[0].data sp_median /= sp_median.max() # determine minimum and maximum useful wavelength jmin, jmax = find_pix_borders(sp_median, 0) naxis1 = main_header['naxis1'] naxis2 = main_header['naxis2'] crpix1 = main_header['crpix1'] crval1 = main_header['crval1'] cdelt1 = main_header['cdelt1'] xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1 if grism_name == 'LR': wv_parameters = set_wv_parameters(filter_name, grism_name) wave_min = wv_parameters['wvmin_useful'] wave_max = wv_parameters['wvmax_useful'] else: wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1 wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1 logger.info('Setting wave_min to {}'.format(wave_min)) logger.info('Setting wave_max to {}'.format(wave_max)) # extract subset of catalogue lines within current wavelength range lok1 = catlines_all_wave >= wave_min lok2 = catlines_all_wave <= wave_max catlines_reference_wave = catlines_all_wave[lok1*lok2] catlines_reference_flux = catlines_all_flux[lok1*lok2] catlines_reference_flux /= catlines_reference_flux.max() # estimate sigma to broaden catalogue lines csu_config = CsuConfiguration.define_from_header(main_header) # segregate slitlets list_useful_slitlets = csu_config.widths_in_range_mm( minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm ) # remove missing slitlets if len(refined_rectwv_coeff.missing_slitlets) > 0: for iremove in refined_rectwv_coeff.missing_slitlets: if iremove in list_useful_slitlets: list_useful_slitlets.remove(iremove) list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1)) if i not in list_useful_slitlets] logger.info('list of useful slitlets: {}'.format( list_useful_slitlets)) logger.info('list of not useful slitlets: {}'.format( list_not_useful_slitlets)) tempwidths = np.array([csu_config.csu_bar_slit_width(islitlet) for islitlet in list_useful_slitlets]) widths_summary = summary(tempwidths) logger.info('Statistics of useful slitlet widths (mm):') logger.info('- npoints....: {0:d}'.format(widths_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(widths_summary['median'])) logger.info('- std........: {0:7.3f}'.format(widths_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std'])) # empirical transformation of slit width (mm) to pixels sigma_broadening = cdelt1 * widths_summary['median'] # convolve location of catalogue lines to generate expected spectrum xwave_reference, sp_reference = convolve_comb_lines( catlines_reference_wave, catlines_reference_flux, sigma_broadening, crpix1, crval1, cdelt1, naxis1 ) sp_reference /= sp_reference.max() # generate image2d with expected lines image2d_expected_lines = np.tile(sp_reference, (naxis2, 1)) hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header) expected_cat_image = fits.HDUList([hdu]) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xwave, sp_median, 'C1-', xlabel='Wavelength (Angstroms, in vacuum)', ylabel='Normalized number of counts', title='Median spectrum', label='observed spectrum', show=False) # overplot reference catalogue lines ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-', markerfmt=' ', basefmt='C4-', label='tabulated lines') # overplot convolved reference lines ax.plot(xwave_reference, sp_reference, 'C0-', label='expected spectrum') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) # compute baseline signal in sp_median baseline = np.percentile(sp_median[sp_median > 0], q=10) if (abs(debugplot) % 10 != 0) or (pdf is not None): fig = plt.figure() ax = fig.add_subplot(111) ax.hist(sp_median, bins=1000, log=True) ax.set_xlabel('Normalized number of counts') ax.set_ylabel('Number of pixels') ax.set_title('Median spectrum') ax.axvline(float(baseline), linestyle='--', color='grey') if pdf is not None: pdf.savefig() else: geometry = (0, 0, 640, 480) set_window_geometry(geometry) plt.show() # subtract baseline to sp_median (only pixels with signal above zero) lok = np.where(sp_median > 0) sp_median[lok] -= baseline # compute global offset through periodic correlation logger.info('Computing global offset') global_offset, fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=sp_median, fminmax=None, naround_zero=50, plottitle='Median spectrum (cross-correlation)', pdf=pdf, debugplot=debugplot ) logger.info('Global offset: {} pixels'.format(-global_offset)) missing_slitlets = rectwv_coeff.missing_slitlets if mode == 1: # apply computed offset to obtain refined_rectwv_coeff_global for islitlet in range(1, EMIR_NBARS + 1): if islitlet not in missing_slitlets: i = islitlet - 1 dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= global_offset*cdelt1 elif mode == 2: # compute individual offset for each slitlet logger.info('Computing individual offsets') median_55sp = median_slitlets_rectified(input_image, mode=1) offset_array = np.zeros(EMIR_NBARS) xplot = [] yplot = [] xplot_skipped = [] yplot_skipped = [] cout = '0' for islitlet in range(1, EMIR_NBARS + 1): if islitlet in list_useful_slitlets: i = islitlet - 1 sp_median = median_55sp[0].data[i, :] lok = np.where(sp_median > 0) if np.any(lok): baseline = np.percentile(sp_median[lok], q=10) sp_median[lok] -= baseline sp_median /= sp_median.max() offset_array[i], fpeak = periodic_corr1d( sp_reference=sp_reference, sp_offset=median_55sp[0].data[i, :], fminmax=None, naround_zero=50, plottitle='slitlet #{0} (cross-correlation)'.format( islitlet), pdf=pdf, debugplot=debugplot ) else: offset_array[i] = 0.0 dumdict = refined_rectwv_coeff.contents[i] dumdict['wpoly_coeff'][0] -= offset_array[i]*cdelt1 xplot.append(islitlet) yplot.append(-offset_array[i]) # second correction wpoly_coeff_refined = check_wlcalib_sp( sp=median_55sp[0].data[i, :], crpix1=crpix1, crval1=crval1-offset_array[i]*cdelt1, cdelt1=cdelt1, wv_master=catlines_reference_wave, coeff_ini=dumdict['wpoly_coeff'], naxis1_ini=EMIR_NAXIS1, title='slitlet #{0} (after applying offset)'.format( islitlet), ylogscale=False, pdf=pdf, debugplot=debugplot ) dumdict['wpoly_coeff'] = wpoly_coeff_refined cout += '.' else: xplot_skipped.append(islitlet) yplot_skipped.append(0) cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) logger.info(cout) # show offsets with opposite sign stat_summary = summary(np.array(yplot)) logger.info('Statistics of individual slitlet offsets (pixels):') logger.info('- npoints....: {0:d}'.format(stat_summary['npoints'])) logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean'])) logger.info('- median.....: {0:7.3f}'.format(stat_summary['median'])) logger.info('- std........: {0:7.3f}'.format(stat_summary['std'])) logger.info('- robust_std.: {0:7.3f}'.format(stat_summary[ 'robust_std'])) if (abs(debugplot) % 10 != 0) or (pdf is not None): ax = ximplotxy(xplot, yplot, linestyle='', marker='o', color='C0', xlabel='slitlet number', ylabel='-offset (pixels) = offset to be applied', title='cross-correlation result', show=False, **{'label': 'individual slitlets'}) if len(xplot_skipped) > 0: ax.plot(xplot_skipped, yplot_skipped, 'mx') ax.axhline(-global_offset, linestyle='--', color='C1', label='global offset') ax.legend() if pdf is not None: pdf.savefig() else: pause_debugplot(debugplot=debugplot, pltshow=True) else: raise ValueError('Unexpected mode={}'.format(mode)) # close output PDF file if pdf is not None: pdf.close() # return result return refined_rectwv_coeff, expected_cat_image
def rectwv_coeff_add_longslit_model(rectwv_coeff, geometry, debugplot=0): """Compute longslit_model coefficients for RectWaveCoeff object. Parameters ---------- rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for a particular CSU configuration corresponding to a longslit observation. geometry : TBD debugplot : int Debugging level for messages and plots. For details see 'numina.array.display.pause_debugplot.py'. Returns ------- rectwv_coeff : RectWaveCoeff instance Updated object with longslit_model coefficients computed. """ logger = logging.getLogger(__name__) # check grism and filter grism_name = rectwv_coeff.tags['grism'] logger.info('Grism: ' + grism_name) filter_name = rectwv_coeff.tags['filter'] logger.info('Filter: ' + filter_name) # list of slitlets to be computed list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: list_valid_islitlets.remove(idel) if abs(debugplot) >= 10: print('>>> valid slitlet numbers:\n', list_valid_islitlets) # --- # check that the CSU configuration corresponds to longslit csu_bar_slit_center_list = [] for islitlet in list_valid_islitlets: csu_bar_slit_center_list.append( rectwv_coeff.contents[islitlet - 1]['csu_bar_slit_center'] ) if abs(debugplot) >= 10: logger.debug('Checking csu_bar_slit_center values:') summary(np.array(csu_bar_slit_center_list), debug=True) pause_debugplot(debugplot) # --- # polynomial coefficients corresponding to the wavelength calibration # step 0: determine poldeg_refined, checking that it is the same for # all the slitlets poldeg_refined_list = [] for islitlet in list_valid_islitlets: poldeg_refined_list.append( len(rectwv_coeff.contents[islitlet - 1]['wpoly_coeff']) - 1 ) # remove duplicates poldeg_refined_list = list(set(poldeg_refined_list)) if len(poldeg_refined_list) != 1: raise ValueError('Unexpected different poldeg_refined found: ' + str(poldeg_refined_list)) poldeg_refined = poldeg_refined_list[0] # step 1: compute variation of each coefficient as a function of # y0_reference_middle of each slitlet list_poly = [] for i in range(poldeg_refined + 1): xp = [] yp = [] for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] wpoly_coeff = tmp_dict['wpoly_coeff'] if wpoly_coeff is not None: xp.append(tmp_dict['y0_reference_middle']) yp.append(wpoly_coeff[i]) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp), deg=2, times_sigma_reject=5, xlabel='y0_rectified', ylabel='coeff[' + str(i) + ']', title="Fit to refined wavelength calibration coefficients", geometry=geometry, debugplot=debugplot ) list_poly.append(poly) # step 2: use the variation of each polynomial coefficient with # y0_reference_middle to infer the expected wavelength calibration # polynomial for each rectifified slitlet for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] y0_reference_middle = tmp_dict['y0_reference_middle'] list_new_coeff = [] for i in range(poldeg_refined + 1): new_coeff = list_poly[i](y0_reference_middle) list_new_coeff.append(new_coeff) tmp_dict['wpoly_coeff_longslit_model'] = list_new_coeff # --- # rectification transformation coefficients aij and bij # step 0: determine order_fmap, checking that it is the same for # all the slitlets order_fmap_list = [] for islitlet in list_valid_islitlets: order_fmap_list.append( rectwv_coeff.contents[islitlet - 1]['ttd_order'] ) # remove duplicates order_fmap_list = list(set(order_fmap_list)) if len(order_fmap_list) != 1: raise ValueError('Unexpected different order_fmap found') order_fmap = order_fmap_list[0] # step 1: compute variation of each coefficient as a function of # y0_reference_middle of each slitlet list_poly_ttd_aij = [] list_poly_ttd_bij = [] list_poly_tti_aij = [] list_poly_tti_bij = [] ncoef_ttd = ncoef_fmap(order_fmap) for i in range(ncoef_ttd): xp = [] yp_ttd_aij = [] yp_ttd_bij = [] yp_tti_aij = [] yp_tti_bij = [] for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] ttd_aij = tmp_dict['ttd_aij'] ttd_bij = tmp_dict['ttd_bij'] tti_aij = tmp_dict['tti_aij'] tti_bij = tmp_dict['tti_bij'] if ttd_aij is not None: xp.append(tmp_dict['y0_reference_middle']) yp_ttd_aij.append(ttd_aij[i]) yp_ttd_bij.append(ttd_bij[i]) yp_tti_aij.append(tti_aij[i]) yp_tti_bij.append(tti_bij[i]) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_ttd_aij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='ttd_aij[' + str(i) + ']', geometry=geometry, debugplot=debugplot ) list_poly_ttd_aij.append(poly) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_ttd_bij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='ttd_bij[' + str(i) + ']', geometry=geometry, debugplot=debugplot ) list_poly_ttd_bij.append(poly) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_tti_aij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='tti_aij[' + str(i) + ']', geometry=geometry, debugplot=debugplot ) list_poly_tti_aij.append(poly) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_tti_bij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='tti_bij[' + str(i) + ']', geometry=geometry, debugplot=debugplot ) list_poly_tti_bij.append(poly) # step 2: use the variation of each coefficient with y0_reference_middle # to infer the expected rectification transformation for each slitlet for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] y0_reference_middle = tmp_dict['y0_reference_middle'] tmp_dict['ttd_order_longslit_model'] = order_fmap ttd_aij_longslit_model = [] ttd_bij_longslit_model = [] tti_aij_longslit_model = [] tti_bij_longslit_model = [] for i in range(ncoef_ttd): new_coeff = list_poly_ttd_aij[i](y0_reference_middle) ttd_aij_longslit_model.append(new_coeff) new_coeff = list_poly_ttd_bij[i](y0_reference_middle) ttd_bij_longslit_model.append(new_coeff) new_coeff = list_poly_tti_aij[i](y0_reference_middle) tti_aij_longslit_model.append(new_coeff) new_coeff = list_poly_tti_bij[i](y0_reference_middle) tti_bij_longslit_model.append(new_coeff) tmp_dict['ttd_aij_longslit_model'] = ttd_aij_longslit_model tmp_dict['ttd_bij_longslit_model'] = ttd_bij_longslit_model tmp_dict['tti_aij_longslit_model'] = tti_aij_longslit_model tmp_dict['tti_bij_longslit_model'] = tti_bij_longslit_model # --- # update uuid and meta_info in output JSON structure rectwv_coeff.uuid = str(uuid4()) rectwv_coeff.meta_info['creation_date'] = datetime.now().isoformat() # return updated object return rectwv_coeff
def compute_slitlet_boundaries( filename, grism, spfilter, list_slitlets, size_x_medfilt, size_y_savgol, times_sigma_threshold, bounddict, debugplot=0): """Compute slitlet boundaries using continuum lamp images. Parameters ---------- filename : string Input continumm lamp image. grism : string Grism name. It must be one in EMIR_VALID_GRISMS. spfilter : string Filter name. It must be one in EMIR_VALID_FILTERS. list_slitlets : list of integers Number of slitlets to be updated. size_x_medfilt : int Window in the X (spectral) direction, in pixels, to apply the 1d median filter in order to remove bad pixels. size_y_savgol : int Window in the Y (spatial) direction to be used when using the 1d Savitzky-Golay filter. times_sigma_threshold : float Times sigma to detect peaks in derivatives. bounddict : dictionary of dictionaries Structure to store the boundaries. debugplot : int Determines whether intermediate computations and/or plots are displayed. """ # read 2D image hdulist = fits.open(filename) image_header = hdulist[0].header image2d = hdulist[0].data naxis2, naxis1 = image2d.shape hdulist.close() if debugplot >= 10: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) # ToDo: replace this by application of cosmetic defect mask! for j in range(1024): image2d[1024, j] = (image2d[1023, j] + image2d[1025, j]) / 2 image2d[1023, j + 1024] = (image2d[1022, j + 1024] + image2d[1024, j + 1024]) / 2 # remove path from filename sfilename = os.path.basename(filename) # check that the FITS file has been obtained with EMIR instrument = image_header['instrume'] if instrument != 'EMIR': raise ValueError("INSTRUME keyword is not 'EMIR'!") # read CSU configuration from FITS header csu_config = CsuConfiguration.define_from_fits(filename) # read DTU configuration from FITS header dtu_config = DtuConfiguration.define_from_fits(filename) # read grism grism_in_header = image_header['grism'] if grism != grism_in_header: raise ValueError("GRISM keyword=" + grism_in_header + " is not the expected value=" + grism) # read filter spfilter_in_header = image_header['filter'] if spfilter != spfilter_in_header: raise ValueError("FILTER keyword=" + spfilter_in_header + " is not the expected value=" + spfilter) # read rotator position angle rotang = image_header['rotang'] # read date-obs date_obs = image_header['date-obs'] for islitlet in list_slitlets: if debugplot < 10: sys.stdout.write('.') sys.stdout.flush() sltlim = SlitletLimits(grism, spfilter, islitlet) # extract slitlet2d slitlet2d = extract_slitlet2d(image2d, sltlim) if debugplot % 10 != 0: ximshow(slitlet2d, title=sfilename + " [original]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), debugplot=debugplot) # apply 1d median filtering (along the spectral direction) # to remove bad pixels size_x = size_x_medfilt size_y = 1 slitlet2d_smooth = ndimage.filters.median_filter( slitlet2d, size=(size_y, size_x)) if debugplot % 10 != 0: ximshow(slitlet2d_smooth, title=sfilename + " [smoothed]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), debugplot=debugplot) # apply 1d Savitzky-Golay filter (along the spatial direction) # to compute first derivative slitlet2d_savgol = savgol_filter( slitlet2d_smooth, window_length=size_y_savgol, polyorder=2, deriv=1, axis=0) # compute basic statistics q25, q50, q75 = np.percentile(slitlet2d_savgol, q=[25.0, 50.0, 75.0]) sigmag = 0.7413 * (q75 - q25) # robust standard deviation if debugplot >= 10: print("q50, sigmag:", q50, sigmag) if debugplot % 10 != 0: ximshow(slitlet2d_savgol, title=sfilename + " [S.-G.filt.]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), z1z2=(q50-times_sigma_threshold*sigmag, q50+times_sigma_threshold*sigmag), debugplot=debugplot) # identify objects in slitlet2d_savgol: pixels with positive # derivatives are identify independently from pixels with # negative derivaties; then the two set of potential features # are merged; this approach avoids some problems when, in # nearby regions, there are pixels with positive and negative # derivatives (in those circumstances a single search as # np.logical_or( # slitlet2d_savgol < q50 - times_sigma_threshold * sigmag, # slitlet2d_savgol > q50 + times_sigma_threshold * sigmag) # led to erroneous detections!) # # search for positive derivatives labels2d_objects_pos, no_objects_pos = ndimage.label( slitlet2d_savgol > q50 + times_sigma_threshold * sigmag) # search for negative derivatives labels2d_objects_neg, no_objects_neg = ndimage.label( slitlet2d_savgol < q50 - times_sigma_threshold * sigmag) # merge both sets non_zero_neg = np.where(labels2d_objects_neg > 0) labels2d_objects = np.copy(labels2d_objects_pos) labels2d_objects[non_zero_neg] += \ labels2d_objects_neg[non_zero_neg] + no_objects_pos no_objects = no_objects_pos + no_objects_neg if debugplot >= 10: print("Number of objects with positive derivative:", no_objects_pos) print("Number of objects with negative derivative:", no_objects_neg) print("Total number of objects initially found...:", no_objects) if debugplot % 10 != 0: ximshow(labels2d_objects, z1z2=(0, no_objects), title=sfilename + " [objects]" "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), cbar_label="Object number", debugplot=debugplot) # select boundaries as the largest objects found with # positive and negative derivatives n_der_pos = 0 # number of pixels covered by the object with deriv > 0 i_der_pos = 0 # id of the object with deriv > 0 n_der_neg = 0 # number of pixels covered by the object with deriv < 0 i_der_neg = 0 # id of the object with deriv < 0 for i in range(1, no_objects+1): xy_tmp = np.where(labels2d_objects == i) n_pix = len(xy_tmp[0]) if i <= no_objects_pos: if n_pix > n_der_pos: i_der_pos = i n_der_pos = n_pix else: if n_pix > n_der_neg: i_der_neg = i n_der_neg = n_pix # determine which boundary is lower and which is upper y_center_mass_der_pos = ndimage.center_of_mass( slitlet2d_savgol, labels2d_objects, [i_der_pos])[0][0] y_center_mass_der_neg = ndimage.center_of_mass( slitlet2d_savgol, labels2d_objects, [i_der_neg])[0][0] if y_center_mass_der_pos < y_center_mass_der_neg: i_lower = i_der_pos i_upper = i_der_neg if debugplot >= 10: print("-> lower boundary has positive derivatives") else: i_lower = i_der_neg i_upper = i_der_pos if debugplot >= 10: print("-> lower boundary has negative derivatives") list_slices_ok = [i_lower, i_upper] # adjust individual boundaries passing the selection: # - select points in the image belonging to a given boundary # - compute weighted mean of the pixels of the boundary, column # by column (this reduces dramatically the number of points # to be fitted to determine the boundary) list_boundaries = [] for k in range(2): # k=0 lower boundary, k=1 upper boundary # select points to be fitted for a particular boundary # (note: be careful with array indices and pixel # coordinates) xy_tmp = np.where(labels2d_objects == list_slices_ok[k]) xmin = xy_tmp[1].min() # array indices (integers) xmax = xy_tmp[1].max() # array indices (integers) xfit = [] yfit = [] # fix range for fit if k == 0: xmineff = max(sltlim.xmin_lower_boundary_fit, xmin) xmaxeff = min(sltlim.xmax_lower_boundary_fit, xmax) else: xmineff = max(sltlim.xmin_upper_boundary_fit, xmin) xmaxeff = min(sltlim.xmax_upper_boundary_fit, xmax) # loop in columns of the image belonging to the boundary for xdum in range(xmineff, xmaxeff + 1): # array indices (integer) iok = np.where(xy_tmp[1] == xdum) y_tmp = xy_tmp[0][iok] + sltlim.bb_ns1_orig # image pixel weight = slitlet2d_savgol[xy_tmp[0][iok], xy_tmp[1][iok]] y_wmean = sum(y_tmp * weight) / sum(weight) xfit.append(xdum + sltlim.bb_nc1_orig) yfit.append(y_wmean) xfit = np.array(xfit) yfit = np.array(yfit) # declare new SpectrumTrail instance boundary = SpectrumTrail() # define new boundary boundary.fit(x=xfit, y=yfit, deg=sltlim.deg_boundary, times_sigma_reject=10, title="slit:" + str(sltlim.islitlet) + ", deg=" + str(sltlim.deg_boundary), debugplot=0) list_boundaries.append(boundary) if debugplot % 10 != 0: for tmp_img, tmp_label in zip( [slitlet2d_savgol, slitlet2d], [' [S.-G.filt.]', ' [original]'] ): ax = ximshow(tmp_img, title=sfilename + tmp_label + "\nslitlet=" + str(islitlet) + ", grism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig), show=False, debugplot=debugplot) for k in range(2): xpol, ypol = list_boundaries[k].linspace_pix( start=1, stop=EMIR_NAXIS1) ax.plot(xpol, ypol, 'b--', linewidth=1) for k in range(2): xpol, ypol = list_boundaries[k].linspace_pix() ax.plot(xpol, ypol, 'g--', linewidth=4) # show plot pause_debugplot(debugplot, pltshow=True) # update bounddict tmp_dict = { 'boundary_coef_lower': list_boundaries[0].poly_funct.coef.tolist(), 'boundary_xmin_lower': list_boundaries[0].xlower_line, 'boundary_xmax_lower': list_boundaries[0].xupper_line, 'boundary_coef_upper': list_boundaries[1].poly_funct.coef.tolist(), 'boundary_xmin_upper': list_boundaries[1].xlower_line, 'boundary_xmax_upper': list_boundaries[1].xupper_line, 'csu_bar_left': csu_config.csu_bar_left(islitlet), 'csu_bar_right': csu_config.csu_bar_right(islitlet), 'csu_bar_slit_center': csu_config.csu_bar_slit_center(islitlet), 'csu_bar_slit_width': csu_config.csu_bar_slit_width(islitlet), 'rotang': rotang, 'xdtu': dtu_config.xdtu, 'ydtu': dtu_config.ydtu, 'zdtu': dtu_config.zdtu, 'xdtu_0': dtu_config.xdtu_0, 'ydtu_0': dtu_config.ydtu_0, 'zdtu_0': dtu_config.zdtu_0, 'zzz_info1': os.getlogin() + '@' + socket.gethostname(), 'zzz_info2': datetime.now().isoformat() } slitlet_label = "slitlet" + str(islitlet).zfill(2) if slitlet_label not in bounddict['contents']: bounddict['contents'][slitlet_label] = {} bounddict['contents'][slitlet_label][date_obs] = tmp_dict if debugplot < 10: print("")
def display_slitlet_arrangement(fileobj, grism=None, spfilter=None, bbox=None, adjust=None, geometry=None, debugplot=0): """Display slitlet arrangment from CSUP keywords in FITS header. Parameters ---------- fileobj : file object FITS or TXT file object. grism : str Grism. grism : str Filter. bbox : tuple of 4 floats If not None, values for xmin, xmax, ymin and ymax. adjust : bool Adjust X range according to minimum and maximum csu_bar_left and csu_bar_right (note that this option is overriden by 'bbox') geometry : tuple (4 integers) or None x, y, dx, dy values employed to set the Qt backend geometry. debugplot : int Determines whether intermediate computations and/or plots are displayed. The valid codes are defined in numina.array.display.pause_debugplot Returns ------- csu_bar_left : list of floats Location (mm) of the left bar for each slitlet. csu_bar_right : list of floats Location (mm) of the right bar for each slitlet, using the same origin employed for csu_bar_left (which is not the value stored in the FITS keywords. csu_bar_slit_center : list of floats Middle point (mm) in between the two bars defining a slitlet. csu_bar_slit_width : list of floats Slitlet width (mm), computed as the distance between the two bars defining the slitlet. """ if fileobj.name[-4:] == ".txt": if grism is None: raise ValueError("Undefined grism!") if spfilter is None: raise ValueError("Undefined filter!") # define CsuConfiguration object csu_config = CsuConfiguration() csu_config._csu_bar_left = [] csu_config._csu_bar_right = [] csu_config._csu_bar_slit_center = [] csu_config._csu_bar_slit_width = [] # since the input filename has been opened with argparse in binary # mode, it is necessary to close it and open it in text mode fileobj.close() # read TXT file with open(fileobj.name, mode='rt') as f: file_content = f.read().splitlines() next_id_bar = 1 for line in file_content: if len(line) > 0: if line[0] not in ['#']: line_contents = line.split() id_bar = int(line_contents[0]) position = float(line_contents[1]) if id_bar == next_id_bar: if id_bar <= EMIR_NBARS: csu_config._csu_bar_left.append(position) next_id_bar = id_bar + EMIR_NBARS else: csu_config._csu_bar_right.append(341.5 - position) next_id_bar = id_bar - EMIR_NBARS + 1 else: raise ValueError("Unexpected id_bar:" + str(id_bar)) # compute slit width and center for i in range(EMIR_NBARS): csu_config._csu_bar_slit_center.append( (csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i]) / 2) csu_config._csu_bar_slit_width.append( csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i]) else: # read input FITS file hdulist = fits.open(fileobj.name) image_header = hdulist[0].header hdulist.close() # additional info from header grism = image_header['grism'] spfilter = image_header['filter'] # define slitlet arrangement csu_config = CsuConfiguration.define_from_fits(fileobj) # determine calibration if grism in ["J", "OPEN"] and spfilter == "J": wv_parameters = set_wv_parameters("J", "J") elif grism in ["H", "OPEN"] and spfilter == "H": wv_parameters = set_wv_parameters("H", "H") elif grism in ["K", "OPEN"] and spfilter == "Ksp": wv_parameters = set_wv_parameters("Ksp", "K") elif grism in ["LR", "OPEN"] and spfilter == "YJ": wv_parameters = set_wv_parameters("YJ", "LR") elif grism in ["LR", "OPEN"] and spfilter == "HK": wv_parameters = set_wv_parameters("HK", "LR") else: raise ValueError("Invalid grism + filter configuration") crval1 = wv_parameters['poly_crval1_linear'] cdelt1 = wv_parameters['poly_cdelt1_linear'] wvmin_useful = wv_parameters['wvmin_useful'] wvmax_useful = wv_parameters['wvmax_useful'] # display arrangement if abs(debugplot) >= 10: print("slit left right center width min.wave max.wave") print("==== ======= ======= ======= ===== ======== ========") for i in range(EMIR_NBARS): ibar = i + 1 csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar)) csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar)) csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1 if wvmin_useful is not None: csu_crval1 = np.amax([csu_crval1, wvmin_useful]) if wvmax_useful is not None: csu_crvaln = np.amin([csu_crvaln, wvmax_useful]) print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} " "{5:8.2f} {6:8.2f}".format( ibar, csu_config.csu_bar_left(ibar), csu_config.csu_bar_right(ibar), csu_config.csu_bar_slit_center(ibar), csu_config.csu_bar_slit_width(ibar), csu_crval1, csu_crvaln)) print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format( np.mean(csu_config._csu_bar_left), np.mean(csu_config._csu_bar_right), np.mean(csu_config._csu_bar_slit_center), np.mean(csu_config._csu_bar_slit_width))) print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format( np.mean(csu_config._csu_bar_left[::2]), np.mean(csu_config._csu_bar_right[::2]), np.mean(csu_config._csu_bar_slit_center[::2]), np.mean(csu_config._csu_bar_slit_width[::2]))) print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format( np.mean(csu_config._csu_bar_left[1::2]), np.mean(csu_config._csu_bar_right[1::2]), np.mean(csu_config._csu_bar_slit_center[1::2]), np.mean(csu_config._csu_bar_slit_width[1::2]))) # display slit arrangement if abs(debugplot) % 10 != 0: fig = plt.figure() set_window_geometry(geometry) ax = fig.add_subplot(111) if bbox is None: if adjust: xmin = min(csu_config._csu_bar_left) xmax = max(csu_config._csu_bar_right) dx = xmax - xmin if dx == 0: dx = 1 xmin -= dx / 20 xmax += dx / 20 ax.set_xlim(xmin, xmax) else: ax.set_xlim(0., 341.5) ax.set_ylim(0, 56) else: ax.set_xlim(bbox[0], bbox[1]) ax.set_ylim(bbox[2], bbox[3]) ax.set_xlabel('csu_bar_position (mm)') ax.set_ylabel('slit number') for i in range(EMIR_NBARS): ibar = i + 1 ax.add_patch( patches.Rectangle((csu_config.csu_bar_left(ibar), ibar - 0.5), csu_config.csu_bar_slit_width(ibar), 1.0)) ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar], '-', color='gray') ax.plot([csu_config.csu_bar_right(ibar), 341.5], [ibar, ibar], '-', color='gray') plt.title("File: " + fileobj.name + "\ngrism=" + grism + ", filter=" + spfilter) pause_debugplot(debugplot, pltshow=True) # return results return csu_config._csu_bar_left, csu_config._csu_bar_right, \ csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def main(args=None): # parse command-line options parser = argparse.ArgumentParser() # positional arguments parser.add_argument("fitsfile", help="FITS file name to be displayed", type=argparse.FileType('rb')) parser.add_argument("--fitted_bound_param", required=True, help="JSON file with fitted boundary coefficients " "corresponding to the multislit model", type=argparse.FileType('rt')) parser.add_argument("--slitlets", required=True, help="Slitlet selection: string between double " "quotes providing tuples of the form " "n1[,n2[,step]]", type=str) # optional arguments parser.add_argument("--outfile", help="Output FITS file name", type=lambda x: arg_file_is_new(parser, x, mode='wb')) parser.add_argument("--maskonly", help="Generate mask for the indicated slitlets", action="store_true") parser.add_argument("--debugplot", help="Integer indicating plotting/debugging" + " (default=0)", type=int, default=0, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args() if args.echo: print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n') # read input FITS file hdulist_image = fits.open(args.fitsfile.name) image_header = hdulist_image[0].header image2d = hdulist_image[0].data naxis1 = image_header['naxis1'] naxis2 = image_header['naxis2'] if image2d.shape != (naxis2, naxis1): raise ValueError("Unexpected error with NAXIS1, NAXIS2") if image2d.shape != (EMIR_NAXIS2, EMIR_NAXIS1): raise ValueError("NAXIS1, NAXIS2 unexpected for EMIR detector") # remove path from fitsfile if args.outfile is None: sfitsfile = os.path.basename(args.fitsfile.name) else: sfitsfile = os.path.basename(args.outfile.name) # check that the FITS file has been obtained with EMIR instrument = image_header['instrume'] if instrument != 'EMIR': raise ValueError("INSTRUME keyword is not 'EMIR'!") # read GRISM, FILTER and ROTANG from FITS header grism = image_header['grism'] spfilter = image_header['filter'] rotang = image_header['rotang'] # read fitted_bound_param JSON file fittedpar_dict = json.loads(open(args.fitted_bound_param.name).read()) params = bound_params_from_dict(fittedpar_dict) if abs(args.debugplot) in [21, 22]: params.pretty_print() parmodel = fittedpar_dict['meta_info']['parmodel'] if parmodel != 'multislit': raise ValueError("Unexpected parameter model: ", parmodel) # define slitlet range islitlet_min = fittedpar_dict['tags']['islitlet_min'] islitlet_max = fittedpar_dict['tags']['islitlet_max'] list_islitlet = list_slitlets_from_string( s=args.slitlets, islitlet_min=islitlet_min, islitlet_max=islitlet_max ) # read CsuConfiguration object from FITS file csu_config = CsuConfiguration.define_from_fits(args.fitsfile) # define csu_bar_slit_center associated to each slitlet list_csu_bar_slit_center = [] for islitlet in list_islitlet: list_csu_bar_slit_center.append( csu_config.csu_bar_slit_center(islitlet)) # initialize output data array image2d_output = np.zeros((naxis2, naxis1)) # main loop for islitlet, csu_bar_slit_center in \ zip(list_islitlet, list_csu_bar_slit_center): image2d_tmp = select_unrectified_slitlet( image2d=image2d, islitlet=islitlet, csu_bar_slit_center=csu_bar_slit_center, params=params, parmodel=parmodel, maskonly=args.maskonly ) image2d_output += image2d_tmp # update the array of the output file hdulist_image[0].data = image2d_output # save output FITS file hdulist_image.writeto(args.outfile) # close original image hdulist_image.close() # display full image if abs(args.debugplot) % 10 != 0: ax = ximshow(image2d=image2d_output, title=sfitsfile + "\n" + args.slitlets, image_bbox=(1, naxis1, 1, naxis2), show=False) # overplot boundaries overplot_boundaries_from_params( ax=ax, params=params, parmodel=parmodel, list_islitlet=list_islitlet, list_csu_bar_slit_center=list_csu_bar_slit_center ) # overplot frontiers overplot_frontiers_from_params( ax=ax, params=params, parmodel=parmodel, list_islitlet=list_islitlet, list_csu_bar_slit_center=list_csu_bar_slit_center, micolors=('b', 'b'), linetype='-', labels=False # already displayed with the boundaries ) # show plot pause_debugplot(12, pltshow=True)
def run(self, rinput): self.logger.info('starting generation of flatlowfreq') self.logger.info('rectwv_coeff..........................: {}'.format( rinput.rectwv_coeff)) self.logger.info('master_rectwv.........................: {}'.format( rinput.master_rectwv)) self.logger.info('Minimum slitlet width (mm)............: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info('Maximum slitlet width (mm)............: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info('Global offset X direction (pixels)....: {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info('Global offset Y direction (pixels)....: {}'.format( rinput.global_integer_offset_y_pix)) self.logger.info('nwindow_x_median......................: {}'.format( rinput.nwindow_x_median)) self.logger.info('nwindow_y_median......................: {}'.format( rinput.nwindow_y_median)) self.logger.info('Minimum fraction......................: {}'.format( rinput.minimum_fraction)) self.logger.info('Minimum value in output...............: {}'.format( rinput.minimum_value_in_output)) self.logger.info('Maximum value in output...............: {}'.format( rinput.maximum_value_in_output)) # check rectification and wavelength calibration information if rinput.master_rectwv is None and rinput.rectwv_coeff is None: raise ValueError('No master_rectwv nor rectwv_coeff data have ' 'been provided') elif rinput.master_rectwv is not None and \ rinput.rectwv_coeff is not None: self.logger.warning('rectwv_coeff will be used instead of ' 'master_rectwv') if rinput.rectwv_coeff is not None and \ (rinput.global_integer_offset_x_pix != 0 or rinput.global_integer_offset_y_pix != 0): raise ValueError('global_integer_offsets cannot be used ' 'simultaneously with rectwv_coeff') # check headers to detect lamp status (on/off) list_lampincd = [] for fname in rinput.obresult.frames: with fname.open() as f: list_lampincd.append(f[0].header['lampincd']) # check number of images nimages = len(rinput.obresult.frames) n_on = list_lampincd.count(1) n_off = list_lampincd.count(0) self.logger.info( 'Number of images with lamp ON.........: {}'.format(n_on)) self.logger.info( 'Number of images with lamp OFF........: {}'.format(n_off)) self.logger.info( 'Total number of images................: {}'.format(nimages)) if n_on == 0: raise ValueError('Insufficient number of images with lamp ON') if n_on + n_off != nimages: raise ValueError('Number of images does not match!') # check combination method if rinput.method_kwargs == {}: method_kwargs = None else: if rinput.method == 'sigmaclip': method_kwargs = rinput.method_kwargs else: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # build object to proceed with bpm, bias, and dark (not flat) flow = self.init_filters(rinput) # available combination methods method = getattr(combine, rinput.method) # basic reduction of images with lamp ON or OFF lampmode = {0: 'off', 1: 'on'} reduced_image_on = None reduced_image_off = None for imode in lampmode.keys(): self.logger.info('starting basic reduction of images with' ' lamp {}'.format(lampmode[imode])) tmplist = [ rinput.obresult.frames[i] for i, lampincd in enumerate(list_lampincd) if lampincd == imode ] if len(tmplist) > 0: with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in tmplist ] reduced_image = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) if imode == 0: reduced_image_off = flow(reduced_image) hdr = reduced_image_off[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced_image_off, 'reduced_image_off.fits') elif imode == 1: reduced_image_on = flow(reduced_image) hdr = reduced_image_on[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced_image_on, 'reduced_image_on.fits') else: raise ValueError('Unexpected imode={}'.format(imode)) # computation of ON-OFF header_on = reduced_image_on[0].header data_on = reduced_image_on[0].data.astype('float32') if n_off > 0: header_off = reduced_image_off[0].header data_off = reduced_image_off[0].data.astype('float32') else: header_off = None data_off = np.zeros_like(data_on) reduced_data = data_on - data_off # update reduced image header reduced_image = self.create_reduced_image(rinput, reduced_data, header_on, header_off, list_lampincd) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # define rectification and wavelength calibration coefficients if rinput.rectwv_coeff is None: rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv) # set global offsets rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix else: rectwv_coeff = rinput.rectwv_coeff # save as JSON in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # apply global offsets (to both, the original and the cleaned version) image2d = apply_integer_offsets( image2d=reduced_data, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix) # load CSU configuration csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header) # determine (pseudo) longslits dict_longslits = csu_conf.pseudo_longslits() # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: self.logger.info('-> Removing slitlet (not defined): ' + str(idel)) list_valid_islitlets.remove(idel) # filter out slitlets with widths outside valid range list_outside_valid_width = [] for islitlet in list_valid_islitlets: slitwidth = csu_conf.csu_bar_slit_width(islitlet) if (slitwidth < rinput.minimum_slitlet_width_mm) or \ (slitwidth > rinput.maximum_slitlet_width_mm): list_outside_valid_width.append(islitlet) self.logger.info('-> Removing slitlet (width out of range): ' + str(islitlet)) if len(list_outside_valid_width) > 0: for idel in list_outside_valid_width: list_valid_islitlets.remove(idel) # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop grism_name = rectwv_coeff.tags['grism'] filter_name = rectwv_coeff.tags['filter'] cout = '0' debugplot = rinput.debugplot for islitlet in list(range(1, EMIR_NBARS + 1)): if islitlet in list_valid_islitlets: # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=debugplot) if abs(slt.debugplot) > 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d, subtitle='original image') # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d=slitlet2d, resampling=2, subtitle='original (cleaned) rectified') naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") # get useful slitlet region (use boundaries) spectrail = slt.list_spectrails[0] yy0 = slt.corr_yrect_a + \ slt.corr_yrect_b * spectrail(slt.x0_reference) ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig spectrail = slt.list_spectrails[2] yy0 = slt.corr_yrect_a + \ slt.corr_yrect_b * spectrail(slt.x0_reference) ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction sp_median = ndimage.median_filter(sp_collapsed, 5, mode='nearest') ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * rinput.minimum_fraction lremove = np.where(sp_median < y_threshold) sp_median[lremove] = 0.0 if abs(slt.debugplot) % 10 != 0: xaxis1 = np.arange(1, naxis1_slitlet2d + 1) title = 'Slitlet#' + str(islitlet) + ' (median spectrum)' ax = ximplotxy(xaxis1, sp_collapsed, title=title, show=False, **{'label': 'collapsed spectrum'}) ax.plot(xaxis1, sp_median, label='fitted spectrum') ax.plot([1, naxis1_slitlet2d], 2 * [y_threshold], label='threshold') # ax.plot(xknots, yknots, 'o', label='knots') ax.legend() ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian) pause_debugplot(slt.debugplot, pltshow=True, tight_layout=True) # generate rectified slitlet region filled with the # median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(slt.debugplot) % 10 != 0: slt.ximshow_rectified( slitlet2d_rect=slitlet2d_rect_spmedian, subtitle='rectified, filled with median spectrum') # compute smooth surface # clipped region slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy() slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0 slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0 # unrectified clipped image slitlet2d_unrect_clipped = slt.rectify( slitlet2d=slitlet2d_rect_clipped, resampling=2, inverse=True, subtitle='unrectified, filled with median spectrum ' '(clipped)') # normalize initial slitlet image (avoid division by zero) slitlet2d_norm_clipped = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_clipped[i, j] if den == 0: slitlet2d_norm_clipped[i, j] = 1.0 else: slitlet2d_norm_clipped[i, j] = \ slitlet2d[i, j] / den # set to 1.0 one additional pixel at each side (since # 'den' above is small at the borders and generates wrong # bright pixels) slitlet2d_norm_clipped = fix_pix_borders( image2d=slitlet2d_norm_clipped, nreplace=1, sought_value=1.0, replacement_value=1.0) slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose() slitlet2d_norm_clipped = fix_pix_borders( image2d=slitlet2d_norm_clipped, nreplace=1, sought_value=1.0, replacement_value=1.0) slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose() slitlet2d_norm_smooth = ndimage.median_filter( slitlet2d_norm_clipped, size=(rinput.nwindow_y_median, rinput.nwindow_x_median), mode='nearest') if abs(slt.debugplot) % 10 != 0: slt.ximshow_unrectified( slitlet2d=slitlet2d_norm_clipped, subtitle='unrectified, pixel-to-pixel (clipped)') slt.ximshow_unrectified( slitlet2d=slitlet2d_norm_smooth, subtitle='unrectified, pixel-to-pixel (smoothed)') # --- # check for (pseudo) longslit with previous and next slitlet imin = dict_longslits[islitlet].imin() imax = dict_longslits[islitlet].imax() if islitlet > 1: same_slitlet_below = (islitlet - 1) >= imin else: same_slitlet_below = False if islitlet < EMIR_NBARS: same_slitlet_above = (islitlet + 1) <= imax else: same_slitlet_above = False for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm_smooth[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers if not same_slitlet_below: image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 if not same_slitlet_above: image2d_flatfielded[(n2 - 5):n2, j] = 1 cout += '.' else: cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) self.logger.info(cout) # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded, offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix) # set pixels below minimum value to 1.0 filtered = np.where( image2d_flatfielded < rinput.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # set pixels above maximum value to 1.0 filtered = np.where( image2d_flatfielded > rinput.maximum_value_in_output) image2d_flatfielded[filtered] = 1.0 # update image header reduced_flatlowfreq = self.create_reduced_image( rinput, image2d_flatfielded, header_on, header_off, list_lampincd) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # save results in results directory self.logger.info('end of flatlowfreq generation') result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq) return result
def rectwv_coeff_from_arc_image(reduced_image, bound_param, lines_catalog, args_nbrightlines=None, args_ymargin_bb=2, args_remove_sp_background=True, args_times_sigma_threshold=10, args_order_fmap=2, args_sigma_gaussian_filtering=2, args_margin_npix=50, args_poldeg_initial=3, args_poldeg_refined=5, args_interactive=False, args_threshold_wv=0, args_ylogscale=False, args_pdf=None, args_geometry=(0, 0, 640, 480), debugplot=0): """Evaluate rect.+wavecal. coefficients from arc image Parameters ---------- reduced_image : HDUList object Image with preliminary basic reduction: bpm, bias, dark and flatfield. bound_param : RefinedBoundaryModelParam instance Refined boundary model. lines_catalog : Numpy array 2D numpy array with the contents of the master file with the expected arc line wavelengths. args_nbrightlines : int TBD args_ymargin_bb : int TBD args_remove_sp_background : bool TBD args_times_sigma_threshold : float TBD args_order_fmap : int TBD args_sigma_gaussian_filtering : float TBD args_margin_npix : int TBD args_poldeg_initial : int TBD args_poldeg_refined : int TBD args_interactive : bool TBD args_threshold_wv : float TBD args_ylogscale : bool TBD args_pdf : TBD args_geometry : TBD debugplot : int Debugging level for messages and plots. For details see 'numina.array.display.pause_debugplot.py'. Returns ------- rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for the particular CSU configuration of the input arc image. reduced_55sp : HDUList object Image with 55 spectra corresponding to the median spectrum for each slitlet, employed to derived the wavelength calibration polynomial. """ logger = logging.getLogger(__name__) # protections if args_interactive and args_pdf is not None: logger.error('--interactive and --pdf are incompatible options') raise ValueError('--interactive and --pdf are incompatible options') # header and data array header = reduced_image[0].header image2d = reduced_image[0].data # check grism and filter filter_name = header['filter'] logger.info('Filter: ' + filter_name) if filter_name != bound_param.tags['filter']: raise ValueError('Filter name does not match!') grism_name = header['grism'] logger.info('Grism: ' + grism_name) if grism_name != bound_param.tags['grism']: raise ValueError('Grism name does not match!') # read the CSU configuration from the image header csu_conf = CsuConfiguration.define_from_header(header) logger.debug(csu_conf) # read the DTU configuration from the image header dtu_conf = DtuConfiguration.define_from_header(header) logger.debug(dtu_conf) # set boundary parameters parmodel = bound_param.meta_info['parmodel'] params = bound_params_from_dict(bound_param.__getstate__()) if abs(debugplot) >= 10: print('-' * 83) print('* FITTED BOUND PARAMETERS') params.pretty_print() pause_debugplot(debugplot) # determine parameters according to grism+filter combination wv_parameters = set_wv_parameters(filter_name, grism_name) islitlet_min = wv_parameters['islitlet_min'] islitlet_max = wv_parameters['islitlet_max'] if args_nbrightlines is None: nbrightlines = wv_parameters['nbrightlines'] else: nbrightlines = [int(idum) for idum in args_nbrightlines.split(',')] poly_crval1_linear = wv_parameters['poly_crval1_linear'] poly_cdelt1_linear = wv_parameters['poly_cdelt1_linear'] wvmin_expected = wv_parameters['wvmin_expected'] wvmax_expected = wv_parameters['wvmax_expected'] wvmin_useful = wv_parameters['wvmin_useful'] wvmax_useful = wv_parameters['wvmax_useful'] # list of slitlets to be computed logger.info('list_slitlets: [' + str(islitlet_min) + ',... ' + str(islitlet_max) + ']') # read master arc line wavelengths (only brightest lines) wv_master = read_wv_master_from_array(master_table=lines_catalog, lines='brightest', debugplot=debugplot) # read master arc line wavelengths (whole data set) wv_master_all = read_wv_master_from_array(master_table=lines_catalog, lines='all', debugplot=debugplot) # check that the arc lines in the master file are properly sorted # in ascending order for i in range(len(wv_master_all) - 1): if wv_master_all[i] >= wv_master_all[i + 1]: logger.error('>>> wavelengths: ' + str(wv_master_all[i]) + ' ' + str(wv_master_all[i + 1])) raise ValueError('Arc lines are not sorted in master file') # --- image2d_55sp = np.zeros((EMIR_NBARS, EMIR_NAXIS1)) # compute rectification transformation and wavelength calibration # polynomials measured_slitlets = [] cout = '0' for islitlet in range(1, EMIR_NBARS + 1): if islitlet_min <= islitlet <= islitlet_max: # define Slitlet2dArc object slt = Slitlet2dArc(islitlet=islitlet, csu_conf=csu_conf, ymargin_bb=args_ymargin_bb, params=params, parmodel=parmodel, debugplot=debugplot) # extract 2D image corresponding to the selected slitlet, clipping # the image beyond the unrectified slitlet (in order to isolate # the arc lines of the current slitlet; otherwise there are # problems with arc lines from neighbour slitlets) image2d_tmp = select_unrectified_slitlet( image2d=image2d, islitlet=islitlet, csu_bar_slit_center=csu_conf.csu_bar_slit_center(islitlet), params=params, parmodel=parmodel, maskonly=False) slitlet2d = slt.extract_slitlet2d(image2d_tmp) # subtract smooth background computed as follows: # - median collapsed spectrum of the whole slitlet2d # - independent median filtering of the previous spectrum in the # two halves in the spectral direction if args_remove_sp_background: spmedian = np.median(slitlet2d, axis=0) naxis1_tmp = spmedian.shape[0] jmidpoint = naxis1_tmp // 2 sp1 = medfilt(spmedian[:jmidpoint], [201]) sp2 = medfilt(spmedian[jmidpoint:], [201]) spbackground = np.concatenate((sp1, sp2)) slitlet2d -= spbackground # locate unknown arc lines slt.locate_unknown_arc_lines( slitlet2d=slitlet2d, times_sigma_threshold=args_times_sigma_threshold) # continue working with current slitlet only if arc lines have # been detected if slt.list_arc_lines is not None: # compute intersections between spectrum trails and arc lines slt.xy_spectrail_arc_intersections(slitlet2d=slitlet2d) # compute rectification transformation slt.estimate_tt_to_rectify(order=args_order_fmap, slitlet2d=slitlet2d) # rectify image slitlet2d_rect = slt.rectify(slitlet2d, resampling=2, transformation=1) # median spectrum and line peaks from rectified image sp_median, fxpeaks = slt.median_spectrum_from_rectified_image( slitlet2d_rect, sigma_gaussian_filtering=args_sigma_gaussian_filtering, nwinwidth_initial=5, nwinwidth_refined=5, times_sigma_threshold=5, npix_avoid_border=6, nbrightlines=nbrightlines) image2d_55sp[islitlet - 1, :] = sp_median # determine expected wavelength limits prior to the wavelength # calibration csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet) crval1_linear = poly_crval1_linear(csu_bar_slit_center) cdelt1_linear = poly_cdelt1_linear(csu_bar_slit_center) expected_wvmin = crval1_linear - \ args_margin_npix * cdelt1_linear naxis1_linear = sp_median.shape[0] crvaln_linear = crval1_linear + \ (naxis1_linear - 1) * cdelt1_linear expected_wvmax = crvaln_linear + \ args_margin_npix * cdelt1_linear # override previous estimates when necessary if wvmin_expected is not None: expected_wvmin = wvmin_expected if wvmax_expected is not None: expected_wvmax = wvmax_expected # clip initial master arc line list with bright lines to # the expected wavelength range lok1 = expected_wvmin <= wv_master lok2 = wv_master <= expected_wvmax lok = lok1 * lok2 wv_master_eff = wv_master[lok] # perform initial wavelength calibration solution_wv = wvcal_spectrum( sp=sp_median, fxpeaks=fxpeaks, poly_degree_wfit=args_poldeg_initial, wv_master=wv_master_eff, wv_ini_search=expected_wvmin, wv_end_search=expected_wvmax, wvmin_useful=wvmin_useful, wvmax_useful=wvmax_useful, geometry=args_geometry, debugplot=slt.debugplot) # store initial wavelength calibration polynomial in current # slitlet instance slt.wpoly = np.polynomial.Polynomial(solution_wv.coeff) pause_debugplot(debugplot) # clip initial master arc line list with all the lines to # the expected wavelength range lok1 = expected_wvmin <= wv_master_all lok2 = wv_master_all <= expected_wvmax lok = lok1 * lok2 wv_master_all_eff = wv_master_all[lok] # clip master arc line list to useful region if wvmin_useful is not None: lok = wvmin_useful <= wv_master_all_eff wv_master_all_eff = wv_master_all_eff[lok] if wvmax_useful is not None: lok = wv_master_all_eff <= wvmax_useful wv_master_all_eff = wv_master_all_eff[lok] # refine wavelength calibration if args_poldeg_refined > 0: plottitle = '[slitlet#{}, refined]'.format(islitlet) poly_refined, yres_summary = refine_arccalibration( sp=sp_median, poly_initial=slt.wpoly, wv_master=wv_master_all_eff, poldeg=args_poldeg_refined, ntimes_match_wv=1, interactive=args_interactive, threshold=args_threshold_wv, plottitle=plottitle, ylogscale=args_ylogscale, geometry=args_geometry, pdf=args_pdf, debugplot=slt.debugplot) # store refined wavelength calibration polynomial in # current slitlet instance slt.wpoly = poly_refined # compute approximate linear values for CRVAL1 and CDELT1 naxis1_linear = sp_median.shape[0] crmin1_linear = slt.wpoly(1) crmax1_linear = slt.wpoly(naxis1_linear) slt.crval1_linear = crmin1_linear slt.cdelt1_linear = \ (crmax1_linear - crmin1_linear) / (naxis1_linear - 1) # check that the trimming of wv_master and wv_master_all has # preserved the wavelength range [crmin1_linear, crmax1_linear] if crmin1_linear < expected_wvmin: logger.warning(">>> islitlet: " + str(islitlet)) logger.warning("expected_wvmin: " + str(expected_wvmin)) logger.warning("crmin1_linear.: " + str(crmin1_linear)) logger.warning("WARNING: Unexpected crmin1_linear < " "expected_wvmin") if crmax1_linear > expected_wvmax: logger.warning(">>> islitlet: " + str(islitlet)) logger.warning("expected_wvmax: " + str(expected_wvmax)) logger.warning("crmax1_linear.: " + str(crmax1_linear)) logger.warning("WARNING: Unexpected crmax1_linear > " "expected_wvmax") cout += '.' else: cout += 'x' if islitlet % 10 == 0: if cout != 'x': cout = str(islitlet // 10) if debugplot != 0: pause_debugplot(debugplot) else: # define Slitlet2dArc object slt = Slitlet2dArc(islitlet=islitlet, csu_conf=csu_conf, ymargin_bb=args_ymargin_bb, params=None, parmodel=None, debugplot=debugplot) cout += 'i' # store current slitlet in list of measured slitlets measured_slitlets.append(slt) logger.info(cout) # --- # generate FITS file structure with 55 spectra corresponding to the # median spectrum for each slitlet reduced_55sp = fits.PrimaryHDU(data=image2d_55sp) reduced_55sp.header['crpix1'] = (0.0, 'reference pixel') reduced_55sp.header['crval1'] = (0.0, 'central value at crpix2') reduced_55sp.header['cdelt1'] = (1.0, 'increment') reduced_55sp.header['ctype1'] = 'PIXEL' reduced_55sp.header['cunit1'] = ('Pixel', 'units along axis2') reduced_55sp.header['crpix2'] = (0.0, 'reference pixel') reduced_55sp.header['crval2'] = (0.0, 'central value at crpix2') reduced_55sp.header['cdelt2'] = (1.0, 'increment') reduced_55sp.header['ctype2'] = 'PIXEL' reduced_55sp.header['cunit2'] = ('Pixel', 'units along axis2') # --- # Generate structure to store intermediate results outdict = {} outdict['instrument'] = 'EMIR' outdict['meta_info'] = {} outdict['meta_info']['creation_date'] = datetime.now().isoformat() outdict['meta_info']['description'] = \ 'computation of rectification and wavelength calibration polynomial ' \ 'coefficients for a particular CSU configuration' outdict['meta_info']['recipe_name'] = 'undefined' outdict['meta_info']['origin'] = {} outdict['meta_info']['origin']['bound_param_uuid'] = \ bound_param.uuid outdict['meta_info']['origin']['arc_image_uuid'] = 'undefined' outdict['tags'] = {} outdict['tags']['grism'] = grism_name outdict['tags']['filter'] = filter_name outdict['tags']['islitlet_min'] = islitlet_min outdict['tags']['islitlet_max'] = islitlet_max outdict['dtu_configuration'] = dtu_conf.outdict() outdict['uuid'] = str(uuid4()) outdict['contents'] = {} missing_slitlets = [] for slt in measured_slitlets: islitlet = slt.islitlet if islitlet_min <= islitlet <= islitlet_max: # avoid error when creating a python list of coefficients from # numpy polynomials when the polynomials do not exist (note that # the JSON format doesn't handle numpy arrays and such arrays must # be transformed into native python lists) if slt.wpoly is None: wpoly_coeff = None else: wpoly_coeff = slt.wpoly.coef.tolist() if slt.wpoly_longslit_model is None: wpoly_coeff_longslit_model = None else: wpoly_coeff_longslit_model = \ slt.wpoly_longslit_model.coef.tolist() # avoid similar error when creating a python list of coefficients # when the numpy array does not exist; note that this problem # does not happen with tt?_aij_longslit_model and # tt?_bij_longslit_model because the latter have already been # created as native python lists if slt.ttd_aij is None: ttd_aij = None else: ttd_aij = slt.ttd_aij.tolist() if slt.ttd_bij is None: ttd_bij = None else: ttd_bij = slt.ttd_bij.tolist() if slt.tti_aij is None: tti_aij = None else: tti_aij = slt.tti_aij.tolist() if slt.tti_bij is None: tti_bij = None else: tti_bij = slt.tti_bij.tolist() # creating temporary dictionary with the information corresponding # to the current slitlett that will be saved in the JSON file tmp_dict = { 'csu_bar_left': slt.csu_bar_left, 'csu_bar_right': slt.csu_bar_right, 'csu_bar_slit_center': slt.csu_bar_slit_center, 'csu_bar_slit_width': slt.csu_bar_slit_width, 'x0_reference': slt.x0_reference, 'y0_reference_lower': slt.y0_reference_lower, 'y0_reference_middle': slt.y0_reference_middle, 'y0_reference_upper': slt.y0_reference_upper, 'y0_reference_lower_expected': slt.y0_reference_lower_expected, 'y0_reference_middle_expected': slt.y0_reference_middle_expected, 'y0_reference_upper_expected': slt.y0_reference_upper_expected, 'y0_frontier_lower': slt.y0_frontier_lower, 'y0_frontier_upper': slt.y0_frontier_upper, 'y0_frontier_lower_expected': slt.y0_frontier_lower_expected, 'y0_frontier_upper_expected': slt.y0_frontier_upper_expected, 'corr_yrect_a': slt.corr_yrect_a, 'corr_yrect_b': slt.corr_yrect_b, 'min_row_rectified': slt.min_row_rectified, 'max_row_rectified': slt.max_row_rectified, 'ymargin_bb': slt.ymargin_bb, 'bb_nc1_orig': slt.bb_nc1_orig, 'bb_nc2_orig': slt.bb_nc2_orig, 'bb_ns1_orig': slt.bb_ns1_orig, 'bb_ns2_orig': slt.bb_ns2_orig, 'spectrail': { 'poly_coef_lower': slt.list_spectrails[ slt.i_lower_spectrail].poly_funct.coef.tolist(), 'poly_coef_middle': slt.list_spectrails[ slt.i_middle_spectrail].poly_funct.coef.tolist(), 'poly_coef_upper': slt.list_spectrails[ slt.i_upper_spectrail].poly_funct.coef.tolist(), }, 'frontier': { 'poly_coef_lower': slt.list_frontiers[0].poly_funct.coef.tolist(), 'poly_coef_upper': slt.list_frontiers[1].poly_funct.coef.tolist(), }, 'ttd_order': slt.ttd_order, 'ttd_aij': ttd_aij, 'ttd_bij': ttd_bij, 'tti_aij': tti_aij, 'tti_bij': tti_bij, 'ttd_order_longslit_model': slt.ttd_order_longslit_model, 'ttd_aij_longslit_model': slt.ttd_aij_longslit_model, 'ttd_bij_longslit_model': slt.ttd_bij_longslit_model, 'tti_aij_longslit_model': slt.tti_aij_longslit_model, 'tti_bij_longslit_model': slt.tti_bij_longslit_model, 'wpoly_coeff': wpoly_coeff, 'wpoly_coeff_longslit_model': wpoly_coeff_longslit_model, 'crval1_linear': slt.crval1_linear, 'cdelt1_linear': slt.cdelt1_linear } else: missing_slitlets.append(islitlet) tmp_dict = { 'csu_bar_left': slt.csu_bar_left, 'csu_bar_right': slt.csu_bar_right, 'csu_bar_slit_center': slt.csu_bar_slit_center, 'csu_bar_slit_width': slt.csu_bar_slit_width, 'x0_reference': slt.x0_reference, 'y0_frontier_lower_expected': slt.y0_frontier_lower_expected, 'y0_frontier_upper_expected': slt.y0_frontier_upper_expected } slitlet_label = "slitlet" + str(islitlet).zfill(2) outdict['contents'][slitlet_label] = tmp_dict # --- # OBSOLETE ''' # save JSON file needed to compute the MOS model with open(args.out_json.name, 'w') as fstream: json.dump(outdict, fstream, indent=2, sort_keys=True) print('>>> Saving file ' + args.out_json.name) ''' # --- # Create object of type RectWaveCoeff with coefficients for # rectification and wavelength calibration rectwv_coeff = RectWaveCoeff(instrument='EMIR') rectwv_coeff.quality_control = numina.types.qc.QC.GOOD rectwv_coeff.tags['grism'] = grism_name rectwv_coeff.tags['filter'] = filter_name rectwv_coeff.meta_info['origin']['bound_param'] = \ 'uuid' + bound_param.uuid rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration'] rectwv_coeff.total_slitlets = EMIR_NBARS rectwv_coeff.missing_slitlets = missing_slitlets for i in range(EMIR_NBARS): islitlet = i + 1 dumdict = {'islitlet': islitlet} cslitlet = 'slitlet' + str(islitlet).zfill(2) if cslitlet in outdict['contents']: dumdict.update(outdict['contents'][cslitlet]) else: raise ValueError("Unexpected error") rectwv_coeff.contents.append(dumdict) # debugging __getstate__ and __setstate__ # rectwv_coeff.writeto(args.out_json.name) # print('>>> Saving file ' + args.out_json.name) # check_setstate_getstate(rectwv_coeff, args.out_json.name) logger.info('Generating RectWaveCoeff object with uuid=' + rectwv_coeff.uuid) return rectwv_coeff, reduced_55sp
def rectwv_coeff_add_longslit_model(rectwv_coeff, geometry, debugplot=0): """Compute longslit_model coefficients for RectWaveCoeff object. Parameters ---------- rectwv_coeff : RectWaveCoeff instance Rectification and wavelength calibration coefficients for a particular CSU configuration corresponding to a longslit observation. geometry : TBD debugplot : int Debugging level for messages and plots. For details see 'numina.array.display.pause_debugplot.py'. Returns ------- rectwv_coeff : RectWaveCoeff instance Updated object with longslit_model coefficients computed. """ logger = logging.getLogger(__name__) # check grism and filter grism_name = rectwv_coeff.tags['grism'] logger.info('Grism: ' + grism_name) filter_name = rectwv_coeff.tags['filter'] logger.info('Filter: ' + filter_name) # list of slitlets to be computed list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: list_valid_islitlets.remove(idel) if abs(debugplot) >= 10: print('>>> valid slitlet numbers:\n', list_valid_islitlets) # --- # check that the CSU configuration corresponds to longslit csu_bar_slit_center_list = [] for islitlet in list_valid_islitlets: csu_bar_slit_center_list.append( rectwv_coeff.contents[islitlet - 1]['csu_bar_slit_center']) if abs(debugplot) >= 10: logger.debug('Checking csu_bar_slit_center values:') summary(np.array(csu_bar_slit_center_list), debug=True) pause_debugplot(debugplot) # --- # polynomial coefficients corresponding to the wavelength calibration # step 0: determine poldeg_refined, checking that it is the same for # all the slitlets poldeg_refined_list = [] for islitlet in list_valid_islitlets: poldeg_refined_list.append( len(rectwv_coeff.contents[islitlet - 1]['wpoly_coeff']) - 1) # remove duplicates poldeg_refined_list = list(set(poldeg_refined_list)) if len(poldeg_refined_list) != 1: raise ValueError('Unexpected different poldeg_refined found: ' + str(poldeg_refined_list)) poldeg_refined = poldeg_refined_list[0] # step 1: compute variation of each coefficient as a function of # y0_reference_middle of each slitlet list_poly = [] for i in range(poldeg_refined + 1): xp = [] yp = [] for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] wpoly_coeff = tmp_dict['wpoly_coeff'] if wpoly_coeff is not None: xp.append(tmp_dict['y0_reference_middle']) yp.append(wpoly_coeff[i]) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp), deg=2, times_sigma_reject=5, xlabel='y0_rectified', ylabel='coeff[' + str(i) + ']', title="Fit to refined wavelength calibration coefficients", geometry=geometry, debugplot=debugplot) list_poly.append(poly) # step 2: use the variation of each polynomial coefficient with # y0_reference_middle to infer the expected wavelength calibration # polynomial for each rectifified slitlet for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] y0_reference_middle = tmp_dict['y0_reference_middle'] list_new_coeff = [] for i in range(poldeg_refined + 1): new_coeff = list_poly[i](y0_reference_middle) list_new_coeff.append(new_coeff) tmp_dict['wpoly_coeff_longslit_model'] = list_new_coeff # --- # rectification transformation coefficients aij and bij # step 0: determine order_fmap, checking that it is the same for # all the slitlets order_fmap_list = [] for islitlet in list_valid_islitlets: order_fmap_list.append(rectwv_coeff.contents[islitlet - 1]['ttd_order']) # remove duplicates order_fmap_list = list(set(order_fmap_list)) if len(order_fmap_list) != 1: raise ValueError('Unexpected different order_fmap found') order_fmap = order_fmap_list[0] # step 1: compute variation of each coefficient as a function of # y0_reference_middle of each slitlet list_poly_ttd_aij = [] list_poly_ttd_bij = [] list_poly_tti_aij = [] list_poly_tti_bij = [] ncoef_ttd = ncoef_fmap(order_fmap) for i in range(ncoef_ttd): xp = [] yp_ttd_aij = [] yp_ttd_bij = [] yp_tti_aij = [] yp_tti_bij = [] for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] ttd_aij = tmp_dict['ttd_aij'] ttd_bij = tmp_dict['ttd_bij'] tti_aij = tmp_dict['tti_aij'] tti_bij = tmp_dict['tti_bij'] if ttd_aij is not None: xp.append(tmp_dict['y0_reference_middle']) yp_ttd_aij.append(ttd_aij[i]) yp_ttd_bij.append(ttd_bij[i]) yp_tti_aij.append(tti_aij[i]) yp_tti_bij.append(tti_bij[i]) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_ttd_aij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='ttd_aij[' + str(i) + ']', geometry=geometry, debugplot=debugplot) list_poly_ttd_aij.append(poly) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_ttd_bij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='ttd_bij[' + str(i) + ']', geometry=geometry, debugplot=debugplot) list_poly_ttd_bij.append(poly) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_tti_aij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='tti_aij[' + str(i) + ']', geometry=geometry, debugplot=debugplot) list_poly_tti_aij.append(poly) poly, yres, reject = polfit_residuals_with_sigma_rejection( x=np.array(xp), y=np.array(yp_tti_bij), deg=5, times_sigma_reject=5, xlabel='y0_rectified', ylabel='tti_bij[' + str(i) + ']', geometry=geometry, debugplot=debugplot) list_poly_tti_bij.append(poly) # step 2: use the variation of each coefficient with y0_reference_middle # to infer the expected rectification transformation for each slitlet for islitlet in list_valid_islitlets: tmp_dict = rectwv_coeff.contents[islitlet - 1] y0_reference_middle = tmp_dict['y0_reference_middle'] tmp_dict['ttd_order_longslit_model'] = order_fmap ttd_aij_longslit_model = [] ttd_bij_longslit_model = [] tti_aij_longslit_model = [] tti_bij_longslit_model = [] for i in range(ncoef_ttd): new_coeff = list_poly_ttd_aij[i](y0_reference_middle) ttd_aij_longslit_model.append(new_coeff) new_coeff = list_poly_ttd_bij[i](y0_reference_middle) ttd_bij_longslit_model.append(new_coeff) new_coeff = list_poly_tti_aij[i](y0_reference_middle) tti_aij_longslit_model.append(new_coeff) new_coeff = list_poly_tti_bij[i](y0_reference_middle) tti_bij_longslit_model.append(new_coeff) tmp_dict['ttd_aij_longslit_model'] = ttd_aij_longslit_model tmp_dict['ttd_bij_longslit_model'] = ttd_bij_longslit_model tmp_dict['tti_aij_longslit_model'] = tti_aij_longslit_model tmp_dict['tti_bij_longslit_model'] = tti_bij_longslit_model # --- # update uuid and meta_info in output JSON structure rectwv_coeff.uuid = str(uuid4()) rectwv_coeff.meta_info['creation_date'] = datetime.now().isoformat() # return updated object return rectwv_coeff
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: compute pixel-to-pixel flatfield' ) # required arguments parser.add_argument("fitsfile", help="Input FITS file (flat ON-OFF)", type=argparse.FileType('rb')) parser.add_argument("--rectwv_coeff", required=True, help="Input JSON file with rectification and " "wavelength calibration coefficients", type=argparse.FileType('rt')) parser.add_argument("--minimum_fraction", required=True, help="Minimum allowed flatfielding value", type=float, default=0.01) parser.add_argument("--minimum_value_in_output", help="Minimum value allowed in output file: pixels " "below this value are set to 1.0 (default=0.01)", type=float, default=0.01) parser.add_argument("--nwindow_median", required=True, help="Window size to smooth median spectrum in the " "spectral direction", type=int) parser.add_argument("--outfile", required=True, help="Output FITS file", type=lambda x: arg_file_is_new(parser, x, mode='wb')) # optional arguments parser.add_argument("--delta_global_integer_offset_x_pix", help="Delta global integer offset in the X direction " "(default=0)", default=0, type=int) parser.add_argument("--delta_global_integer_offset_y_pix", help="Delta global integer offset in the Y direction " "(default=0)", default=0, type=int) parser.add_argument("--resampling", help="Resampling method: 1 -> nearest neighbor, " "2 -> linear interpolation (default)", default=2, type=int, choices=(1, 2)) parser.add_argument("--ignore_DTUconf", help="Ignore DTU configurations differences between " "model and input image", action="store_true") parser.add_argument("--debugplot", help="Integer indicating plotting & debugging options" " (default=0)", default=0, type=int, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args) if args.echo: print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n') # read calibration structure from JSON file rectwv_coeff = RectWaveCoeff._datatype_load(args.rectwv_coeff.name) # modify (when requested) global offsets rectwv_coeff.global_integer_offset_x_pix += \ args.delta_global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix += \ args.delta_global_integer_offset_y_pix # read FITS image and its corresponding header hdulist = fits.open(args.fitsfile) header = hdulist[0].header image2d = hdulist[0].data hdulist.close() # apply global offsets image2d = apply_integer_offsets( image2d=image2d, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix ) # protections naxis2, naxis1 = image2d.shape if naxis1 != header['naxis1'] or naxis2 != header['naxis2']: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) raise ValueError('Something is wrong with NAXIS1 and/or NAXIS2') if abs(args.debugplot) >= 10: print('>>> NAXIS1:', naxis1) print('>>> NAXIS2:', naxis2) # check that the input FITS file grism and filter match filter_name = header['filter'] if filter_name != rectwv_coeff.tags['filter']: raise ValueError("Filter name does not match!") grism_name = header['grism'] if grism_name != rectwv_coeff.tags['grism']: raise ValueError("Filter name does not match!") if abs(args.debugplot) >= 10: print('>>> grism.......:', grism_name) print('>>> filter......:', filter_name) # check that the DTU configurations are compatible dtu_conf_fitsfile = DtuConfiguration.define_from_fits(args.fitsfile) dtu_conf_jsonfile = DtuConfiguration.define_from_dictionary( rectwv_coeff.meta_info['dtu_configuration']) if dtu_conf_fitsfile != dtu_conf_jsonfile: print('DTU configuration (FITS file):\n\t', dtu_conf_fitsfile) print('DTU configuration (JSON file):\n\t', dtu_conf_jsonfile) if args.ignore_DTUconf: print('WARNING: DTU configuration differences found!') else: raise ValueError('DTU configurations do not match') else: if abs(args.debugplot) >= 10: print('>>> DTU Configuration match!') print(dtu_conf_fitsfile) # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: list_valid_islitlets.remove(idel) if abs(args.debugplot) >= 10: print('>>> valid slitlet numbers:\n', list_valid_islitlets) # --- # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop for islitlet in list_valid_islitlets: if args.debugplot == 0: islitlet_progress(islitlet, EMIR_NBARS) # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=args.debugplot) if abs(args.debugplot) >= 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d(image2d) # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d, resampling=args.resampling ) naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") # get useful slitlet region (use boundaires instead of frontiers; # note that the nscan_minmax_frontiers() works well independently # of using frontiers of boundaries as arguments) nscan_min, nscan_max = nscan_minmax_frontiers( slt.y0_reference_lower, slt.y0_reference_upper, resize=False ) ii1 = nscan_min - slt.bb_ns1_orig ii2 = nscan_max - slt.bb_ns1_orig + 1 # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction sp_median = ndimage.median_filter(sp_collapsed, args.nwindow_median, mode='nearest') ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * args.minimum_fraction sp_median[np.where(sp_median < y_threshold)] = 0.0 if abs(args.debugplot) > 10: title = 'Slitlet#' + str(islitlet) + '(median spectrum)' xdum = np.arange(1, naxis1_slitlet2d + 1) ax = ximplotxy(xdum, sp_collapsed, title=title, show=False, **{'label' : 'collapsed spectrum'}) ax.plot(xdum, sp_median, label='filtered spectrum') ax.plot([1, naxis1_slitlet2d], 2*[y_threshold], label='threshold') ax.legend() ax.set_ylim(-0.05*ymax_spmedian, 1.05*ymax_spmedian) pause_debugplot(args.debugplot, pltshow=True, tight_layout=True) # generate rectified slitlet region filled with the median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(args.debugplot) > 10: slt.ximshow_rectified(slitlet2d_rect_spmedian) # unrectified image slitlet2d_unrect_spmedian = slt.rectify( slitlet2d_rect_spmedian, resampling=args.resampling, inverse=True ) # normalize initial slitlet image (avoid division by zero) slitlet2d_norm = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_spmedian[i, j] if den == 0: slitlet2d_norm[i, j] = 1.0 else: slitlet2d_norm[i, j] = slitlet2d[i, j] / den if abs(args.debugplot) > 10: slt.ximshow_unrectified(slitlet2d_norm) for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 image2d_flatfielded[(n2 - 5):n2, j] = 1 if args.debugplot == 0: print('OK!') # set pixels below minimum value to 1.0 filtered = np.where(image2d_flatfielded < args.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded , offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix ) # save output file save_ndarray_to_fits( array=image2d_flatfielded, file_name=args.outfile, main_header=header, overwrite=True ) print('>>> Saving file ' + args.outfile.name)
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description='description: overplot boundary model over FITS image') # positional arguments parser.add_argument("fitsfile", help="FITS file name to be displayed", type=argparse.FileType('rb')) parser.add_argument("--rect_wpoly_MOSlibrary", required=True, help="Input JSON file with library of rectification " "and wavelength calibration coefficients", type=argparse.FileType('rt')) # optional arguments parser.add_argument("--global_integer_offset_x_pix", help="Global integer offset in the X direction " "(default=0)", default=0, type=int) parser.add_argument("--global_integer_offset_y_pix", help="Global integer offset in the Y direction " "(default=0)", default=0, type=int) parser.add_argument("--arc_lines", help="Overplot arc lines", action="store_true") parser.add_argument("--oh_lines", help="Overplot OH lines", action="store_true") parser.add_argument("--ds9_frontiers", help="Output ds9 region file with slitlet frontiers", type=lambda x: arg_file_is_new(parser, x)) parser.add_argument("--ds9_boundaries", help="Output ds9 region file with slitlet boundaries", type=lambda x: arg_file_is_new(parser, x)) parser.add_argument("--ds9_lines", help="Output ds9 region file with arc/oh lines", type=lambda x: arg_file_is_new(parser, x)) parser.add_argument("--debugplot", help="Integer indicating plotting/debugging" + " (default=12)", type=int, default=12, choices=DEBUGPLOT_CODES) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args() if args.echo: print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n') # --- # avoid incompatible options if args.arc_lines and args.oh_lines: raise ValueError("--arc_lines and --oh_lines cannot be used " "simultaneously") # --ds9_lines requires --arc_lines or --oh_lines if args.ds9_lines: if not (args.arc_lines or args.oh_lines): raise ValueError("--ds9_lines requires the use of either " "--arc_lines or --oh_lines") # read input FITS file hdulist = fits.open(args.fitsfile) image_header = hdulist[0].header image2d = hdulist[0].data hdulist.close() naxis1 = image_header['naxis1'] naxis2 = image_header['naxis2'] if image2d.shape != (naxis2, naxis1): raise ValueError("Unexpected error with NAXIS1, NAXIS2") if image2d.shape != (EMIR_NAXIS2, EMIR_NAXIS1): raise ValueError("Unexpected values for NAXIS1, NAXIS2") # remove path from fitsfile sfitsfile = os.path.basename(args.fitsfile.name) # check that the FITS file has been obtained with EMIR instrument = image_header['instrume'] if instrument != 'EMIR': raise ValueError("INSTRUME keyword is not 'EMIR'!") # read GRISM, FILTER and ROTANG from FITS header grism = image_header['grism'] spfilter = image_header['filter'] rotang = image_header['rotang'] # --- # generate MasterRectWave object master_rectwv = MasterRectWave._datatype_load( args.rect_wpoly_MOSlibrary.name) # check that grism and filter are the expected ones grism_ = master_rectwv.tags['grism'] if grism_ != grism: raise ValueError('Unexpected grism: ' + str(grism_)) spfilter_ = master_rectwv.tags['filter'] if spfilter_ != spfilter: raise ValueError('Unexpected filter ' + str(spfilter_)) # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in master_rectwv.missing_slitlets: list_valid_islitlets.remove(idel) # read CsuConfiguration object from FITS file csu_config = CsuConfiguration.define_from_fits(args.fitsfile) # list with csu_bar_slit_center for valid slitlets list_csu_bar_slit_center = [] for islitlet in list_valid_islitlets: list_csu_bar_slit_center.append( csu_config.csu_bar_slit_center(islitlet)) # define parmodel and params fitted_bound_param_json = { 'contents': master_rectwv.meta_info['refined_boundary_model'] } parmodel = fitted_bound_param_json['contents']['parmodel'] fitted_bound_param_json.update({'meta_info': {'parmodel': parmodel}}) params = bound_params_from_dict(fitted_bound_param_json) if parmodel != "multislit": raise ValueError('parmodel = "multislit" not found') # --- # define lines to be overplotted if args.arc_lines or args.oh_lines: rectwv_coeff = rectwv_coeff_from_mos_library(hdulist, master_rectwv) rectwv_coeff.global_integer_offset_x_pix = \ args.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ args.global_integer_offset_y_pix # rectwv_coeff.writeto('xxx.json') if args.arc_lines: if grism == 'LR': catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat' else: catlines_file = 'lines_argon_neon_xenon_empirical.dat' dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file) arc_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(arc_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = catlines[:, 0] catlines_all_flux = catlines[:, 1] elif args.oh_lines: dumdata = pkgutil.get_data('emirdrp.instrument.configs', 'Oliva_etal_2013.dat') oh_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(oh_lines_tmpfile) # define wavelength and flux as separate arrays catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0])) catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2])) else: raise ValueError("This should not happen!") else: rectwv_coeff = None catlines_all_wave = None catlines_all_flux = None # --- # generate output ds9 region file with slitlet boundaries if args.ds9_boundaries is not None: save_boundaries_from_params_ds9( params=params, parmodel=parmodel, list_islitlet=list_valid_islitlets, list_csu_bar_slit_center=list_csu_bar_slit_center, uuid=master_rectwv.uuid, grism=grism, spfilter=spfilter, ds9_filename=args.ds9_boundaries.name, global_offset_x_pix=-args.global_integer_offset_x_pix, global_offset_y_pix=-args.global_integer_offset_y_pix) # generate output ds9 region file with slitlet frontiers if args.ds9_frontiers is not None: save_frontiers_from_params_ds9( params=params, parmodel=parmodel, list_islitlet=list_valid_islitlets, list_csu_bar_slit_center=list_csu_bar_slit_center, uuid=master_rectwv.uuid, grism=grism, spfilter=spfilter, ds9_filename=args.ds9_frontiers.name, global_offset_x_pix=-args.global_integer_offset_x_pix, global_offset_y_pix=-args.global_integer_offset_y_pix) # --- # display full image if abs(args.debugplot) % 10 != 0: ax = ximshow(image2d=image2d, title=sfitsfile + "\ngrism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), image_bbox=(1, naxis1, 1, naxis2), show=False) # overplot boundaries overplot_boundaries_from_params( ax=ax, params=params, parmodel=parmodel, list_islitlet=list_valid_islitlets, list_csu_bar_slit_center=list_csu_bar_slit_center, global_offset_x_pix=-args.global_integer_offset_x_pix, global_offset_y_pix=-args.global_integer_offset_y_pix) # overplot frontiers overplot_frontiers_from_params( ax=ax, params=params, parmodel=parmodel, list_islitlet=list_valid_islitlets, list_csu_bar_slit_center=list_csu_bar_slit_center, micolors=('b', 'b'), linetype='-', labels=False, # already displayed with the boundaries global_offset_x_pix=-args.global_integer_offset_x_pix, global_offset_y_pix=-args.global_integer_offset_y_pix) else: ax = None # overplot lines if catlines_all_wave is not None: if args.ds9_lines is None: ds9_file = None else: ds9_file = open(args.ds9_lines.name, 'w') ds9_file.write('# Region file format: DS9 version 4.1\n') ds9_file.write('global color=#00ffff dashlist=0 0 width=2 ' 'font="helvetica 10 normal roman" select=1 ' 'highlite=1 dash=0 fixed=0 edit=1 ' 'move=1 delete=1 include=1 source=1\n') ds9_file.write('physical\n#\n') ds9_file.write('#\n# uuid..: {0}\n'.format(master_rectwv.uuid)) ds9_file.write('# filter: {0}\n'.format(spfilter)) ds9_file.write('# grism.: {0}\n'.format(grism)) ds9_file.write('#\n# global_offset_x_pix: {0}\n'.format( args.global_integer_offset_x_pix)) ds9_file.write('# global_offset_y_pix: {0}\n#\n'.format( args.global_integer_offset_y_pix)) if parmodel == "longslit": for dumpar in EXPECTED_PARAMETER_LIST: parvalue = params[dumpar].value ds9_file.write('# {0}: {1}\n'.format(dumpar, parvalue)) else: for dumpar in EXPECTED_PARAMETER_LIST_EXTENDED: parvalue = params[dumpar].value ds9_file.write('# {0}: {1}\n'.format(dumpar, parvalue)) overplot_lines(ax, catlines_all_wave, list_valid_islitlets, rectwv_coeff, args.global_integer_offset_x_pix, args.global_integer_offset_y_pix, ds9_file, args.debugplot) if ds9_file is not None: ds9_file.close() if ax is not None: # show plot pause_debugplot(12, pltshow=True)
def main(args=None): # parse command-line options parser = argparse.ArgumentParser( description="description: overplot traces" ) # positional parameters parser.add_argument("fits_file", help="FITS image containing the spectra", type=argparse.FileType('r')) parser.add_argument("traces_file", help="JSON file with fiber traces", type=argparse.FileType('r')) # optional parameters parser.add_argument("--rawimage", help="FITS file is a RAW image (otherwise trimmed " "image is assumed)", action="store_true") parser.add_argument("--global_offset", help="Global offset polynomial coefficients " "(+upwards, -downwards)") parser.add_argument("--fibids", help="Display fiber identification number", action="store_true") parser.add_argument("--verbose", help="Enhance verbosity", action="store_true") parser.add_argument("--healing", help="JSON healing file to improve traces", type=argparse.FileType('r')) parser.add_argument("--updated_traces", help="JSON file with modified fiber traces", type=argparse.FileType('w')) parser.add_argument("--z1z2", help="tuple z1,z2, minmax or None (use zscale)") parser.add_argument("--bbox", help="bounding box tuple: nc1,nc2,ns1,ns2") parser.add_argument("--keystitle", help="tuple of FITS keywords.format: " + "key1,key2,...keyn.'format'") parser.add_argument("--geometry", help="tuple x,y,dx,dy", default="0,0,640,480") parser.add_argument("--pdffile", help="ouput PDF file name", type=argparse.FileType('w')) parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args(args=args) if args.echo: print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n') # global_offset in command line if args.global_offset is None: args_global_offset = [0.0] else: args_global_offset = [float(dum) for dum in str(args.global_offset).split(",")] # read pdffile if args.pdffile is not None: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages(args.pdffile.name) else: pdf = None ax = ximshow_file(args.fits_file.name, args_cbar_orientation='vertical', args_z1z2=args.z1z2, args_bbox=args.bbox, args_keystitle=args.keystitle, args_geometry=args.geometry, pdf=pdf, show=False) # trace offsets for RAW images if args.rawimage: ix_offset = 51 else: ix_offset = 1 # read and display traces from JSON file bigdict = json.loads(open(args.traces_file.name).read()) # Load metadata from the traces meta_info = bigdict['meta_info'] origin = meta_info['origin'] insconf_uuid = origin['insconf_uuid'] date_obs = origin['date_obs'] tags = bigdict['tags'] insmode = tags['insmode'] # create instrument model pkg_paths = ['megaradrp.instrument.configs'] store = asb.load_paths_store(pkg_paths) insmodel = asb.assembly_instrument(store, insconf_uuid, date_obs, by_key='uuid') pseudo_slit_config = insmodel.get_value('pseudoslit.boxes', **tags) fibid_with_box = assign_boxes_to_fibers(pseudo_slit_config, insmode) total_fibers = bigdict['total_fibers'] if total_fibers != len(fibid_with_box): raise ValueError('Mismatch between number of fibers and ' 'expected number from account from boxes') if 'global_offset' in bigdict.keys(): global_offset = bigdict['global_offset'] if args_global_offset != [0.0] and global_offset != [0.0]: raise ValueError('global_offset != 0 argument cannot be employed ' 'when global_offset != 0 in JSON file') elif args_global_offset != [0.0]: global_offset = args_global_offset else: global_offset = args_global_offset print('>>> Using global_offset:', global_offset) pol_global_offset = np.polynomial.Polynomial(global_offset) if 'ref_column' in bigdict.keys(): ref_column = bigdict['ref_column'] else: ref_column = 2000 for fiberdict in bigdict['contents']: fibid = fiberdict['fibid'] fiblabel = fibid_with_box[fibid - 1] start = fiberdict['start'] stop = fiberdict['stop'] coeff = np.array(fiberdict['fitparms']) # skip fibers without trace if len(coeff) > 0: pol_trace = np.polynomial.Polynomial(coeff) y_at_ref_column = pol_trace(ref_column) correction = pol_global_offset(y_at_ref_column) coeff[0] += correction # update values in bigdict (JSON structure) bigdict['contents'][fibid-1]['fitparms'] = coeff.tolist() plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, args.fibids, fiblabel, colour='blue') else: print('Warning ---> Missing fiber:', fibid_with_box[fibid - 1]) # if present, read healing JSON file if args.healing is not None: healdict = json.loads(open(args.healing.name).read()) list_operations = healdict['operations'] for operation in list_operations: if operation['description'] == 'vertical_shift_in_pixels': if 'fibid_list' in operation.keys(): fibid_list = operation['fibid_list'] else: fibid_ini = operation['fibid_ini'] fibid_end = operation['fibid_end'] fibid_list = range(fibid_ini, fibid_end + 1) for fibid in fibid_list: if fibid < 1 or fibid > total_fibers: raise ValueError('fibid number outside valid range') fiblabel = fibid_with_box[fibid - 1] coeff = np.array( bigdict['contents'][fibid - 1]['fitparms'] ) if len(coeff) > 0: if args.verbose: print('(vertical_shift_in_pixels) fibid:', fiblabel) vshift = operation['vshift'] coeff[0] += vshift bigdict['contents'][fibid - 1]['fitparms'] = \ coeff.tolist() start = bigdict['contents'][fibid - 1]['start'] stop = bigdict['contents'][fibid - 1]['stop'] plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, True, fiblabel, colour='green') else: print('(vertical_shift_in_pixels SKIPPED) fibid:', fiblabel) elif operation['description'] == 'duplicate_trace': fibid_original = operation['fibid_original'] if fibid_original < 1 or fibid_original > total_fibers: raise ValueError( 'fibid_original number outside valid range' ) fibid_duplicated = operation['fibid_duplicated'] if fibid_duplicated < 1 or fibid_duplicated > total_fibers: raise ValueError( 'fibid_duplicated number outside valid range' ) fiblabel_original = fibid_with_box[fibid_original - 1] fiblabel_duplicated = fibid_with_box[fibid_duplicated - 1] coeff = np.array( bigdict['contents'][fibid_original - 1]['fitparms'] ) if len(coeff) > 0: if args.verbose: print('(duplicated_trace) fibids:', fiblabel_original, '-->', fiblabel_duplicated) vshift = operation['vshift'] coeff[0] += vshift bigdict['contents'][fibid_duplicated - 1]['fitparms'] = \ coeff.tolist() start = bigdict['contents'][fibid_original - 1]['start'] stop = bigdict['contents'][fibid_original - 1]['stop'] bigdict['contents'][fibid_duplicated - 1]['start'] = start bigdict['contents'][fibid_duplicated - 1]['stop'] = stop plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, True, fiblabel_duplicated, colour='green') else: print('(duplicated_trace SKIPPED) fibids:', fiblabel_original, '-->', fiblabel_duplicated) elif operation['description'] == 'extrapolation': if 'fibid_list' in operation.keys(): fibid_list = operation['fibid_list'] else: fibid_ini = operation['fibid_ini'] fibid_end = operation['fibid_end'] fibid_list = range(fibid_ini, fibid_end + 1) for fibid in fibid_list: if fibid < 1 or fibid > total_fibers: raise ValueError('fibid number outside valid range') fiblabel = fibid_with_box[fibid - 1] coeff = np.array( bigdict['contents'][fibid - 1]['fitparms'] ) if len(coeff) > 0: if args.verbose: print('(extrapolation) fibid:', fiblabel) # update values in bigdict (JSON structure) start = operation['start'] stop = operation['stop'] start_orig = bigdict['contents'][fibid - 1]['start'] stop_orig = bigdict['contents'][fibid - 1]['stop'] bigdict['contents'][fibid - 1]['start'] = start bigdict['contents'][fibid - 1]['stop'] = stop if start < start_orig: plot_trace(ax, coeff, start, start_orig, ix_offset, args.rawimage, True, fiblabel, colour='green') if stop_orig < stop: plot_trace(ax, coeff, stop_orig, stop, ix_offset, args.rawimage, True, fiblabel, colour='green') if start_orig <= start <= stop <= stop_orig: plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, True, fiblabel, colour='green') else: print('(extrapolation SKIPPED) fibid:', fiblabel) elif operation['description'] == 'fit_through_user_points': fibid = operation['fibid'] fiblabel = fibid_with_box[fibid - 1] if args.verbose: print('(fit through user points) fibid:', fiblabel) poldeg = operation['poldeg'] start = operation['start'] stop = operation['stop'] xfit = [] yfit = [] for userpoint in operation['user_points']: # assume x, y coordinates in JSON file are given in # image coordinates, starting at (1,1) in the lower # left corner xdum = userpoint['x'] - 1 # use np.array coordinates ydum = userpoint['y'] - 1 # use np.array coordinates xfit.append(xdum) yfit.append(ydum) xfit = np.array(xfit) yfit = np.array(yfit) if len(xfit) <= poldeg: raise ValueError('Insufficient number of points to fit' ' polynomial') poly, residum = polfit_residuals(xfit, yfit, poldeg) coeff = poly.coef plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, args.fibids, fiblabel, colour='green') bigdict['contents'][fibid - 1]['start'] = start bigdict['contents'][fibid - 1]['stop'] = stop bigdict['contents'][fibid - 1]['fitparms'] = coeff.tolist() elif operation['description'] == \ 'extrapolation_through_user_points': fibid = operation['fibid'] fiblabel = fibid_with_box[fibid - 1] if args.verbose: print('(extrapolation_through_user_points):', fiblabel) start_reuse = operation['start_reuse'] stop_reuse = operation['stop_reuse'] resampling = operation['resampling'] poldeg = operation['poldeg'] start = operation['start'] stop = operation['stop'] coeff = bigdict['contents'][fibid - 1]['fitparms'] xfit = np.linspace(start_reuse, stop_reuse, num=resampling) poly = np.polynomial.Polynomial(coeff) yfit = poly(xfit) for userpoint in operation['user_points']: # assume x, y coordinates in JSON file are given in # image coordinates, starting at (1,1) in the lower # left corner xdum = userpoint['x'] - 1 # use np.array coordinates ydum = userpoint['y'] - 1 # use np.array coordinates xfit = np.concatenate((xfit, np.array([xdum]))) yfit = np.concatenate((yfit, np.array([ydum]))) poly, residum = polfit_residuals(xfit, yfit, poldeg) coeff = poly.coef if start < start_reuse: plot_trace(ax, coeff, start, start_reuse, ix_offset, args.rawimage, args.fibids, fiblabel, colour='green') if stop_reuse < stop: plot_trace(ax, coeff, stop_reuse, stop, ix_offset, args.rawimage, args.fibids, fiblabel, colour='green') bigdict['contents'][fibid - 1]['start'] = start bigdict['contents'][fibid - 1]['stop'] = stop bigdict['contents'][fibid - 1]['fitparms'] = coeff.tolist() elif operation['description'] == 'sandwich': fibid = operation['fibid'] fiblabel = fibid_with_box[fibid - 1] if args.verbose: print('(sandwich) fibid:', fiblabel) fraction = operation['fraction'] nf1, nf2 = operation['neighbours'] start = operation['start'] stop = operation['stop'] tmpf1 = bigdict['contents'][nf1 - 1] tmpf2 = bigdict['contents'][nf2 - 1] if nf1 != tmpf1['fibid'] or nf2 != tmpf2['fibid']: raise ValueError( "Unexpected fiber numbers in neighbours" ) coefff1 = np.array(tmpf1['fitparms']) coefff2 = np.array(tmpf2['fitparms']) coeff = coefff1 + fraction * (coefff2 - coefff1) plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, args.fibids, fiblabel, colour='green') # update values in bigdict (JSON structure) bigdict['contents'][fibid - 1]['start'] = start bigdict['contents'][fibid - 1]['stop'] = stop bigdict['contents'][fibid - 1][ 'fitparms'] = coeff.tolist() if fibid in bigdict['error_fitting']: bigdict['error_fitting'].remove(fibid) elif operation['description'] == 'renumber_fibids_within_box': fibid_ini = operation['fibid_ini'] fibid_end = operation['fibid_end'] box_ini = fibid_with_box[fibid_ini - 1][4:] box_end = fibid_with_box[fibid_end - 1][4:] if box_ini != box_end: print('ERROR: box_ini={}, box_end={}'.format(box_ini, box_end)) raise ValueError('fibid_ini and fibid_end correspond to ' 'different fiber boxes') fibid_shift = operation['fibid_shift'] if fibid_shift in [-1, 1]: if fibid_shift == -1: i_start = fibid_ini i_stop = fibid_end + 1 i_step = 1 else: i_start = fibid_end i_stop = fibid_ini - 1 i_step = -1 for fibid in range(i_start, i_stop, i_step): fiblabel_ori = fibid_with_box[fibid - 1] fiblabel_new = fibid_with_box[fibid - 1 + fibid_shift] if args.verbose: print('(renumber_fibids) fibid:', fiblabel_ori, '-->', fiblabel_new) bigdict['contents'][fibid -1 + fibid_shift] = \ deepcopy(bigdict['contents'][fibid -1]) bigdict['contents'][fibid -1 + fibid_shift]['fibid'] += \ fibid_shift # display updated trace coeff = \ bigdict['contents'][fibid -1 + fibid_shift]['fitparms'] start = \ bigdict['contents'][fibid -1 + fibid_shift]['start'] stop = bigdict['contents'][fibid -1 + fibid_shift]['stop'] plot_trace(ax, coeff, start, stop, ix_offset, args.rawimage, args.fibids, fiblabel_ori + '-->' + fiblabel_new, colour='green') if fibid_shift == -1: bigdict['contents'][fibid_end - 1]['fitparms'] = [] else: bigdict['contents'][fibid_ini - 1]['fitparms'] = [] else: raise ValueError('fibid_shift in operation ' 'renumber_fibids_within_box ' 'must be -1 or 1') else: raise ValueError('Unexpected healing method:', operation['description']) # update trace map if args.updated_traces is not None: # avoid overwritting initial JSON file if args.updated_traces.name != args.traces_file.name: # new random uuid for the updated calibration bigdict['uuid'] = str(uuid4()) with open(args.updated_traces.name, 'w') as outfile: json.dump(bigdict, outfile, indent=2) if pdf is not None: pdf.savefig() pdf.close() else: pause_debugplot(12, pltshow=True, tight_layout=True)
def main(args=None): # parse command-line options parser = argparse.ArgumentParser() # positional arguments parser.add_argument("fitsfile", help="FITS file name to be displayed", type=argparse.FileType('rb')) parser.add_argument("--bounddict", required=True, help="bounddict file name", type=argparse.FileType('rt')) parser.add_argument("--tuple_slit_numbers", required=True, help="Tuple n1[,n2[,step]] to define slitlet numbers") # optional arguments parser.add_argument("--echo", help="Display full command line", action="store_true") args = parser.parse_args() if args.echo: print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n') # read slitlet numbers to be computed tmp_str = args.tuple_slit_numbers.split(",") if len(tmp_str) == 3: if int(tmp_str[0]) < 1: raise ValueError("Invalid slitlet number < 1") if int(tmp_str[1]) > EMIR_NBARS: raise ValueError("Invalid slitlet number > EMIR_NBARS") list_slitlets = range(int(tmp_str[0]), int(tmp_str[1]) + 1, int(tmp_str[2])) elif len(tmp_str) == 2: if int(tmp_str[0]) < 1: raise ValueError("Invalid slitlet number < 1") if int(tmp_str[1]) > EMIR_NBARS: raise ValueError("Invalid slitlet number > EMIR_NBARS") list_slitlets = range(int(tmp_str[0]), int(tmp_str[1]) + 1, 1) elif len(tmp_str) == 1: if int(tmp_str[0]) < 1: raise ValueError("Invalid slitlet number < 1") if int(tmp_str[0]) > EMIR_NBARS: raise ValueError("Invalid slitlet number > EMIR_NBARS") list_slitlets = [int(tmp_str[0])] else: raise ValueError("Invalid tuple for slitlet numbers") # read input FITS file hdulist = fits.open(args.fitsfile.name) image_header = hdulist[0].header image2d = hdulist[0].data hdulist.close() naxis1 = image_header['naxis1'] naxis2 = image_header['naxis2'] if image2d.shape != (naxis2, naxis1): raise ValueError("Unexpected error with NAXIS1, NAXIS2") # remove path from fitsfile sfitsfile = os.path.basename(args.fitsfile.name) # check that the FITS file has been obtained with EMIR instrument = image_header['instrume'] if instrument != 'EMIR': raise ValueError("INSTRUME keyword is not 'EMIR'!") # read GRISM, FILTER and ROTANG from FITS header grism = image_header['grism'] spfilter = image_header['filter'] rotang = image_header['rotang'] # display full image ax = ximshow(image2d=image2d, title=sfitsfile + "\ngrism=" + grism + ", filter=" + spfilter + ", rotang=" + str(round(rotang, 2)), image_bbox=(1, naxis1, 1, naxis2), show=False) # overplot boundaries for each slitlet for slitlet_number in list_slitlets: pol_lower_boundary, pol_upper_boundary, \ xmin_lower, xmax_lower, xmin_upper, xmax_upper, \ csu_bar_slit_center = \ get_boundaries(args.bounddict, slitlet_number) if (pol_lower_boundary is not None) and \ (pol_upper_boundary is not None): xp = np.linspace(start=xmin_lower, stop=xmax_lower, num=1000) yp = pol_lower_boundary(xp) ax.plot(xp, yp, 'g-') xp = np.linspace(start=xmin_upper, stop=xmax_upper, num=1000) yp = pol_upper_boundary(xp) ax.plot(xp, yp, 'b-') # slitlet label yc_lower = pol_lower_boundary(EMIR_NAXIS1 / 2 + 0.5) yc_upper = pol_upper_boundary(EMIR_NAXIS1 / 2 + 0.5) tmpcolor = ['r', 'b'][slitlet_number % 2] xcsu = EMIR_NAXIS1 * csu_bar_slit_center / 341.5 ax.text(xcsu, (yc_lower + yc_upper) / 2, str(slitlet_number), fontsize=10, va='center', ha='center', bbox=dict(boxstyle="round,pad=0.1", fc="white", ec="grey"), color=tmpcolor, fontweight='bold', backgroundcolor='white') # show plot pause_debugplot(12, pltshow=True)
def useful_mos_xpixels(reduced_mos_data, base_header, vpix_region, npix_removed_near_ohlines=0, list_valid_wvregions=None, debugplot=0): """Useful X-axis pixels removing +/- npixaround pixels around each OH line """ # get wavelength calibration from image header naxis1 = base_header['naxis1'] naxis2 = base_header['naxis2'] crpix1 = base_header['crpix1'] crval1 = base_header['crval1'] cdelt1 = base_header['cdelt1'] # check vertical region nsmin = int(vpix_region[0] + 0.5) nsmax = int(vpix_region[1] + 0.5) if nsmin > nsmax: raise ValueError('vpix_region values in wrong order') elif nsmin < 1 or nsmax > naxis2: raise ValueError('vpix_region outside valid range') # minimum and maximum pixels in the wavelength direction islitlet_min = get_islitlet(nsmin) ncmin = base_header['jmnslt{:02d}'.format(islitlet_min)] ncmax = base_header['jmxslt{:02d}'.format(islitlet_min)] islitlet_max = get_islitlet(nsmax) if islitlet_max > islitlet_min: for islitlet in range(islitlet_min + 1, islitlet_max + 1): ncmin_ = base_header['jmnslt{:02d}'.format(islitlet)] ncmax_ = base_header['jmnslt{:02d}'.format(islitlet)] ncmin = min(ncmin, ncmin_) ncmax = max(ncmax, ncmax_) # pixels within valid regions xisok_wvreg = np.zeros(naxis1, dtype='bool') if list_valid_wvregions is None: for ipix in range(ncmin, ncmax + 1): xisok_wvreg[ipix - 1] = True else: for wvregion in list_valid_wvregions: wvmin = float(wvregion[0]) wvmax = float(wvregion[1]) if wvmin > wvmax: raise ValueError('wvregion values in wrong order:' ' {}, {}'.format(wvmin, wvmax)) minpix = int((wvmin - crval1) / cdelt1 + crpix1 + 0.5) maxpix = int((wvmax - crval1) / cdelt1 + crpix1 + 0.5) for ipix in range(minpix, maxpix + 1): if 1 <= ipix <= naxis1: xisok_wvreg[ipix - 1] = True if np.sum(xisok_wvreg) < 1: raise ValueError('no valid wavelength ranges provided') # pixels affected by OH lines xisok_oh = np.ones(naxis1, dtype='bool') if int(npix_removed_near_ohlines) > 0: dumdata = pkgutil.get_data( 'emirdrp.instrument.configs', 'Oliva_etal_2013.dat' ) oh_lines_tmpfile = StringIO(dumdata.decode('utf8')) catlines = np.genfromtxt(oh_lines_tmpfile) catlines_all_wave = np.concatenate( (catlines[:, 1], catlines[:, 0])) for waveline in catlines_all_wave: expected_pixel = int( (waveline - crval1) / cdelt1 + crpix1 + 0.5) minpix = expected_pixel - int(npix_removed_near_ohlines) maxpix = expected_pixel + int(npix_removed_near_ohlines) for ipix in range(minpix, maxpix + 1): if 1 <= ipix <= naxis1: xisok_oh[ipix - 1] = False # pixels in valid regions not affected by OH lines xisok = np.logical_and(xisok_wvreg, xisok_oh) naxis1_effective = np.sum(xisok) if naxis1_effective < 1: raise ValueError('no valid wavelength range available after ' 'removing OH lines') if abs(debugplot) in [21, 22]: slitlet2d = reduced_mos_data[(nsmin - 1):nsmax, :].copy() ximshow(slitlet2d, title='Rectified region', first_pixel=(1, nsmin), crval1=crval1, cdelt1=cdelt1, debugplot=debugplot) ax = ximshow(slitlet2d, title='Rectified region\nafter blocking ' 'removed wavelength ranges', first_pixel=(1, nsmin), crval1=crval1, cdelt1=cdelt1, show=False) for idum in range(1, naxis1 + 1): if not xisok_wvreg[idum - 1]: ax.plot([idum, idum], [nsmin, nsmax], 'g-') pause_debugplot(debugplot, pltshow=True) ax = ximshow(slitlet2d, title='Rectified slitlet\nuseful regions after ' 'removing OH lines', first_pixel=(1, nsmin), crval1=crval1, cdelt1=cdelt1, show=False) for idum in range(1, naxis1 + 1): if not xisok[idum - 1]: ax.plot([idum, idum], [nsmin, nsmax], 'm-') pause_debugplot(debugplot, pltshow=True) return xisok