class WavelengthCalibrationRecipe(EmirRecipe): """Recipe to calibrate the spectral response. **Observing modes:** * Wavelength calibration (4.5) **Inputs:** * List of line positions * Calibrations up to spectral flatfielding **Outputs:** * Wavelength calibration structure **Procedure:** * TBD """ master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_spectral_ff = reqs.MasterSpectralFlatFieldRequirement() cal = Result(prods.WavelengthCalibration) def run(self, rinput): return self.create_result(cal=prods.WavelengthCalibration())
class SkySpecRecipe(EmirRecipe): """Recipe to process data taken in spectral sky mode. """ obresult = ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() skyspec = Product(prods.SkySpectrum) def run(self, rinput): self.logger.info('starting spectral sky reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=True) hdr = hdulist[0].header self.set_base_headers(hdr) self.logger.info('end sky spectral reduction') result = self.create_result(skyspec=hdulist) return result
class StareSpectraRecipe(EmirRecipe): """Process images in Stare spectra mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_sky = reqs.SpectralSkyRequirement(optional=True) stare = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting stare spectra reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median) hdr = hdulist[0].header self.set_base_headers(hdr) # Update EXP to 0 hdr['EXP'] = 0 self.logger.info('end stare spectra reduction') result = self.create_result(stare=hdulist) return result
class SkySpecRecipe(EmirRecipe): """Recipe to process data taken in spectral sky mode. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() skyspec = Result(prods.SkySpectrum) reduced_image = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting spectral sky reduction') flow = self.init_filters(rinput) reduced_image = basic_processing_with_combination( rinput, flow, method=median, errors=True ) hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength calibration # coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv ) # save as JSON file in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # generate associated ds9 region files and save them in work directory if self.intermediate_results: save_four_ds9(rectwv_coeff) # apply rectification and wavelength calibration skyspec = apply_rectwv_coeff( reduced_image, rectwv_coeff ) self.logger.info('end sky spectral reduction') result = self.create_result( reduced_image=reduced_image, skyspec=skyspec ) return result
class SpecFlatLowFreq(EmirRecipe): """Process continuum exposures of continuum lamp (lamp ON-OFF) """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True) master_rectwv = reqs.MasterRectWaveRequirement(optional=True) # note that 'sum' is not allowed as combination method method = Parameter('sigmaclip', description='Combination method', choices=['mean', 'median', 'sigmaclip']) method_kwargs = Parameter(dict(), description='Arguments for combination method', optional=True) minimum_slitlet_width_mm = Parameter( float(EMIR_MINIMUM_SLITLET_WIDTH_MM), description='Minimum width (mm) for a valid slitlet', optional=True) maximum_slitlet_width_mm = Parameter( float(EMIR_MAXIMUM_SLITLET_WIDTH_MM), description='Maximum width (mm) for a valid slitlet', optional=True) global_integer_offset_x_pix = Parameter( 0, description='Global offset (pixels) in wavelength direction (integer)', optional=True) global_integer_offset_y_pix = Parameter( 0, description='Global offset (pixels) in spatial direction (integer)', optional=True) nwindow_x_median = Parameter( 301, description='Window size to smooth the flatfield in the spectral ' 'direction', optional=True) nwindow_y_median = Parameter( 11, description='Window size to smooth the flatfield in the spatial ' 'direction', optional=True) minimum_fraction = Parameter( 0.01, description='Minimum useful flatfielding value (fraction of maximum)', optional=True) minimum_value_in_output = Parameter( 0.01, description='Minimum value allowed in output: pixels below this value' ' are set to 1.0', optional=True) maximum_value_in_output = Parameter( 10.0, description='Maximum value allowed in output: pixels above this value' ' are set to 1.0', optional=True) debugplot = Parameter(0, description='Debugger parameter', optional=True) reduced_flatlowfreq = Result(prods.MasterSpectralFlat) def run(self, rinput): self.logger.info('starting generation of flatlowfreq') self.logger.info('rectwv_coeff..........................: {}'.format( rinput.rectwv_coeff)) self.logger.info('master_rectwv.........................: {}'.format( rinput.master_rectwv)) self.logger.info('Minimum slitlet width (mm)............: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info('Maximum slitlet width (mm)............: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info('Global offset X direction (pixels)....: {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info('Global offset Y direction (pixels)....: {}'.format( rinput.global_integer_offset_y_pix)) self.logger.info('nwindow_x_median......................: {}'.format( rinput.nwindow_x_median)) self.logger.info('nwindow_y_median......................: {}'.format( rinput.nwindow_y_median)) self.logger.info('Minimum fraction......................: {}'.format( rinput.minimum_fraction)) self.logger.info('Minimum value in output...............: {}'.format( rinput.minimum_value_in_output)) self.logger.info('Maximum value in output...............: {}'.format( rinput.maximum_value_in_output)) # check rectification and wavelength calibration information if rinput.master_rectwv is None and rinput.rectwv_coeff is None: raise ValueError('No master_rectwv nor rectwv_coeff data have ' 'been provided') elif rinput.master_rectwv is not None and \ rinput.rectwv_coeff is not None: self.logger.warning('rectwv_coeff will be used instead of ' 'master_rectwv') if rinput.rectwv_coeff is not None and \ (rinput.global_integer_offset_x_pix != 0 or rinput.global_integer_offset_y_pix != 0): raise ValueError('global_integer_offsets cannot be used ' 'simultaneously with rectwv_coeff') # check headers to detect lamp status (on/off) list_lampincd = [] for fname in rinput.obresult.frames: with fname.open() as f: list_lampincd.append(f[0].header['lampincd']) # check number of images nimages = len(rinput.obresult.frames) n_on = list_lampincd.count(1) n_off = list_lampincd.count(0) self.logger.info( 'Number of images with lamp ON.........: {}'.format(n_on)) self.logger.info( 'Number of images with lamp OFF........: {}'.format(n_off)) self.logger.info( 'Total number of images................: {}'.format(nimages)) if n_on == 0: raise ValueError('Insufficient number of images with lamp ON') if n_on + n_off != nimages: raise ValueError('Number of images does not match!') # check combination method if rinput.method_kwargs == {}: method_kwargs = None else: if rinput.method == 'sigmaclip': method_kwargs = rinput.method_kwargs else: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # build object to proceed with bpm, bias, and dark (not flat) flow = self.init_filters(rinput) # available combination methods method = getattr(combine, rinput.method) # basic reduction of images with lamp ON or OFF lampmode = {0: 'off', 1: 'on'} reduced_image_on = None reduced_image_off = None for imode in lampmode.keys(): self.logger.info('starting basic reduction of images with' ' lamp {}'.format(lampmode[imode])) tmplist = [ rinput.obresult.frames[i] for i, lampincd in enumerate(list_lampincd) if lampincd == imode ] if len(tmplist) > 0: with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in tmplist ] reduced_image = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) if imode == 0: reduced_image_off = flow(reduced_image) hdr = reduced_image_off[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced_image_off, 'reduced_image_off.fits') elif imode == 1: reduced_image_on = flow(reduced_image) hdr = reduced_image_on[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced_image_on, 'reduced_image_on.fits') else: raise ValueError('Unexpected imode={}'.format(imode)) # computation of ON-OFF header_on = reduced_image_on[0].header data_on = reduced_image_on[0].data.astype('float32') if n_off > 0: header_off = reduced_image_off[0].header data_off = reduced_image_off[0].data.astype('float32') else: header_off = None data_off = np.zeros_like(data_on) reduced_data = data_on - data_off # update reduced image header reduced_image = self.create_reduced_image(rinput, reduced_data, header_on, header_off, list_lampincd) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # define rectification and wavelength calibration coefficients if rinput.rectwv_coeff is None: rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv) # set global offsets rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix else: rectwv_coeff = rinput.rectwv_coeff # save as JSON in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # apply global offsets (to both, the original and the cleaned version) image2d = apply_integer_offsets( image2d=reduced_data, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix) # load CSU configuration csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header) # determine (pseudo) longslits dict_longslits = csu_conf.pseudo_longslits() # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: self.logger.info('-> Removing slitlet (not defined): ' + str(idel)) list_valid_islitlets.remove(idel) # filter out slitlets with widths outside valid range list_outside_valid_width = [] for islitlet in list_valid_islitlets: slitwidth = csu_conf.csu_bar_slit_width(islitlet) if (slitwidth < rinput.minimum_slitlet_width_mm) or \ (slitwidth > rinput.maximum_slitlet_width_mm): list_outside_valid_width.append(islitlet) self.logger.info('-> Removing slitlet (width out of range): ' + str(islitlet)) if len(list_outside_valid_width) > 0: for idel in list_outside_valid_width: list_valid_islitlets.remove(idel) # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop grism_name = rectwv_coeff.tags['grism'] filter_name = rectwv_coeff.tags['filter'] cout = '0' debugplot = rinput.debugplot for islitlet in list(range(1, EMIR_NBARS + 1)): if islitlet in list_valid_islitlets: # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=debugplot) if abs(slt.debugplot) > 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d, subtitle='original image') # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d=slitlet2d, resampling=2, subtitle='original (cleaned) rectified') naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") # get useful slitlet region (use boundaries) spectrail = slt.list_spectrails[0] yy0 = slt.corr_yrect_a + \ slt.corr_yrect_b * spectrail(slt.x0_reference) ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig spectrail = slt.list_spectrails[2] yy0 = slt.corr_yrect_a + \ slt.corr_yrect_b * spectrail(slt.x0_reference) ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction sp_median = ndimage.median_filter(sp_collapsed, 5, mode='nearest') ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * rinput.minimum_fraction lremove = np.where(sp_median < y_threshold) sp_median[lremove] = 0.0 if abs(slt.debugplot) % 10 != 0: xaxis1 = np.arange(1, naxis1_slitlet2d + 1) title = 'Slitlet#' + str(islitlet) + ' (median spectrum)' ax = ximplotxy(xaxis1, sp_collapsed, title=title, show=False, **{'label': 'collapsed spectrum'}) ax.plot(xaxis1, sp_median, label='fitted spectrum') ax.plot([1, naxis1_slitlet2d], 2 * [y_threshold], label='threshold') # ax.plot(xknots, yknots, 'o', label='knots') ax.legend() ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian) pause_debugplot(slt.debugplot, pltshow=True, tight_layout=True) # generate rectified slitlet region filled with the # median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(slt.debugplot) % 10 != 0: slt.ximshow_rectified( slitlet2d_rect=slitlet2d_rect_spmedian, subtitle='rectified, filled with median spectrum') # compute smooth surface # clipped region slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy() slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0 slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0 # unrectified clipped image slitlet2d_unrect_clipped = slt.rectify( slitlet2d=slitlet2d_rect_clipped, resampling=2, inverse=True, subtitle='unrectified, filled with median spectrum ' '(clipped)') # normalize initial slitlet image (avoid division by zero) slitlet2d_norm_clipped = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_clipped[i, j] if den == 0: slitlet2d_norm_clipped[i, j] = 1.0 else: slitlet2d_norm_clipped[i, j] = \ slitlet2d[i, j] / den # set to 1.0 one additional pixel at each side (since # 'den' above is small at the borders and generates wrong # bright pixels) slitlet2d_norm_clipped = fix_pix_borders( image2d=slitlet2d_norm_clipped, nreplace=1, sought_value=1.0, replacement_value=1.0) slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose() slitlet2d_norm_clipped = fix_pix_borders( image2d=slitlet2d_norm_clipped, nreplace=1, sought_value=1.0, replacement_value=1.0) slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose() slitlet2d_norm_smooth = ndimage.median_filter( slitlet2d_norm_clipped, size=(rinput.nwindow_y_median, rinput.nwindow_x_median), mode='nearest') if abs(slt.debugplot) % 10 != 0: slt.ximshow_unrectified( slitlet2d=slitlet2d_norm_clipped, subtitle='unrectified, pixel-to-pixel (clipped)') slt.ximshow_unrectified( slitlet2d=slitlet2d_norm_smooth, subtitle='unrectified, pixel-to-pixel (smoothed)') # --- # check for (pseudo) longslit with previous and next slitlet imin = dict_longslits[islitlet].imin() imax = dict_longslits[islitlet].imax() if islitlet > 1: same_slitlet_below = (islitlet - 1) >= imin else: same_slitlet_below = False if islitlet < EMIR_NBARS: same_slitlet_above = (islitlet + 1) <= imax else: same_slitlet_above = False for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm_smooth[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers if not same_slitlet_below: image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 if not same_slitlet_above: image2d_flatfielded[(n2 - 5):n2, j] = 1 cout += '.' else: cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) self.logger.info(cout) # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded, offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix) # set pixels below minimum value to 1.0 filtered = np.where( image2d_flatfielded < rinput.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # set pixels above maximum value to 1.0 filtered = np.where( image2d_flatfielded > rinput.maximum_value_in_output) image2d_flatfielded[filtered] = 1.0 # update image header reduced_flatlowfreq = self.create_reduced_image( rinput, image2d_flatfielded, header_on, header_off, list_lampincd) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # save results in results directory self.logger.info('end of flatlowfreq generation') result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq) return result def create_reduced_image(self, rinput, reduced_data, header_on, header_off, list_lampincd): with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] # Copy header of first image base_header = hduls[0][0].header.copy() hdu = fits.PrimaryHDU(reduced_data, header=base_header) self.set_base_headers(hdu.header) self.logger.debug('update result header') # update additional keywords hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = 'flatlowfreq' hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2'] hdu.header['history'] = "Processed flatlowfreq" hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames') # update history dm = emirdrp.datamodel.EmirDataModel() for img, lampincd in zip(hduls, list_lampincd): imgid = dm.get_imgid(img) hdu.header['HISTORY'] = "Image '{}' has lampincd='{}'".format( imgid, lampincd) hdu.header['HISTORY'] = "Processed flatlowfreq" hdu.header['HISTORY'] = '--- Reduction of images with lamp ON ---' for line in header_on['HISTORY']: hdu.header['HISTORY'] = line if header_off is not None: hdu.header['HISTORY'] = '--- Reduction of images with lamp OFF ---' for line in header_off['HISTORY']: hdu.header['HISTORY'] = line hdu.header.add_history('--- rinput.stored() (BEGIN) ---') for item in rinput.stored(): value = getattr(rinput, item) cline = '{}: {}'.format(item, value) hdu.header.add_history(cline) hdu.header.add_history('--- rinput.stored() (END) ---') result = fits.HDUList([hdu]) return result def set_base_headers(self, hdr): newhdr = super(SpecFlatLowFreq, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class ABBASpectraFastRectwv(EmirRecipe): """Process AB or ABBA images applying a single wavelength calibration Note that the wavelength calibration has already been determined. """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement() pattern = Parameter('ABBA', description='Observation pattern', choices=['AB', 'ABBA']) # do not allow 'sum' as combination method in order to avoid # confusion with number of counts in resulting image method = Parameter('mean', description='Combination method', choices=['mean', 'median', 'sigmaclip']) method_kwargs = Parameter(dict(), description='Arguments for combination method') voffset_pix = Parameter(0.0, description='Shift (pixels) to move A into B', optional=True) reduced_mos_abba = Result(prods.ProcessedMOS) reduced_mos_abba_combined = Result(prods.ProcessedMOS) def run(self, rinput): nimages = len(rinput.obresult.frames) pattern = rinput.pattern pattern_length = len(pattern) # check combination method if rinput.method != 'sigmaclip': if rinput.method_kwargs != {}: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # check pattern sequence matches number of images if nimages % pattern_length != 0: raise ValueError('Number of images is not a multiple of pattern ' 'length: {}, {}'.format(nimages, pattern_length)) nsequences = nimages // pattern_length rectwv_coeff = rinput.rectwv_coeff self.logger.info(rectwv_coeff) self.logger.info('observation pattern: {}'.format(pattern)) self.logger.info('nsequences.........: {}'.format(nsequences)) full_set = pattern * nsequences self.logger.info('full set of images.: {}'.format(full_set)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # available combination methods method = getattr(combine, rinput.method) method_kwargs = rinput.method_kwargs # basic reduction of A images list_a = [ rinput.obresult.frames[i] for i, char in enumerate(full_set) if char == 'A' ] with contextlib.ExitStack() as stack: self.logger.info('starting basic reduction of A images') hduls = [stack.enter_context(fname.open()) for fname in list_a] reduced_image_a = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) reduced_image_a = flow(reduced_image_a) hdr = reduced_image_a[0].header self.set_base_headers(hdr) # basic reduction of B images list_b = [ rinput.obresult.frames[i] for i, char in enumerate(full_set) if char == 'B' ] with contextlib.ExitStack() as stack: self.logger.info('starting basic reduction of B images') hduls = [stack.enter_context(fname.open()) for fname in list_b] reduced_image_b = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) reduced_image_b = flow(reduced_image_b) hdr = reduced_image_b[0].header self.set_base_headers(hdr) # save intermediate reduced_image_a and reduced_image_b self.save_intermediate_img(reduced_image_a, 'reduced_image_a.fits') self.save_intermediate_img(reduced_image_b, 'reduced_image_b.fits') # computation of A-B header_a = reduced_image_a[0].header header_b = reduced_image_b[0].header data_a = reduced_image_a[0].data.astype('float32') data_b = reduced_image_b[0].data.astype('float32') reduced_data = data_a - data_b # update reduced image header reduced_image = self.create_reduced_image( rinput, reduced_data, header_a, header_b, rinput.pattern, full_set, voffset_pix=0, header_mos_abba=None, ) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # apply rectification and wavelength calibration self.logger.info('begin rect.+wavecal. reduction of ABBA spectra') reduced_mos_abba = apply_rectwv_coeff(reduced_image, rectwv_coeff) header_mos_abba = reduced_mos_abba[0].header # combine A and B data by shifting B on top of A voffset_pix = rinput.voffset_pix if voffset_pix is not None and voffset_pix != 0: self.logger.info( 'correcting vertical offset (pixesl): {}'.format(voffset_pix)) reduced_mos_abba_data = reduced_mos_abba[0].data.astype('float32') shifted_a_minus_b_data = shift_image2d( reduced_mos_abba_data, yoffset=-voffset_pix, ).astype('float32') reduced_mos_abba_combined_data = \ reduced_mos_abba_data - shifted_a_minus_b_data # scale signal to exposure of a single image reduced_mos_abba_combined_data /= 2.0 else: reduced_mos_abba_combined_data = None # update reduced combined image header reduced_mos_abba_combined = self.create_reduced_image( rinput, reduced_mos_abba_combined_data, header_a, header_b, rinput.pattern, full_set, voffset_pix, header_mos_abba=header_mos_abba) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos_abba, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of ABBA spectra') result = self.create_result( reduced_mos_abba=reduced_mos_abba, reduced_mos_abba_combined=reduced_mos_abba_combined) return result def create_reduced_image(self, rinput, reduced_data, header_a, header_b, pattern, full_set, voffset_pix, header_mos_abba): with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] # Copy header of first image base_header = hduls[0][0].header.copy() hdu = fits.PrimaryHDU(reduced_data, header=base_header) self.set_base_headers(hdu.header) self.logger.debug('update result header') if header_mos_abba is not None: self.logger.debug('update result header') crpix1 = header_mos_abba['crpix1'] crval1 = header_mos_abba['crval1'] cdelt1 = header_mos_abba['cdelt1'] # update wavelength calibration in FITS header for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']: if keyword in hdu.header: hdu.header.remove(keyword) hdu.header['crpix1'] = (crpix1, 'reference pixel') hdu.header['crval1'] = (crval1, 'central wavelength at crpix1') hdu.header['cdelt1'] = \ (cdelt1, 'linear dispersion (Angstrom/pixel)') hdu.header['cunit1'] = ('Angstrom', 'units along axis1') hdu.header['ctype1'] = 'WAVELENGTH' hdu.header['crpix2'] = (0.0, 'reference pixel') hdu.header['crval2'] = (0.0, 'central value at crpix2') hdu.header['cdelt2'] = (1.0, 'increment') hdu.header['ctype2'] = 'PIXEL' hdu.header['cunit2'] = ('Pixel', 'units along axis2') for keyword in [ 'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2', 'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2' ]: if keyword in hdu.header: hdu.header.remove(keyword) # update additional keywords hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = pattern + ' pattern' hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2'] hdu.header['history'] = "Processed " + pattern + " pattern" hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames') # update history dm = emirdrp.datamodel.EmirDataModel() for img, key in zip(hduls, full_set): imgid = dm.get_imgid(img) hdu.header['HISTORY'] = "Image '{}' is '{}'".format(imgid, key) hdu.header['HISTORY'] = "Processed " + pattern + " pattern" hdu.header['HISTORY'] = '--- Reduction of A images ---' for line in header_a['HISTORY']: hdu.header['HISTORY'] = line hdu.header['HISTORY'] = '--- Reduction of B images ---' for line in header_b['HISTORY']: hdu.header['HISTORY'] = line if voffset_pix is not None and voffset_pix != 0: hdu.header['HISTORY'] = '--- Combination of AB spectra ---' hdu.header['HISTORY'] = "voffset_pix between A and B {}".format( voffset_pix) hdu.header.add_history('--- rinput.stored() (BEGIN) ---') for item in rinput.stored(): value = getattr(rinput, item) cline = '{}: {}'.format(item, value) hdu.header.add_history(cline) hdu.header.add_history('--- rinput.stored() (END) ---') result = fits.HDUList([hdu]) return result def set_base_headers(self, hdr): newhdr = super(ABBASpectraFastRectwv, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr def extract_tags_from_obsres(self, obsres, tag_keys): # this function is necessary to make use of recipe requirements # that are provided as lists final_tags = [] for frame in obsres.frames: ref_img = frame.open() tag = self.extract_tags_from_ref(ref_img, tag_keys, base=obsres.labels) final_tags.append(tag) return final_tags
class ABBASpectraRectwv(EmirRecipe): """Process images in AB or ABBA mode applying wavelength calibration Note that in this case the wavelength calibration has already been determined. """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True) list_rectwv_coeff = reqs.ListOfRectWaveCoeffRequirement(optional=True) pattern = Parameter('ABBA', description='Observation pattern', choices=['AB', 'ABBA']) # do not allow 'sum' as combination method in order to avoid # confusion with number of counts in resulting image method = Parameter('mean', description='Combination method', choices=['mean', 'median', 'sigmaclip']) method_kwargs = Parameter(dict(), description='Arguments for combination method', optional=True) refine_target_along_slitlet = Parameter( dict(), description='Parameters to refine location of target along the slitlet' ) reduced_mos_abba = Result(prods.ProcessedMOS) reduced_mos_abba_combined = Result(prods.ProcessedMOS) def run(self, rinput): nimages = len(rinput.obresult.frames) pattern = rinput.pattern pattern_length = len(pattern) # check combination method if rinput.method != 'sigmaclip': if rinput.method_kwargs != {}: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # check pattern sequence matches number of images if nimages % pattern_length != 0: raise ValueError('Number of images is not a multiple of pattern ' 'length: {}, {}'.format(nimages, pattern_length)) nsequences = nimages // pattern_length # check rectification and wavelength calibration information if rinput.rectwv_coeff is None and rinput.list_rectwv_coeff is None: raise ValueError('No rectwv_coeff nor list_rectwv_coeff data have ' 'been provided') elif rinput.rectwv_coeff is not None and \ rinput.list_rectwv_coeff is not None: raise ValueError("rectwv_coeff and list_rectwv_coeff cannot be " "used simultaneously") elif rinput.rectwv_coeff is not None: list_rectwv_coeff = [rinput.rectwv_coeff] * nimages elif rinput.list_rectwv_coeff is not None: if len(rinput.list_rectwv_coeff) != nimages: raise ValueError("Unexpected number of rectwv_coeff files " "in list_rectwv_coeff") else: list_rectwv_coeff = rinput.list_rectwv_coeff # check filter and grism are the same in all JSON files for item in ['grism', 'filter']: list_values = [] for calib in list_rectwv_coeff: list_values.append(calib.tags[item]) if len(set(list_values)) != 1: raise ValueError( 'list_rectwv_coeff contains coefficients for ' 'different {}s'.format(item)) else: raise ValueError("Unexpected error!") # grism and filter names grism_name = list_rectwv_coeff[0].tags['grism'] filter_name = list_rectwv_coeff[0].tags['filter'] # compute offsets from WCS info in image headers with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] sep_arcsec, spatial_scales = compute_wcs_offsets(hduls) sep_pixel = np.round(sep_arcsec / spatial_scales, 6) for i in range(nimages): self.logger.info(list_rectwv_coeff[i]) self.logger.info( 'observation pattern..................: {}'.format(pattern)) self.logger.info( 'nsequences...........................: {}'.format(nsequences)) self.logger.info( 'offsets (arcsec, from WCS)...........: {}'.format(sep_arcsec)) self.logger.info( 'spatial scales (arcsec/pix, from WCS): {}'.format(spatial_scales)) self.logger.info( 'spatial scales (pixels, from WCS)....: {}'.format(sep_pixel)) full_set = pattern * nsequences self.logger.info( 'full set of images...................: {}'.format(full_set)) # basic parameters to determine useful pixels in the wavelength # direction dict_rtas = rinput.refine_target_along_slitlet valid_keys = [ 'npix_removed_near_ohlines', 'nwidth_medfilt', 'save_individual_images', 'ab_different_target', 'vpix_region_a_target', 'vpix_region_a_sky', 'vpix_region_b_target', 'vpix_region_b_sky', 'list_valid_wvregions_a', 'list_valid_wvregions_b' ] for dumkey in dict_rtas.keys(): if dumkey not in valid_keys: raise ValueError('Unexpected key={}'.format(dumkey)) if 'vpix_region_a_target' in dict_rtas.keys(): vpix_region_a_target = dict_rtas['vpix_region_a_target'] else: vpix_region_a_target = None if 'vpix_region_a_sky' in dict_rtas.keys(): vpix_region_a_sky = dict_rtas['vpix_region_a_sky'] else: vpix_region_a_sky = None if 'vpix_region_b_target' in dict_rtas.keys(): vpix_region_b_target = dict_rtas['vpix_region_b_target'] else: vpix_region_b_target = None if 'vpix_region_b_sky' in dict_rtas.keys(): vpix_region_b_sky = dict_rtas['vpix_region_b_sky'] else: vpix_region_b_sky = None if 'ab_different_target' in dict_rtas.keys(): ab_different_target = int(dict_rtas['ab_different_target']) if ab_different_target not in [-1, 0, 1, 9]: raise ValueError('Invalid ab_different_target={} value'.format( ab_different_target)) else: raise ValueError('Missing ab_different_target value') try: npix_removed_near_ohlines = \ int(dict_rtas['npix_removed_near_ohlines']) except KeyError: npix_removed_near_ohlines = 0 except ValueError: raise ValueError('wrong value: npix_removed_near_ohlines=' '{}'.format( dict_rtas['npix_removed_near_ohlines'])) if 'list_valid_wvregions_a' in dict_rtas.keys(): if vpix_region_a_target is None: raise ValueError('Unexpected list_valid_wvregions_a when ' 'vpix_region_a_target is not set') list_valid_wvregions_a = dict_rtas['list_valid_wvregions_a'] else: list_valid_wvregions_a = None if 'list_valid_wvregions_b' in dict_rtas.keys(): if vpix_region_b_target is None: raise ValueError('Unexpected list_valid_wvregions_b when ' 'vpix_region_b_target is not set') list_valid_wvregions_b = dict_rtas['list_valid_wvregions_b'] else: list_valid_wvregions_b = None try: nwidth_medfilt = int(dict_rtas['nwidth_medfilt']) except KeyError: nwidth_medfilt = 0 except ValueError: raise ValueError('wrong value: nwidth_medfilt={}'.format( dict_rtas['nwidth_medfilt'])) try: save_individual_images = int(dict_rtas['save_individual_images']) except KeyError: save_individual_images = 0 except ValueError: raise ValueError('wrong value: save_individual_images={}'.format( dict_rtas['save_individual_images'])) self.logger.info( 'npix_removed_near_ohlines: {}'.format(npix_removed_near_ohlines)) self.logger.info('nwidth_medfilt: {}'.format(nwidth_medfilt)) self.logger.info( 'vpix_region_a_target: {}'.format(vpix_region_a_target)) self.logger.info( 'vpix_region_b_target: {}'.format(vpix_region_b_target)) self.logger.info( 'list_valid_wvregions_a: {}'.format(list_valid_wvregions_a)) self.logger.info( 'list_valid_wvregions_b: {}'.format(list_valid_wvregions_b)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # basic reduction, rectification and wavelength calibration of # all the individual images list_reduced_mos_images = [] self.logger.info('starting reduction of individual images') for i, char in enumerate(full_set): frame = rinput.obresult.frames[i] self.logger.info('image {} ({} of {})'.format( char, i + 1, nimages)) self.logger.info('image: {}'.format(frame.filename)) with frame.open() as f: base_header = f[0].header.copy() data = f[0].data.astype('float32') grism_name_ = base_header['grism'] if grism_name_ != grism_name: raise ValueError('Incompatible grism name in ' 'rectwv_coeff.json file and FITS image') filter_name_ = base_header['filter'] if filter_name_ != filter_name: raise ValueError('Incompatible filter name in ' 'rectwv_coeff.json file and FITS image') hdu = fits.PrimaryHDU(data, header=base_header) hdu.header['UUID'] = str(uuid.uuid1()) hdul = fits.HDUList([hdu]) # basic reduction reduced_image = flow(hdul) hdr = reduced_image[0].header self.set_base_headers(hdr) # rectification and wavelength calibration reduced_mos_image = apply_rectwv_coeff(reduced_image, list_rectwv_coeff[i]) if save_individual_images != 0: self.save_intermediate_img( reduced_mos_image, 'reduced_mos_image_' + char + '_' + frame.filename[:10] + '.fits') list_reduced_mos_images.append(reduced_mos_image) # intermediate PDF file with crosscorrelation plots if self.intermediate_results: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages('crosscorrelation_ab.pdf') else: pdf = None # compute offsets between images self.logger.info('computing offsets between individual images') first_a = True first_b = True xisok_a = None xisok_b = None reference_profile_a = None reference_profile_b = None refine_a = vpix_region_a_target is not None refine_b = vpix_region_b_target is not None list_offsets = [] for i, char in enumerate(full_set): if char == 'A' and refine_a: refine_image = True elif char == 'B' and refine_b: refine_image = True else: refine_image = False if refine_image: frame = rinput.obresult.frames[i] isky = get_isky(i, pattern) frame_sky = rinput.obresult.frames[isky] self.logger.info('image {} ({} of {})'.format( char, i + 1, nimages)) self.logger.info('image: {}'.format(frame.filename)) self.logger.info('(sky): {}'.format(frame_sky.filename)) data = list_reduced_mos_images[i][0].data.copy() data_sky = list_reduced_mos_images[isky][0].data data -= data_sky base_header = list_reduced_mos_images[i][0].header # get useful pixels in the wavelength direction if char == 'A' and first_a: xisok_a = useful_mos_xpixels( data, base_header, vpix_region=vpix_region_a_target, npix_removed_near_ohlines=npix_removed_near_ohlines, list_valid_wvregions=list_valid_wvregions_a, debugplot=0) elif char == 'B' and first_b: xisok_b = useful_mos_xpixels( data, base_header, vpix_region=vpix_region_b_target, npix_removed_near_ohlines=npix_removed_near_ohlines, list_valid_wvregions=list_valid_wvregions_b, debugplot=0) if char == 'A': nsmin = vpix_region_a_target[0] nsmax = vpix_region_a_target[1] xisok = xisok_a if vpix_region_a_sky is not None: nsmin_sky = vpix_region_a_sky[0] nsmax_sky = vpix_region_a_sky[1] skysubtraction = True else: nsmin_sky = None nsmax_sky = None skysubtraction = False elif char == 'B': nsmin = vpix_region_b_target[0] nsmax = vpix_region_b_target[1] xisok = xisok_b if vpix_region_b_sky is not None: nsmin_sky = vpix_region_b_sky[0] nsmax_sky = vpix_region_b_sky[1] skysubtraction = True else: nsmin_sky = None nsmax_sky = None skysubtraction = False else: raise ValueError('Unexpected char value: {}'.format(char)) # initial slitlet region slitlet2d = data[(nsmin - 1):nsmax, :].copy() if skysubtraction: slitlet2d_sky = data[(nsmin_sky - 1):nsmax_sky, :].copy() median_sky = np.median(slitlet2d_sky, axis=0) slitlet2d -= median_sky # selected wavelength regions after blocking OH lines slitlet2d_blocked = slitlet2d[:, xisok] # apply median filter in the X direction if nwidth_medfilt > 1: slitlet2d_blocked_smoothed = ndimage.filters.median_filter( slitlet2d_blocked, size=(1, nwidth_medfilt)) else: slitlet2d_blocked_smoothed = slitlet2d_blocked # --- # convert to 1D series of spatial profiles (note: this # option does not work very well when the signal is low; # a simple mean profile does a better job) # profile = slitlet2d_blocked_smoothed.transpose().ravel() # --- profile = np.mean(slitlet2d_blocked_smoothed, axis=1) if char == 'A' and first_a: reference_profile_a = profile.copy() elif char == 'B' and first_b: reference_profile_b = profile.copy() # --- # # To write pickle file # import pickle # with open(frame.filename[:10] + '_profile.pkl', 'wb') as f: # pickle.dump(profile, f) # # To read pickle file # with open('test.pkl', 'rb') as f: # x = pickle.load(f) # --- # crosscorrelation to find offset naround_zero = (nsmax - nsmin) // 3 if char == 'A': reference_profile = reference_profile_a elif char == 'B': reference_profile = reference_profile_b else: raise ValueError('Unexpected char value: {}'.format(char)) offset, fpeak = periodic_corr1d( sp_reference=reference_profile, sp_offset=profile, remove_mean=False, frac_cosbell=0.10, zero_padding=11, fminmax=None, nfit_peak=5, naround_zero=naround_zero, sp_label='spatial profile', plottitle='Image #{} (type {}), {}'.format( i + 1, char, frame.filename[:10]), pdf=pdf) # round to 4 decimal places if abs(offset) < 1E-4: offset = 0.0 # avoid -0.0 else: offset = round(offset, 4) list_offsets.append(offset) # end of loop if char == 'A' and first_a: first_a = False elif char == 'B' and first_b: first_b = False else: list_offsets.append(0.0) self.logger.info('computed offsets: {}'.format(list_offsets)) self.logger.info('correcting vertical offsets between individual ' 'images') list_a = [] list_b = [] for i, (char, offset) in enumerate(zip(full_set, list_offsets)): frame = rinput.obresult.frames[i] self.logger.info('image {} ({} of {})'.format( char, i + 1, nimages)) self.logger.info('image: {}'.format(frame.filename)) reduced_mos_image = list_reduced_mos_images[i] data = reduced_mos_image[0].data base_header = reduced_mos_image[0].header self.logger.info( 'correcting vertical offset (pixesl): {}'.format(offset)) if offset != 0: reduced_mos_image[0].data = shift_image2d( data, yoffset=-offset).astype('float32') base_header['HISTORY'] = 'Applying voffset_pix {}'.format(offset) if save_individual_images != 0: self.save_intermediate_img( reduced_mos_image, 'reduced_mos_image_refined_' + char + '_' + frame.filename[:10] + '.fits') # store reduced_mos_image if char == 'A': list_a.append(reduced_mos_image) elif char == 'B': list_b.append(reduced_mos_image) else: raise ValueError('Unexpected char value: {}'.format(char)) # combination method method = getattr(combine, rinput.method) method_kwargs = rinput.method_kwargs # final combination of A images self.logger.info('combining individual A images') reduced_mos_image_a = combine_imgs(list_a, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) self.save_intermediate_img(reduced_mos_image_a, 'reduced_mos_image_a.fits') # final combination of B images self.logger.info('combining individual B images') reduced_mos_image_b = combine_imgs(list_b, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) self.save_intermediate_img(reduced_mos_image_b, 'reduced_mos_image_b.fits') self.logger.info('mixing A and B spectra') header_a = reduced_mos_image_a[0].header header_b = reduced_mos_image_b[0].header data_a = reduced_mos_image_a[0].data.astype('float32') data_b = reduced_mos_image_b[0].data.astype('float32') reduced_mos_abba_data = data_a - data_b # update reduced mos image header reduced_mos_abba = self.create_mos_abba_image(rinput, dict_rtas, reduced_mos_abba_data, header_a, header_b, pattern, full_set, list_offsets, voffset_pix=0) # combine A and B data by shifting B on top of A if abs(ab_different_target) == 0: len_prof_a = len(reference_profile_a) len_prof_b = len(reference_profile_b) if len_prof_a == len_prof_b: reference_profile = reference_profile_a profile = reference_profile_b naround_zero = len_prof_a // 3 elif len_prof_a > len_prof_b: ndiff = len_prof_a - len_prof_b reference_profile = reference_profile_a profile = np.concatenate( (reference_profile_b, np.zeros(ndiff, dtype='float'))) naround_zero = len_prof_a // 3 else: ndiff = len_prof_b - len_prof_a reference_profile = np.concatenate( (reference_profile_a, np.zeros(ndiff, dtype='float'))) profile = reference_profile_b naround_zero = len_prof_b // 3 offset, fpeak = periodic_corr1d( sp_reference=reference_profile, sp_offset=profile, remove_mean=False, frac_cosbell=0.10, zero_padding=11, fminmax=None, nfit_peak=5, naround_zero=naround_zero, sp_label='spatial profile', plottitle='Comparison of A and B profiles', pdf=pdf) voffset_pix = vpix_region_b_target[0] - vpix_region_a_target[0] voffset_pix += offset elif abs(ab_different_target) == 1: # apply nominal offset (from WCS info) between first A # and first B voffset_pix = sep_pixel[1] * ab_different_target elif ab_different_target == 9: voffset_pix = None else: raise ValueError( 'Invalid ab_different_target={}'.format(ab_different_target)) # close output PDF file if pdf is not None: pdf.close() if voffset_pix is not None: self.logger.info( 'correcting vertical offset (pixesl): {}'.format(voffset_pix)) shifted_a_minus_b_data = shift_image2d( reduced_mos_abba_data, yoffset=-voffset_pix, ).astype('float32') reduced_mos_abba_combined_data = \ reduced_mos_abba_data - shifted_a_minus_b_data # scale signal to exposure of a single image reduced_mos_abba_combined_data /= 2.0 else: reduced_mos_abba_combined_data = None # update reduced mos combined image header reduced_mos_abba_combined = self.create_mos_abba_image( rinput, dict_rtas, reduced_mos_abba_combined_data, header_a, header_b, pattern, full_set, list_offsets, voffset_pix) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(list_rectwv_coeff[0]) save_spectral_lines_ds9(list_rectwv_coeff[0]) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos_abba, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of ABBA spectra') result = self.create_result( reduced_mos_abba=reduced_mos_abba, reduced_mos_abba_combined=reduced_mos_abba_combined) return result def create_mos_abba_image(self, rinput, dict_rtas, reduced_mos_abba_data, header_a, header_b, pattern, full_set, list_offsets, voffset_pix): with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] # Copy header of first image base_header = hduls[0][0].header.copy() hdu = fits.PrimaryHDU(reduced_mos_abba_data, header=base_header) self.set_base_headers(hdu.header) # check consistency of wavelength calibration paramenters for param in ['crpix1', 'crval1', 'cdelt1']: if header_a[param] != header_b[param]: raise ValueError( 'Headers of A and B images have different ' 'values of {}'.format(param)) self.logger.debug('update result header') crpix1 = header_a['crpix1'] crval1 = header_a['crval1'] cdelt1 = header_a['cdelt1'] # update wavelength calibration in FITS header for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']: if keyword in hdu.header: hdu.header.remove(keyword) hdu.header['crpix1'] = (crpix1, 'reference pixel') hdu.header['crval1'] = (crval1, 'central wavelength at crpix1') hdu.header['cdelt1'] = (cdelt1, 'linear dispersion (Angstrom/pixel)') hdu.header['cunit1'] = ('Angstrom', 'units along axis1') hdu.header['ctype1'] = 'WAVELENGTH' hdu.header['crpix2'] = (0.0, 'reference pixel') hdu.header['crval2'] = (0.0, 'central value at crpix2') hdu.header['cdelt2'] = (1.0, 'increment') hdu.header['ctype2'] = 'PIXEL' hdu.header['cunit2'] = ('Pixel', 'units along axis2') for keyword in [ 'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2', 'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2' ]: if keyword in hdu.header: hdu.header.remove(keyword) # update additional keywords hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = pattern + ' pattern' hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2'] hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames') # update history hdu.header['HISTORY'] = "Processed " + pattern + " pattern" hdu.header['HISTORY'] = '--- Reduction of A images ---' for line in header_a['HISTORY']: hdu.header['HISTORY'] = line hdu.header['HISTORY'] = '--- Reduction of B images ---' for line in header_b['HISTORY']: hdu.header['HISTORY'] = line hdu.header['HISTORY'] = '--- Combination of ABBA images ---' for key in dict_rtas: hdu.header['HISTORY'] = '{}: {}'.format(key, dict_rtas[key]) dm = emirdrp.datamodel.EmirDataModel() for img, key, offset in zip(hduls, full_set, list_offsets): imgid = dm.get_imgid(img) hdu.header['HISTORY'] = \ "Image '{}' is '{}', with voffset_pix {}".format( imgid, key, offset) if voffset_pix is not None and voffset_pix != 0: hdu.header['HISTORY'] = '--- Combination of AB spectra ---' hdu.header['HISTORY'] = "voffset_pix between A and B {}".format( voffset_pix) hdu.header.add_history('--- rinput.stored() (BEGIN) ---') for item in rinput.stored(): value = getattr(rinput, item) cline = '{}: {}'.format(item, value) hdu.header.add_history(cline) hdu.header.add_history('--- rinput.stored() (END) ---') result = fits.HDUList([hdu]) return result def set_base_headers(self, hdr): newhdr = super(ABBASpectraRectwv, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr def extract_tags_from_obsres(self, obsres, tag_keys): # this function is necessary to make use of recipe requirements # that are provided as lists final_tags = [] for frame in obsres.frames: ref_img = frame.open() tag = self.extract_tags_from_ref(ref_img, tag_keys, base=obsres.labels) final_tags.append(tag) return final_tags
class StareSpectraWaveRecipe(EmirRecipe): """Process images in Stare spectra at the GTC. This recipe is intended to be used at GTC. The rectification and wavelength calibration can computed from a model if this model (master_rectwv) is provided as input. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement(optional=True) master_sky = reqs.SpectralSkyRequirement(optional=True) reduced_image = Result(prods.ProcessedImage) stare = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting reduction of stare spectra') self.logger.info(rinput.master_rectwv) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # rectification and wavelength calibration (if a model has # been provided) if rinput.master_rectwv: # RectWaveCoeff object with rectification and wavelength # calibration coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv) # apply rectification and wavelength calibration stare_image = apply_rectwv_coeff(reduced_image, rectwv_coeff) # save as JSON file in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(stare_image, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # image_wl_calibrated = True else: stare_image = reduced_image self.logger.info('No wavelength calibration provided') grism_value = hdr.get('GRISM', 'unknown') self.logger.debug('GRISM is %s', grism_value) if grism_value.lower() == 'open': self.logger.debug('GRISM is %s, so this seems OK', grism_value) # image_wl_calibrated = False if rinput.master_sky: # Sky subtraction after rectification msky = rinput.master_sky.open() # Check if images have the same size. # if so, go ahead if msky[0].data.shape != stare_image[0].data.shape: self.logger.warning( "sky and current image don't have the same shape") else: sky_corrector = proc.SkyCorrector( msky[0].data, datamodel=self.datamodel, calibid=self.datamodel.get_imgid(msky)) stare_image = sky_corrector(stare_image) else: self.logger.info('No sky image provided') # save results in results directory self.logger.info('end reduction of stare spectra') result = self.create_result(reduced_image=reduced_image, stare=stare_image) return result def set_base_headers(self, hdr): newhdr = super(StareSpectraWaveRecipe, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class StareSpectraRectwv(EmirRecipe): """Process images in Stare spectra mode applying wavelength calibration Note that in this case the wavelength calibration has already been determined. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement() reduced_mos = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('applying existing rect.+wavecal. calibration of ' 'stare spectra') # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rinput.rectwv_coeff) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rinput.rectwv_coeff) save_spectral_lines_ds9(rinput.rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of stare spectra') result = self.create_result(reduced_mos=reduced_mos) return result def set_base_headers(self, hdr): newhdr = super(StareSpectraRectwv, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class GenerateRectwvCoeff(EmirRecipe): """Process images in Stare spectra. This recipe generates a rectified and wavelength calibrated image after applying a model (master_rectwv). This calibration can be refined (using refine_wavecalib_mode != 0). """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() refine_wavecalib_mode = Parameter( 0, description='Apply wavelength calibration refinement', choices=[0, 1, 2, 11, 12]) minimum_slitlet_width_mm = Parameter( float(EMIR_MINIMUM_SLITLET_WIDTH_MM), description='Minimum width (mm) for a valid slitlet', ) maximum_slitlet_width_mm = Parameter( float(EMIR_MAXIMUM_SLITLET_WIDTH_MM), description='Maximum width (mm) for a valid slitlet', ) global_integer_offset_x_pix = Parameter( 0, description='Global offset (pixels) in wavelength direction (integer)', ) global_integer_offset_y_pix = Parameter( 0, description='Global offset (pixels) in spatial direction (integer)', ) reduced_mos = Result(prods.ProcessedMOS) rectwv_coeff = Result(RectWaveCoeff) def run(self, rinput): self.logger.info('starting rect.+wavecal. reduction of stare spectra') self.logger.info(rinput.master_rectwv) self.logger.info('Wavelength calibration refinement mode: {}'.format( rinput.refine_wavecalib_mode)) self.logger.info('Minimum slitlet width (mm)............: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info('Maximum slitlet width (mm)............: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info('Global offset X direction (pixels)....: {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info('Global offset Y direction (pixels)....: {}'.format( rinput.global_integer_offset_y_pix)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength # calibration coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image, rinput.master_rectwv) # set global offsets rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix # apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) # wavelength calibration refinement # 0 -> no refinement # 1 -> apply global offset to all the slitlets (using ARC lines) # 2 -> apply individual offset to each slitlet (using ARC lines) # 11 -> apply global offset to all the slitlets (using OH lines) # 12 -> apply individual offset to each slitlet (using OH lines) if rinput.refine_wavecalib_mode != 0: self.logger.info( 'Refining wavelength calibration (mode={})'.format( rinput.refine_wavecalib_mode)) # refine RectWaveCoeff object rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff( reduced_mos, rectwv_coeff, rinput.refine_wavecalib_mode, rinput.minimum_slitlet_width_mm, rinput.maximum_slitlet_width_mm, save_intermediate_results=self.intermediate_results) self.save_intermediate_img(expected_catalog_lines, 'expected_catalog_lines.fits') # re-apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of stare spectra') result = self.create_result(reduced_mos=reduced_mos, rectwv_coeff=rectwv_coeff) return result def set_base_headers(self, hdr): newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class GenerateRectwvCoeff(EmirRecipe): """Process images in Stare spectra. This recipe generates a rectified and wavelength calibrated image after applying a model (master_rectwv). This calibration can be refined (using refine_wavecalib_mode != 0). """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() refine_wavecalib_mode = Parameter( 0, description='Apply wavelength calibration refinement', optional=True, choices=[0, 1, 2, 11, 12]) minimum_slitlet_width_mm = Parameter( float(EMIR_MINIMUM_SLITLET_WIDTH_MM), description='Minimum width (mm) for a valid slitlet', optional=True) maximum_slitlet_width_mm = Parameter( float(EMIR_MAXIMUM_SLITLET_WIDTH_MM), description='Maximum width (mm) for a valid slitlet', optional=True) global_integer_offsets_mode = Parameter( 'fixed', description='Global integer offsets computation', choices=['auto', 'fixed']) global_integer_offset_x_pix = Parameter( 0, description='Global integer offset (pixels) in wavelength direction', optional=True) global_integer_offset_y_pix = Parameter( 0, description='Global integer offset (pixels) in spatial direction', optional=True) reduced_mos = Result(prods.ProcessedMOS) rectwv_coeff = Result(RectWaveCoeff) def run(self, rinput): self.logger.info('starting rect.+wavecal. reduction of stare spectra') self.logger.info(rinput.master_rectwv) self.logger.info( 'Wavelength calibration refinement mode....: {}'.format( rinput.refine_wavecalib_mode)) self.logger.info( 'Minimum slitlet width (mm)................: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info( 'Maximum slitlet width (mm)................: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info( 'Global integer offsets mode...............: {}'.format( rinput.global_integer_offsets_mode)) self.logger.info( 'Global integer offset X direction (pixels): {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info( 'Global integer offset Y direction (pixels): {}'.format( rinput.global_integer_offset_y_pix)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=sigmaclip) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength # calibration coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image, rinput.master_rectwv) # wavelength calibration refinement # 0 -> no refinement # 1 -> apply global offset to all the slitlets (using ARC lines) # 2 -> apply individual offset to each slitlet (using ARC lines) # 11 -> apply global offset to all the slitlets (using OH lines) # 12 -> apply individual offset to each slitlet (using OH lines) if rinput.refine_wavecalib_mode != 0: main_header = reduced_image[0].header # determine useful slitlets csu_config = CsuConfiguration.define_from_header(main_header) # segregate slitlets list_useful_slitlets = csu_config.widths_in_range_mm( minwidth=rinput.minimum_slitlet_width_mm, maxwidth=rinput.maximum_slitlet_width_mm) # remove missing slitlets if len(rectwv_coeff.missing_slitlets) > 0: for iremove in rectwv_coeff.missing_slitlets: if iremove in list_useful_slitlets: list_useful_slitlets.remove(iremove) list_not_useful_slitlets = [ i for i in list(range(1, EMIR_NBARS + 1)) if i not in list_useful_slitlets ] self.logger.info( 'list of useful slitlets: {}'.format(list_useful_slitlets)) self.logger.info('list of unusable slitlets: {}'.format( list_not_useful_slitlets)) # retrieve arc/OH lines catlines_all_wave, catlines_all_flux = retrieve_catlines( rinput.refine_wavecalib_mode, main_header['grism']) # global integer offsets if rinput.global_integer_offsets_mode == 'auto': if (rinput.global_integer_offset_x_pix != 0) or \ (rinput.global_integer_offset_y_pix != 0): raise ValueError('Global integer offsets must be zero when' ' mode=auto') # ToDo: include additional airglow emission lines self.logger.info('computing synthetic image') # generate synthetic image synthetic_raw_data = synthetic_lines_rawdata( catlines_all_wave, catlines_all_flux, list_useful_slitlets, rectwv_coeff) synthetic_raw_header = main_header.copy() synthetic_raw_header['DATE-OBS'] = \ datetime.now().strftime('%Y-%m-%dT%H:%M:%S') chistory = 'Synthetic image' synthetic_raw_header.add_history(chistory) hdu = fits.PrimaryHDU(synthetic_raw_data.astype('float32'), header=synthetic_raw_header) synthetic_raw_image = fits.HDUList([hdu]) if self.intermediate_results: self.save_intermediate_img(synthetic_raw_image, 'synthetic_raw_image.fits') # cross-correlation to determine global integer offsets # (rescaling data arrays to [0, 1] before using skimage # function) data1_rs, coef1_rs = rescale_array_to_z1z2( reduced_image[0].data, (0, 1)) data2_rs, coef2_rs = rescale_array_to_z1z2( synthetic_raw_data, (0, 1)) shifts, error, diffphase = register_translation( data1_rs, data2_rs, 100) self.logger.info( 'global_float_offset_x_pix..: {}'.format(-shifts[1])) self.logger.info( 'global_float_offset_y_pix..: {}'.format(-shifts[0])) rectwv_coeff.global_integer_offset_x_pix = \ -int(round(shifts[1])) rectwv_coeff.global_integer_offset_y_pix = \ -int(round(shifts[0])) self.logger.info('global_integer_offset_x_pix: {}'.format( rectwv_coeff.global_integer_offset_x_pix)) self.logger.info('global_integer_offset_y_pix: {}'.format( rectwv_coeff.global_integer_offset_y_pix)) if self.intermediate_results: data_product = np.fft.fft2(data1_rs) * \ np.fft.fft2(data2_rs).conj() cc_image = np.fft.fftshift(np.fft.ifft2(data_product)) power = np.log10(cc_image.real) hdu_power = fits.PrimaryHDU(power) hdul_power = fits.HDUList([hdu_power]) hdul_power.writeto('power.fits', overwrite=True) else: rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix # apply initial rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) self.logger.info( 'Refining wavelength calibration (mode={})'.format( rinput.refine_wavecalib_mode)) # refine RectWaveCoeff object rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff( reduced_mos, rectwv_coeff, catlines_all_wave, catlines_all_flux, rinput.refine_wavecalib_mode, list_useful_slitlets, save_intermediate_results=self.intermediate_results) self.save_intermediate_img(expected_catalog_lines, 'expected_catalog_lines.fits') # apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of stare spectra') result = self.create_result(reduced_mos=reduced_mos, rectwv_coeff=rectwv_coeff) return result def set_base_headers(self, hdr): newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr