class SkySpecRecipe(EmirRecipe): """Recipe to process data taken in spectral sky mode. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() skyspec = Result(prods.SkySpectrum) reduced_image = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting spectral sky reduction') flow = self.init_filters(rinput) reduced_image = basic_processing_with_combination( rinput, flow, method=median, errors=True ) hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength calibration # coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv ) # save as JSON file in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # generate associated ds9 region files and save them in work directory if self.intermediate_results: save_four_ds9(rectwv_coeff) # apply rectification and wavelength calibration skyspec = apply_rectwv_coeff( reduced_image, rectwv_coeff ) self.logger.info('end sky spectral reduction') result = self.create_result( reduced_image=reduced_image, skyspec=skyspec ) return result
class StareSpectraRecipe(EmirRecipe): """Process images in Stare spectra mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_sky = reqs.SpectralSkyRequirement(optional=True) stare = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting stare spectra reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median) hdr = hdulist[0].header self.set_base_headers(hdr) # Update EXP to 0 hdr['EXP'] = 0 self.logger.info('end stare spectra reduction') result = self.create_result(stare=hdulist) return result
class Flat(BaseRecipe): obresult = Requirement(ObservationResultType, "Observation Result") master_bias = Requirement(MasterBias, "Master Bias") polynomial_degree = Parameter(5, 'Polynomial degree of arc calibration', as_list=True, nelem='+', validator=range_validator(minval=1)) @master_bias.validator def custom_validator(self, value): print('func', self, value) return True master_flat = Result(MasterFlat) def run(self, rinput): # Here the raw images are processed # and a final image myframe is created obresult = rinput.obresult fr0 = obresult.frames[0].open() data = numpy.ones_like(fr0[0].data) hdu = fits.PrimaryHDU(data, header=fr0[0].header) myframe = fits.HDUList([hdu]) # result = self.create_result(master_flat=myframe) return result
class DTUFocusRecipe(EmirRecipe): """ Recipe to compute the DTU focus. **Observing modes:** * EMIR focus control **Inputs:** * A list of images * A list of sky images * Bias, dark, flat * A model of the detector * List of focii **Outputs:** * Best focus """ master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() objects = Parameter([], 'List of x-y pair of object coordinates'), msm_pattern = Parameter([], 'List of x-y pair of slit coordinates'), dtu_focus_range = Parameter([], 'Focus range of the DTU: begin, end and step') focus = Result(prods.DTUFocus) def run(self, rinput): return self.create_result(focus=prods.DTUFocus())
class DitherSkyRecipe(EmirRecipe): """Recipe to process data taken in dither sky mode. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() skyframe = Result(prods.MasterSky) def run(self, rinput): _logger.debug('instrument %s, mode %s', rinput.obresult.instrument, rinput.obresult.mode) _logger.info('starting sky reduction with dither') flow = self.init_filters(rinput) hdulist = basic_processing_with_segmentation(rinput, flow, method=median, errors=True) hdr = hdulist[0].header self.set_base_headers(hdr) _logger.info('end sky reduction with dither') result = self.create_result(skyframe=hdulist) return result
class TestSkyCorrectRecipe(EmirRecipe): obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = Requirement(prods.MasterIntensityFlat, 'Master Sky calibration') frame = Result(prods.ProcessedImage) def run(self, rinput): self.logger.info('starting simple sky reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median) hdr = hdulist[0].header self.set_base_headers(hdr) # Update SEC to 0 hdr['SEC'] = 0 result = self.create_result(frame=hdulist) return result
class TelescopeFineFocusRecipe(EmirRecipe): """ Recipe to compute the telescope focus. **Observing modes:** * Telescope fine focus **Inputs:** * A list of images * A list of sky images * Bias, dark, flat * A model of the detector * List of focii **Outputs:** * Best focus """ master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() objects = Parameter([], 'List of x-y pair of object coordinates'), focus = Result(prods.TelescopeFocus) def run(self, rinput): return self.create_result(focus=prods.TelescopeFocus())
class WavelengthCalibrationRecipe(EmirRecipe): """Recipe to calibrate the spectral response. **Observing modes:** * Wavelength calibration (4.5) **Inputs:** * List of line positions * Calibrations up to spectral flatfielding **Outputs:** * Wavelength calibration structure **Procedure:** * TBD """ master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_spectral_ff = reqs.MasterSpectralFlatFieldRequirement() cal = Result(prods.WavelengthCalibration) def run(self, rinput): return self.create_result(cal=prods.WavelengthCalibration())
class SimpleSkyRecipe(EmirRecipe): """Recipe to process data taken in intensity flat-field mode. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() skyframe = Result(prods.MasterSky) def run(self, rinput): _logger.info('starting sky reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=True) hdr = hdulist[0].header self.set_base_headers(hdr) result = self.create_result(skyframe=hdulist) return result
class SlitTransmissionRecipe(EmirRecipe): """Recipe to calibrate the slit transmission. **Observing modes:** * Slit transmission calibration (4.4) **Inputs:** * A list of uniformly illuminated images of MSM **Outputs:** * A list of slit transmission functions **Procedure:** * TBD """ master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() slit = Result(prods.SlitTransmissionCalibration) def run(self, rinput): return self.create_result(slit=prods.SlitTransmissionCalibration())
class BiasRecipe(EmirRecipe): """ Recipe to process data taken in Bias image Mode. Bias images only appear in Simple Readout mode. **Observing modes:** * Bias Image (3.1) **Inputs:** **Outputs:** * A combined bias frame, with variance extension. * Statistics of the final image per channel (mean, median, variance) **Procedure:** The list of images can be readly processed by combining them with a median algorithm. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() biasframe = Result(prods.MasterBias) def run(self, rinput): _logger.info('starting bias reduction') iinfo = gather_info_frames(rinput.obresult.frames) if iinfo: mode = iinfo[0]['readmode'] if mode.lower() not in EMIR_BIAS_MODES: msg = 'readmode %s, is not a bias mode' % mode _logger.error(msg) raise RecipeError(msg) flow = lambda x: x hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=False) pdata = hdulist[0].data # update hdu header with # reduction keywords hdr = hdulist[0].header self.set_base_headers(hdr) hdr['CCDMEAN'] = pdata.mean() _logger.info('bias reduction ended') result = self.create_result(biasframe=DataFrame(hdulist)) return result
class IntensityFlatRecipe(EmirRecipe): """Recipe to process data taken in intensity flat-field mode. Recipe to process intensity flat-fields. The flat-on and flat-off images are combined (method?) separately and the subtracted to obtain a thermal subtracted flat-field. **Observing modes:** * Intensity Flat-Field **Inputs:** * A master dark frame * Non linearity * A model of the detector. **Outputs:** * TBD **Procedure:** * A combined thermal subtracted flat field, normalized to median 1, with with variance extension and quality flag. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() flatframe = Result(prods.MasterIntensityFlat) def run(self, rinput): _logger.info('starting flat reduction') errors = True flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=errors) hdr = hdulist[0].header self.set_base_headers(hdr) mm = hdulist[0].data.mean() hdr['CCDMEAN'] = mm hdulist[0].data /= mm if errors: hdulist['variance'].data /= (mm * mm) result = self.create_result(flatframe=hdulist) return result
class CoaddRecipe(EmirRecipe): """Generic Coadd Recipe""" obresult = ObservationResultRequirement() result_coadd = Result(prods.ProcessedMOS) def build_recipe_input(self, obsres, dal): if numina.ext.gtc.check_gtc(): self.logger.debug('running in GTC environment') return self.build_recipe_input_gtc(obsres, dal) else: self.logger.debug('running outside of GTC environment') return super(CoaddRecipe, self).build_recipe_input(obsres, dal) def build_recipe_input_gtc(self, obsres, dal): self.logger.debug('start recipe input builder') # This depends on the RecipeResult result_field = 'reduced_mos_abba' stareImagesIds = obsres.stareSpectraIds self.logger.debug('Coadd images IDS %s: ', stareImagesIds) stareImages = [] for subresId in stareImagesIds: subres = dal.getRecipeResult(subresId) stareImages.append(subres['elements'][result_field]) newOR = numina.core.ObservationResult() newOR.frames = stareImages newRI = self.create_input(obresult=newOR) self.logger.debug('end recipe input builder') return newRI def run(self, rinput): self.logger.info('starting coadd reduction') flow = self.init_filters(rinput) nimages = len(rinput.obresult.frames) self.logger.info('we receive %d images', nimages) if nimages == 0: msg = 'Received %d images' % nimages raise numina.exceptions.RecipeError(msg) hdulist = basic_processing_with_combination( rinput, flow, method=combine.mean, prolog="Process Generic Coadd") hdr = hdulist[0].header self.set_base_headers(hdr) # Update SEC to 0 # hdr['SEC'] = 0 result = self.create_result(spec_coadd_abba=hdulist) self.logger.info('end coadd reduction') return result
class RecWaveRecipe(EmirRecipe): """Builds a MasterRecWave from a seralized version""" filename = Requirement(str, 'Full path of MasterRecWave') master_rectwv = Result(prods.MasterRectWave) def run(self, rinput): filename = rinput.filename self.logger.debug('filename is %s', filename) master_rectwv = load(prods.MasterRectWave, filename) return self.create_result(master_rectwv=master_rectwv)
class DarkRecipe(EmirRecipe): """Recipe to process data taken in Dark current image Mode. Recipe to process dark images. The dark images will be combined using the median. They do have to be of the same exposure time t. **Observing mode:** * Dark current Image (3.2) **Inputs:** **Outputs:** * A combined dark frame, with variance extension. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() darkframe = Result(prods.MasterDark) def run(self, rinput): _logger.info('starting dark reduction') flow = self.init_filters(rinput) iinfo = gather_info_frames(rinput.obresult.frames) ref_exptime = 0.0 for el in iinfo[1:]: if abs(el['texp'] - ref_exptime) > 1e-4: _logger.error('image with wrong exposure time') raise RecipeError('image with wrong exposure time') hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=True) pdata = hdulist[0].data # update hdu header with # reduction keywords hdr = hdulist[0].header self.set_base_headers(hdr) hdr['CCDMEAN'] = pdata.mean() _logger.info('dark reduction ended') result = self.create_result(darkframe=hdulist) return result
class StareImageRecipe2(EmirRecipe): """Process images in Stare Image Mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() result_image = Result(prods.ProcessedImage) def run(self, rinput): self.logger.info('starting stare image reduction (offline)') frames = rinput.obresult.frames datamodel = self.datamodel with extra.manage_frame(frames) as list_of: c_img = extra.combine_frames(list_of, datamodel, method=combine.mean, errors=False) flow = self.init_filters(rinput) # Correct Bias if needed # Correct Dark if needed # Correct FF processed_img = flow(c_img) hdr = processed_img[0].header self.set_base_headers(hdr) self.logger.debug('append BPM') if rinput.master_bpm is not None: self.logger.debug('using BPM from inputs') hdul_bpm = rinput.master_bpm.open() hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0]) else: self.logger.debug('using empty BPM') hdu_bpm = extra.generate_empty_bpm_hdu(processed_img[0]) # Append the BPM to the result processed_img.append(hdu_bpm) self.logger.info('end stare image (off) reduction') result = self.create_result(result_image=processed_img) return result def set_base_headers(self, hdr): """Set metadata in FITS headers.""" hdr = super(StareImageRecipe2, self).set_base_headers(hdr) # Set EXP to 0 hdr['EXP'] = 0 return hdr
class CoaddABBARecipe(EmirRecipe): """Process images in ABBA mode""" obresult = ObservationResultRequirement(query_opts=qmod.ResultOf( 'STARE_SPECTRA.stare', node='children', id_field="stareSpectraIds")) spec_coadd_abba = Result(prods.ProcessedMOS) def build_recipe_input(self, obsres, dal): if numina.ext.gtc.check_gtc(): self.logger.debug('running in GTC environment') return self.build_recipe_input_gtc(obsres, dal) else: self.logger.debug('running outside of GTC environment') return super(CoaddABBARecipe, self).build_recipe_input(obsres, dal) def build_recipe_input_gtc(self, obsres, dal): self.logger.debug('start recipe input builder') stareImagesIds = obsres.stareSpectraIds self.logger.debug('ABBA images IDS %s: ', stareImagesIds) stareImages = [] for subresId in stareImagesIds: subres = dal.getRecipeResult(subresId) stareImages.append(subres['elements']['spec_abba']) newOR = numina.core.ObservationResult() newOR.frames = stareImages newRI = self.create_input(obresult=newOR) self.logger.debug('end recipe input builder') return newRI def run(self, rinput): self.logger.info('starting spectroscopy ABBA coadd reduction') flow = self.init_filters(rinput) nimages = len(rinput.obresult.frames) self.logger.info('we receive %d images', nimages) if nimages == 0: msg = 'Received %d images' % nimages raise numina.exceptions.RecipeError(msg) hdulist = basic_processing_with_combination( rinput, flow, method=combine.mean, prolog="Process Coadd ABBA") hdr = hdulist[0].header self.set_base_headers(hdr) # Update SEC to 0 # hdr['SEC'] = 0 result = self.create_result(spec_coadd_abba=hdulist) self.logger.info('end spectroscopy ABBA coadd reduction') return result
class IntensityFlatRecipe2(EmirRecipe): obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flatframe = Result(prods.MasterIntensityFlat) def run(self, rinput): import emirdrp.core.extra as extra from numina.array import combine _logger.info('starting flat reduction') frames = rinput.obresult.frames datamodel = self.datamodel with extra.manage_frame(frames) as list_of: c_img = extra.combine_frames2(list_of, datamodel, method=combine.mean, errors=False) self.save_intermediate_img(c_img, 'p0.fits') flow = self.init_filters(rinput) processed_img = flow(c_img) self.save_intermediate_img(processed_img, 'p1.fits') hdr = processed_img[0].header self.set_base_headers(hdr) import scipy.ndimage.filters _logger.info('median filter') data_smooth = scipy.ndimage.filters.median_filter( processed_img[0].data, size=11) self.save_intermediate_array(data_smooth, 'smooth.fits') mm = processed_img[0].data.mean() hdr['CCDMEAN'] = mm processed_img[0].data /= data_smooth self.save_intermediate_img(processed_img, 'p2.fits') result = self.create_result(master_flatframe=processed_img) return result
class SpectralFlatRecipe(EmirRecipe): """Recipe to process data taken in intensity flat-field mode.""" master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() flatframe = Result(prods.MasterSpectralFlat) def run(self, rinput): return self.create_result(flatframe=prods.MasterSpectralFlat())
class StareImageBaseRecipe(EmirRecipe): """Process images in Stare Image Mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement(optional=True) frame = Result(prods.ProcessedImage) def __init__(self, *args, **kwargs): super(StareImageBaseRecipe, self).__init__(*args, **kwargs) if False: self.query_options['master_sky'] = Ignore() @emirdrp.decorators.loginfo @timeit def run(self, rinput): self.logger.info('starting stare image reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination( rinput, flow, method=combine.median ) hdr = hdulist[0].header self.set_base_headers(hdr) if rinput.master_bpm: hdul_bpm = rinput.master_bpm.open() hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0]) else: hdu_bpm = extra.generate_empty_bpm_hdu(hdulist[0]) # Append the BPM to the result hdulist.append(hdu_bpm) self.logger.info('end stare image reduction') result = self.create_result(frame=hdulist) return result def set_base_headers(self, hdr): """Set metadata in FITS headers.""" hdr = super(StareImageBaseRecipe, self).set_base_headers(hdr) # Update EXP to 0 hdr['EXP'] = 0 return hdr
class BiasRecipe(BaseRecipe): obresult = Requirement(ObservationResultType, "Observation Result") master_bias = Result(MasterBias) def run(self, rinput): # Here the raw images are processed # and a final image myframe is created myframe = rinput.obresult.frames[0].open() # result = self.create_result(master_bias=myframe) return result
class TestBiasCorrectRecipe(EmirRecipe): obresult = ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() frame = Result(prods.ProcessedImage) def run(self, rinput): _logger.info('starting simple bias reduction') flow = self.init_filters(rinput) hdu = basic_processing_with_combination(rinput, flow, method=median) hdr = hdu[0].header hdr['NUMRNAM'] = (self.__class__.__name__, 'Numina recipe name') hdr['NUMRVER'] = (self.__version__, 'Numina recipe version') hdulist = fits.HDUList([hdu]) result = self.create_result(frame=hdulist) return result
class SimpleBiasRecipe(EmirRecipe): """ Recipe to process data taken in SimpleBias image Mode. Bias images only appear in Simple Readout mode. **Outputs:** * A combined bias frame, with variance extension. **Procedure:** The list of images can be readly processed by combining them with a median algorithm. """ obresult = ObservationResultRequirement() biasframe = Result(prods.MasterBias) def run(self, rinput): _logger.info('starting simple bias reduction') flow = lambda x: x hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=True) # update hdu header with # reduction keywords hdr = hdulist[0].header hdr['IMGTYP'] = ('BIAS', 'Image type') hdr['NUMTYP'] = ('MASTER_BIAS', 'Data product type') hdr['NUMRNAM'] = (self.__class__.__name__, 'Numina recipe name') hdr['NUMRVER'] = (self.__version__, 'Numina recipe version') _logger.info('simple bias reduction ended') # qc is QC.UNKNOWN result = self.create_result(biasframe=hdulist) return result
class TestRectImageRecipe(EmirRecipe): """A Recipe to test GCS handling of rectangular images. This appeared as a problem during EMIR first comisioning, it was fixed, date 2016-06-21 """ obresult = ObservationResultRequirement() frame = Result(prods.ProcessedImage) def run(self, rinput): import numpy _logger.info('testing rectangular image') data = numpy.zeros((500, 1000), dtype='float32') data[200:400, 400:600] = 10000.0 hdu = fits.PrimaryHDU(data) hdulist = fits.HDUList([hdu]) print("numpy shape of data is", data.shape) _logger.info('end testing rectangular image') result = self.create_result(frame=hdulist) return result
class CosmeticsRecipe(EmirRecipe): """Detector Cosmetics. Recipe to find and tag bad pixels in the detector. """ obresult = ObservationResultRequirement() insconf = InstrumentConfigurationRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() lowercut = Parameter( 4.0, 'Values below this sigma level are flagged as dead pixels') uppercut = Parameter( 4.0, 'Values above this sigma level are flagged as hot pixels') maxiter = Parameter(30, 'Maximum number of iterations') ratioframe = Result(prods.ProcessedImage) maskframe = Result(prods.MasterBadPixelMask) def run(self, rinput): # FIXME: # We need 2 flats # Of different exposure times # # And their calibrations # if len(rinput.obresult.frames) < 2: raise RecipeError('The recipe requires 2 flat frames') iinfo = [] for frame in rinput.obresult.frames: with frame.open() as hdulist: iinfo.append(gather_info(hdulist)) # Loading calibrations with rinput.master_bias.open() as hdul: readmode = hdul[0].header.get('READMODE', 'undefined') if readmode.lower() in ['simple', 'bias']: self.logger.debug('loading bias') mbias = hdul[0].data bias_corrector = proc.BiasCorrector(mbias) else: self.logger.debug('ignoring bias') bias_corrector = numina.util.node.IdNode() with rinput.master_dark.open() as mdark_hdul: self.logger.debug('loading dark') mdark = mdark_hdul[0].data dark_corrector = proc.DarkCorrector(mdark) flow = numina.util.flow.SerialFlow([bias_corrector, dark_corrector]) self.logger.info('processing flat #1') with rinput.obresult.frames[0].open() as hdul: other = flow(hdul) f1 = other[0].data.copy() * iinfo[0]['texp'] * 1e-3 self.logger.info('processing flat #2') with rinput.obresult.frames[1].open() as hdul: other = flow(hdul) f2 = other[0].data.copy() * iinfo[1]['texp'] * 1e-3 # Preprocess... maxiter = rinput.maxiter lowercut = rinput.lowercut uppercut = rinput.uppercut ninvalid = 0 mask = None if mask: m = fits.getdata(mask) ninvalid = numpy.count_nonzero(m) else: m = numpy.zeros_like(f1, dtype='int') for niter in range(1, maxiter + 1): self.logger.debug('iter %d', niter) ratio, m, sigma = cosmetics(f1, f2, m, lowercut=lowercut, uppercut=uppercut) if self.intermediate_results: with warnings.catch_warnings(): warnings.simplefilter('ignore') fits.writeto('numina-cosmetics-i%02d.fits' % niter, ratio, overwrite=True) fits.writeto('numina-mask-i%02d.fits' % niter, m, overwrite=True) fits.writeto('numina-sigma-i%02d.fits' % niter, m * 0.0 + sigma, overwrite=True) self.logger.debug('iter %d, invalid points in input mask: %d', niter, ninvalid) self.logger.debug('iter %d, estimated sigma is %f', niter, sigma) n_ninvalid = numpy.count_nonzero(m) # Probably there is something wrong here # too much defective pixels if ninvalid / m.size >= 0.10: # This should set a flag in the output msg = 'defective pixels are greater than 10%' self.logger.warning(msg) if n_ninvalid == ninvalid: self.logger.info('convergence reached after %d iterations', niter) break self.logger.info('new invalid points: %d', n_ninvalid - ninvalid) ninvalid = n_ninvalid else: # This should set a flag in the output msg = 'convergence not reached after %d iterations' % maxiter self.logger.warning(msg) self.logger.info('number of dead pixels %d', numpy.count_nonzero(m == PIXEL_DEAD)) self.logger.info('number of hot pixels %d', numpy.count_nonzero(m == PIXEL_HOT)) if self.intermediate_results: with warnings.catch_warnings(): warnings.simplefilter('ignore') fits.writeto('numina-cosmetics.fits', ratio, overwrite=True) fits.writeto('numina-mask.fits', m, overwrite=True) fits.writeto('numina-sigma.fits', sigma * numpy.ones_like(m), overwrite=True) hdu = fits.PrimaryHDU(ratio) hdr = hdu.header hdr['NUMXVER'] = (__version__, 'Numina package version') hdr['NUMRNAM'] = (self.__class__.__name__, 'Numina recipe name') hdr['NUMRVER'] = (self.__version__, 'Numina recipe version') ratiohdl = fits.HDUList([hdu]) maskhdu = fits.PrimaryHDU(m) hdr = maskhdu.header hdr['NUMXVER'] = (__version__, 'Numina package version') hdr['NUMRNAM'] = (self.__class__.__name__, 'Numina recipe name') hdr['NUMRVER'] = (self.__version__, 'Numina recipe version') maskhdl = fits.HDUList([maskhdu]) res = self.create_result(ratioframe=ratiohdl, maskframe=maskhdl) return res
class MaskCheckRecipe(EmirRecipe): """ Acquire a target. Recipe for the processing of multi-slit/long-slit check images. **Observing modes:** * MSM and LSM check """ # Recipe Requirements # obresult = ObservationResultRequirement( query_opts=qmod.ResultOf( 'STARE_IMAGE.frame', node='children', id_field="resultsIds" ) ) master_bpm = reqs.MasterBadPixelMaskRequirement() bars_nominal_positions = Requirement( prods.NominalPositions, 'Nominal positions of the bars' ) # Recipe Products slit_image = Result(prods.ProcessedImage) object_image = Result(prods.ProcessedImage) offset = Result(tarray.ArrayType) angle = Result(float) def run(self, rinput): self.logger.info('starting processing for image acquisition') # Combine and masking flow = self.init_filters(rinput) # count frames frames = rinput.obresult.frames nframes = len(frames) if nframes not in [1, 2, 4]: raise ValueError("expected 1, 2 or 4 frames, got {}".format(nframes)) interm = basic_processing_(frames, flow, self.datamodel) if nframes == 1: hdulist_slit = combine_images(interm[:], self.datamodel) hdulist_object = hdulist_slit # background_subs = False elif nframes == 2: hdulist_slit = combine_images(interm[0:], self.datamodel) hdulist_object = process_ab(interm, self.datamodel) # background_subs = True elif nframes == 4: hdulist_slit = combine_images(interm[0::3], self.datamodel) hdulist_object = process_abba(interm, self.datamodel) # background_subs = True else: raise ValueError("expected 1, 2 or 4 frames, got {}".format(nframes)) self.set_base_headers(hdulist_slit[0].header) self.set_base_headers(hdulist_object[0].header) self.save_intermediate_img(hdulist_slit, 'slit_image.fits') self.save_intermediate_img(hdulist_object, 'object_image.fits') # Get slits # Rotation around (0,0) # For other axis, offset is changed # (Off - raxis) = Rot * (Offnew - raxis) crpix1 = hdulist_slit[0].header['CRPIX1'] crpix2 = hdulist_slit[0].header['CRPIX2'] rotaxis = np.array((crpix1 - 1, crpix2 - 1)) self.logger.debug('center of rotation (from CRPIX) is %s', rotaxis) csu_conf = self.load_csu_conf(hdulist_slit, rinput.bars_nominal_positions) # IF CSU is completely open OR there are no refereces, # this is not needed if not csu_conf.is_open(): self.logger.info('CSU is configured, detecting slits') slits_bb = self.compute_slits(hdulist_slit, csu_conf) image_sep = hdulist_object[0].data.astype('float32') self.logger.debug('center of rotation (from CRPIX) is %s', rotaxis) offset, angle, qc = compute_off_rotation( image_sep, csu_conf, slits_bb, rotaxis=rotaxis, logger=self.logger, debug_plot=False, intermediate_results=self.intermediate_results ) else: self.logger.info('CSU is open, not detecting slits') offset = [0.0, 0.0] angle = 0.0 qc = QC.GOOD # Convert mm to m offset_out = np.array(offset) / 1000.0 # Convert DEG to RAD angle_out = np.deg2rad(angle) result = self.create_result( slit_image=hdulist_slit, object_image=hdulist_object, offset=offset_out, angle=angle_out, qc=qc ) self.logger.info('end processing for image acquisition') return result def load_csu_conf(self, hdulist, bars_nominal_positions): # Get slits hdr = hdulist[0].header # Extract DTU and CSU information from headers dtuconf = self.datamodel.get_dtur_from_header(hdr) # coordinates transformation from DTU coordinates # to image coordinates # Y inverted # XY switched # trans1 = [[1, 0, 0], [0,-1, 0], [0,0,1]] # trans2 = [[0,1,0], [1,0,0], [0,0,1]] trans3 = [[0, -1, 0], [1, 0, 0], [0, 0, 1]] # T3 = T2 * T1 vec = np.dot(trans3, dtuconf.coor_r) / EMIR_PIXSCALE self.logger.debug('DTU shift is %s', vec) self.logger.debug('create bar model') barmodel = csuconf.create_bar_models(bars_nominal_positions) csu_conf = csuconf.read_csu_2(hdr, barmodel) if self.intermediate_results: # FIXME: coordinates are in VIRT pixels self.logger.debug('create bar mask from predictions') mask = np.ones_like(hdulist[0].data) for i in itertools.chain(csu_conf.lbars, csu_conf.rbars): bar = csu_conf.bars[i] mask[bar.bbox().slice] = 0 self.save_intermediate_array(mask, 'mask_bars.fits') self.logger.debug('create slit mask from predictions') mask = np.zeros_like(hdulist[0].data) for slit in csu_conf.slits.values(): mask[slit.bbox().slice] = slit.idx self.save_intermediate_array(mask, 'mask_slit.fits') self.logger.debug('create slit reference mask from predictions') mask1 = np.zeros_like(hdulist[0].data) for slit in csu_conf.slits.values(): if slit.target_type == TargetType.REFERENCE: mask1[slit.bbox().slice] = slit.idx self.save_intermediate_array(mask1, 'mask_slit_ref.fits') return csu_conf def compute_slits(self, hdulist, csu_conf): self.logger.debug('finding borders of slits') self.logger.debug('not strictly necessary...') data = hdulist[0].data self.logger.debug('dtype of data %s', data.dtype) self.logger.debug('median filter (3x3)') image_base = ndi.filters.median_filter(data, size=3) # Cast as original type for skimage self.logger.debug('casting image to unit16 (for skimage)') iuint16 = np.iinfo(np.uint16) image = np.clip(image_base, iuint16.min, iuint16.max).astype(np.uint16) self.logger.debug('compute Sobel filter') # FIXME: compute sob and sob_v is redundant sob = filt.sobel(image) self.save_intermediate_array(sob, 'sobel_image.fits') sob_v = filt.sobel_v(image) self.save_intermediate_array(sob_v, 'sobel_v_image.fits') # Compute detector coordinates of bars all_coords_virt = np.empty((110, 2)) all_coords_real = np.empty((110, 2)) # Origin of coordinates is 1 for bar in csu_conf.bars.values(): all_coords_virt[bar.idx - 1] = bar.xpos, bar.y0 # Origin of coordinates is 1 for this function _x, _y = dist.exvp(all_coords_virt[:, 0], all_coords_virt[:, 1]) all_coords_real[:, 0] = _x all_coords_real[:, 1] = _y # FIXME: hardcoded value h = 16 slit_h_virt = 16.242 slit_h_tol = 3 slits_bb = {} mask1 = np.zeros_like(hdulist[0].data) for idx in range(EMIR_NBARS): lbarid = idx + 1 rbarid = lbarid + EMIR_NBARS ref_x_l_v, ref_y_l_v = all_coords_virt[lbarid - 1] ref_x_r_v, ref_y_r_v = all_coords_virt[rbarid - 1] ref_x_l_d, ref_y_l_d = all_coords_real[lbarid - 1] ref_x_r_d, ref_y_r_d = all_coords_real[rbarid - 1] width_v = ref_x_r_v - ref_x_l_v # width_d = ref_x_r_d - ref_x_l_d if (ref_y_l_d >= 2047 + h) or (ref_y_l_d <= 1 - h): # print('reference y position is outlimits, skipping') continue if width_v < 5: # print('width is less than 5 pixels, skipping') continue plot = False regionw = 12 px1 = coor_to_pix_1d(ref_x_l_d) - 1 px2 = coor_to_pix_1d(ref_x_r_d) - 1 prow = coor_to_pix_1d(ref_y_l_d) - 1 comp_l, comp_r = calc0(image, sob_v, prow, px1, px2, regionw, h=h, plot=plot, lbarid=lbarid, rbarid=rbarid, plot2=False) if np.any(np.isnan([comp_l, comp_r])): self.logger.warning("converting NaN value, border of=%d", idx + 1) self.logger.warning("skipping bar=%d", idx + 1) continue elif comp_l > comp_r: # Not refining self.logger.warning("computed left border of=%d greater than right border", idx + 1) comp2_l, comp2_r = px1, px2 else: region2 = 5 px21 = coor_to_pix_1d(comp_l) px22 = coor_to_pix_1d(comp_r) comp2_l, comp2_r = calc0(image, sob_v, prow, px21, px22, region2, refine=True, plot=plot, lbarid=lbarid, rbarid=rbarid, plot2=False) if np.any(np.isnan([comp2_l, comp2_r])): self.logger.warning("converting NaN value, border of=%d", idx + 1) comp2_l, comp2_r = comp_l, comp_r elif comp2_l > comp2_r: # Not refining self.logger.warning("computed left border of=%d greater than right border", idx + 1) comp2_l, comp2_r = comp_l, comp_r # print('slit', lbarid, '-', rbarid, comp_l, comp_r) # print('pos1', comp_l, comp_r) # print('pos2', comp2_l, comp2_r) xpos1_virt, _ = dist.pvex(comp2_l + 1, ref_y_l_d) xpos2_virt, _ = dist.pvex(comp2_r + 1, ref_y_r_d) y1_virt = ref_y_l_v - slit_h_virt - slit_h_tol y2_virt = ref_y_r_v + slit_h_virt + slit_h_tol _, y1 = dist.exvp(xpos1_virt + 1, y1_virt) _, y2 = dist.exvp(xpos2_virt + 1, y2_virt) # print(comp2_l, comp2_r, y1 - 1, y2 - 1) cbb = BoundingBox.from_coordinates(comp2_l, comp2_r, y1 - 1, y2 - 1) slits_bb[lbarid] = cbb mask1[cbb.slice] = lbarid self.save_intermediate_array(mask1, 'mask_slit_computed.fits') return slits_bb
class FullDitheredImagesRecipe(EmirRecipe): """Recipe for the reduction of imaging mode observations. Recipe to reduce observations obtained in imaging mode, considering different possibilities depending on the size of the offsets between individual images. In particular, the following observing modes are considered: stare imaging, nodded beamswitched imaging, and dithered imaging. A critical piece of information here is a table that clearly specifies which images can be labeled as *science*, and which ones as *sky*. Note that some images are used both as *science* and *sky* (when the size of the targets is small compared to the offsets). **Observing modes:** * StareImage * Nodded/Beam-switched images * Dithered images **Inputs:** * Science frames + [Sky Frames] * Observing mode name: **stare image**, **nodded beamswitched image**, or **dithered imaging** * A table relating each science image with its sky image(s) (TBD if it's in the FITS header and/or in other format) * Offsets between them (Offsets must be integer) * Master Dark * Bad pixel mask (BPM) * Non-linearity correction polynomials * Master flat (twilight/dome flats) * Master background (thermal background, only in K band) * Exposure Time (must be the same in all the frames) * Airmass for each frame * Detector model (gain, RN, lecture mode) * Average extinction in the filter * Astrometric calibration (TBD) **Outputs:** * Image with three extensions: final image scaled to the individual exposure time, variance and exposure time map OR number of images combined (TBD) **Procedure:** Images are corrected from dark, non-linearity and flat. Then, an iterative process starts: * Sky is computed from each frame, using the list of sky images of each science frame. The objects are avoided using a mask (from the second iteration on). * The relative offsets are the nominal from the telescope. From the second iteration on, we refine them using objects of appropriate brightness (not too bright, not to faint). * We combine the sky-subtracted images, output is: a new image, a variance image and a exposure map/number of images used map. * An object mask is generated. * We recompute the sky map, using the object mask as an additional input. From here we iterate (typically 4 times). * Finally, the images are corrected from atmospheric extinction and flux calibrated. * A preliminary astrometric calibration can always be used (using the central coordinates of the pointing and the plate scale in the detector). A better calibration might be computed using available stars (TBD). """ obresult = ObservationResultRequirement( query_opts=ResultOf('result_image', node='children')) master_bpm = reqs.MasterBadPixelMaskRequirement() offsets = Requirement(prods.CoordinateList2DType, 'List of pairs of offsets', optional=True) refine_offsets = Parameter(False, 'Refine offsets by cross-correlation') iterations = Parameter(2, 'Iterations of the recipe') extinction = Parameter(0.0, 'Mean atmospheric extinction') sky_images = Parameter( 5, 'Images used to estimate the ' 'background before and after current image') sky_images_sep_time = Parameter( 10, 'Maximum time interval between target and sky images [minutes]') result_image = Result(prods.ProcessedImage) result_sky = Result(prods.ProcessedImage, optional=True) def run(self, rinput): target_is_sky = True obresult = rinput.obresult sky_images = rinput.sky_images sky_images_sep_time = rinput.sky_images_sep_time baseshape = (2048, 2048) user_offsets = rinput.offsets extinction = rinput.extinction images_info = self.initial_classification(obresult, target_is_sky) # Resizing target frames target_info = [iinfo for iinfo in images_info if iinfo.valid_target] finalshape, offsetsp, refpix, offset_fc0 = self.compute_size( target_info, baseshape, user_offsets) self.resize(target_info, baseshape, offsetsp, finalshape) result = self.process_basic(images_info, target_is_sky=target_is_sky, extinction=extinction) if rinput.refine_offsets: self.logger.debug("Compute cross-correlation of images") # regions_c = self.compute_regions(finalshape, box=200, corners=True) # Regions frm bright objects regions_c = self.compute_regions_from_objs(result[0].data, finalshape, box=40) try: offsets_xy_c = self.compute_offset_xy_crosscor_regions( images_info, regions_c, refine=True, tol=1) # # Combined offsets # Offsets in numpy order, swaping offset_xy0 = numpy.fliplr(offset_fc0) offsets_xy_t = offset_xy0 - offsets_xy_c offsets_fc = numpy.fliplr(offsets_xy_t) offsets_fc_t = numpy.round(offsets_fc).astype('int') self.logger.debug('Total offsets: %s', offsets_xy_t) self.logger.info('Computing relative offsets from cross-corr') finalshape2, offsetsp2 = narray.combine_shape( baseshape, offsets_fc_t) # self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp2) self.logger.info('Shape of resized array (crosscorr) is %s', finalshape2) # Resizing target imgs self.logger.debug("Resize to final offsets") self.resize(target_info, baseshape, offsetsp2, finalshape2) except Exception as error: self.logger.warning('Error during cross-correlation, %s', error) result = self.process_basic(images_info, target_is_sky=target_is_sky, extinction=extinction) step = 1 while step <= rinput.iterations: result = self.process_advanced(images_info, result, step, target_is_sky, maxsep=sky_images_sep_time, nframes=sky_images, extinction=extinction) step += 1 return self.create_result(result_image=result) def compute_offset_xy_crosscor_regions(self, iinfo, regions, refine=False, tol=0.5): names = [frame.lastname for frame in iinfo] print(names) print(regions) with nfcom.manage_fits(names) as imgs: arrs = [img[0].data for img in imgs] offsets_xy = offsets_from_crosscor_regions(arrs, regions, refine=refine, order='xy', tol=tol) self.logger.debug("offsets_xy cross-corr %s", offsets_xy) # Offsets in numpy order, swaping return offsets_xy def compute_size(self, images_info, baseshape, user_offsets=None): # Reference pixel in the center of the frame refpix = numpy.array([[baseshape[0] / 2.0, baseshape[1] / 2.0]]) target_info = [iinfo for iinfo in images_info if iinfo.valid_target] if user_offsets is not None: self.logger.info('Using offsets from parameters') base_ref = numpy.asarray(user_offsets) list_of_offsets = -(base_ref - base_ref[0]) else: self.logger.debug('Computing offsets from WCS information') with nfcom.manage_fits(img.origin for img in target_info) as images: list_of_offsets = offsets_from_wcs_imgs(images, refpix) # FIXME: I am using offsets in row/columns # the values are provided in XY so flip-lr list_of_offsets = numpy.fliplr(list_of_offsets) # Insert pixel offsets between frames for iinfo, off in zip(target_info, list_of_offsets): # Insert pixel offsets between frames iinfo.pix_offset = off self.logger.debug('Frame %s, offset=%s', iinfo.label, off) self.logger.info('Computing relative offsets') offsets = [iinfo.pix_offset for iinfo in target_info] offsets = numpy.round(offsets).astype('int') finalshape, offsetsp = narray.combine_shape(baseshape, offsets) self.logger.debug("Relative offsetsp %s", offsetsp) self.logger.info('Shape of resized array is %s', finalshape) return finalshape, offsetsp, refpix, list_of_offsets def process_basic(self, images_info, target_is_sky=True, extinction=0.0): step = 0 target_info = [iinfo for iinfo in images_info if iinfo.valid_target] sky_info = [iinfo for iinfo in images_info if iinfo.valid_sky] self.logger.info("Step %d, SF: compute superflat", step) sf_arr = self.compute_superflat(images_info) # Apply superflat self.logger.info("Step %d, SF: apply superflat", step) for iinfo in images_info: self.correct_superflat(iinfo, sf_arr, step=step, save=True) self.logger.info('Simple sky correction') if target_is_sky: # Each frame is the closest sky frame available for iinfo in images_info: self.compute_simple_sky_for_frame(iinfo, iinfo) else: # Not implemented self.compute_simple_sky(target_info, sky_info) # Combining the frames self.logger.info("Step %d, Combining target frames", step) result = self.combine_frames(target_info, extinction=extinction) self.logger.info('Step %d, finished', step) return result def process_advanced(self, images_info, result, step, target_is_sky=True, maxsep=5.0, nframes=6, extinction=0): seeing_fwhm = None baseshape = (2048, 2048) target_info = [iinfo for iinfo in images_info if iinfo.valid_target] sky_info = [iinfo for iinfo in images_info if iinfo.valid_sky] self.logger.info('Step %d, generating segmentation image', step) objmask, seeing_fwhm = self.create_mask(result, seeing_fwhm, step=step) for frame in target_info: frame.objmask = name_object_mask(frame.label, step) self.logger.info('Step %d, create object mask %s', step, frame.objmask) frame.objmask_data = objmask[frame.valid_region] fits.writeto(frame.objmask, frame.objmask_data, overwrite=True) if not target_is_sky: # Empty object mask for sky frames bogus_objmask = numpy.zeros(baseshape, dtype='uint8') for frame in sky_info: frame.objmask_data = bogus_objmask self.logger.info("Step %d, SF: compute superflat", step) sf_arr = self.compute_superflat(sky_info, segmask=objmask, step=step) # Apply superflat self.logger.info("Step %d, SF: apply superflat", step) for iinfo in images_info: self.correct_superflat(iinfo, sf_arr, step=step, save=True) self.logger.info('Step %d, advanced sky correction (SC)', step) self.compute_advanced_sky(target_info, objmask, skyframes=sky_info, target_is_sky=target_is_sky, maxsep=maxsep, nframes=nframes, step=step) # Combining the images self.logger.info("Step %d, Combining the images", step) # FIXME: only for science result = self.combine_frames(target_info, extinction, step=step) return result def compute_simple_sky_for_frame(self, frame, skyframe, step=0, save=True): self.logger.info('Correcting sky in frame %s', frame.lastname) self.logger.info('with sky computed from frame %s', skyframe.lastname) if hasattr(skyframe, 'median_sky'): sky = skyframe.median_sky else: with fits.open(skyframe.lastname, mode='readonly') as hdulist: data = hdulist['primary'].data valid = data[frame.valid_region] if skyframe.objmask_data is not None: self.logger.debug('object mask defined') msk = frame.objmask_data sky = numpy.median(valid[msk == 0]) else: self.logger.debug('object mask empty') sky = numpy.median(valid) self.logger.debug('median sky value is %f', sky) skyframe.median_sky = sky dst = name_skysub_proc(frame.label, step) prev = frame.lastname if save: shutil.copyfile(prev, dst) else: os.rename(prev, dst) frame.lastname = dst with fits.open(frame.lastname, mode='update') as hdulist: data = hdulist['primary'].data valid = data[frame.valid_region] valid -= sky def compute_simple_sky(self, frame, skyframe, step=0, save=True): raise NotImplementedError def correct_superflat(self, frame, fitted, step=0, save=True): frame.flat_corrected = name_skyflat_proc(frame.label, step) if save: shutil.copyfile(frame.resized_base, frame.flat_corrected) else: os.rename(frame.resized_base, frame.flat_corrected) self.logger.info("Step %d, SF: apply superflat to frame %s", step, frame.flat_corrected) with fits.open(frame.flat_corrected, mode='update') as hdulist: data = hdulist['primary'].data datar = data[frame.valid_region] data[frame.valid_region] = narray.correct_flatfield(datar, fitted) frame.lastname = frame.flat_corrected def initial_classification(self, obresult, target_is_sky=False): """Classify input frames, """ # lists of targets and sky frames with obresult.frames[0].open() as baseimg: # Initial checks has_bpm_ext = 'BPM' in baseimg self.logger.debug('images have BPM extension: %s', has_bpm_ext) images_info = [] for f in obresult.frames: with f.open() as img: # Getting some metadata from FITS header hdr = img[0].header iinfo = ImageInfo(f) finfo = {} iinfo.metadata = finfo finfo['uuid'] = hdr['UUID'] finfo['exposure'] = hdr['EXPTIME'] # frame.baseshape = get_image_shape(hdr) finfo['airmass'] = hdr['airmass'] finfo['mjd'] = hdr['tstamp'] iinfo.label = 'result_image_{}'.format(finfo['uuid']) iinfo.mask = nfcom.Extension("BPM") # Insert pixel offsets between frames iinfo.objmask_data = None iinfo.valid_target = False iinfo.valid_sky = False # FIXME: hardcode itype for the moment iinfo.itype = 'TARGET' if iinfo.itype == 'TARGET': iinfo.valid_target = True #targetframes.append(iinfo) if target_is_sky: iinfo.valid_sky = True #skyframes.append(iinfo) if iinfo.itype == 'SKY': iinfo.valid_sky = True #skyframes.append(iinfo) images_info.append(iinfo) return images_info def compute_superflat(self, images_info, segmask=None, step=0): self.logger.info("Step %d, SF: combining the frames without offsets", step) base_imgs = [img.resized_base for img in images_info] with nfcom.manage_fits(base_imgs) as imgs: data = [] masks = [] for img, img_info in zip(imgs, images_info): self.logger.debug('Step %d, opening resized frame %s', step, img_info.resized_base) data.append(img['primary'].data[img_info.valid_region]) scales = [numpy.median(d) for d in data] if segmask is not None: masks = [segmask[frame.valid_region] for frame in images_info] else: for frame in images_info: self.logger.debug('Step %d, opening resized mask %s', step, frame.resized_mask) hdulist = fits.open(frame.resized_mask, memmap=True, mode='readonly') #filelist.append(hdulist) masks.append(hdulist['primary'].data[frame.valid_region]) masks = None self.logger.debug('Step %d, combining %d frames', step, len(data)) sf_data, _sf_var, sf_num = nacom.median( data, masks, scales=scales, dtype='float32', #blank=1.0 / scales[0] ) # Normalize, flat has mean = 1 sf_data[sf_data == 0] = 1e-5 sf_data /= sf_data.mean() #sf_data[sf_data <= 0] = 1.0 # Auxiliary data sfhdu = fits.PrimaryHDU(sf_data) self.save_intermediate_img(sfhdu, name_skyflat('comb', step)) return sf_data def run_single(self, rinput): # FIXME: remove this, is deprecated obresult = rinput.obresult # just in case images are in result, instead of frames if not obresult.frames: frames = obresult.results else: frames = obresult.frames img_info = [] data_hdul = [] for f in frames: img = f.open() data_hdul.append(img) info = {} info['tstamp'] = img[0].header['tstamp'] info['airmass'] = img[0].header['airmass'] img_info.append(info) channels = FULL use_errors = True # Initial checks baseimg = data_hdul[0] has_num_ext = 'NUM' in baseimg has_bpm_ext = 'BPM' in baseimg baseshape = baseimg[0].shape subpixshape = baseshape base_header = baseimg[0].header compute_sky = 'NUM-SK' not in base_header compute_sky_advanced = False self.logger.debug('base image is: %s', self.datamodel.get_imgid(baseimg)) self.logger.debug('images have NUM extension: %s', has_num_ext) self.logger.debug('images have BPM extension: %s', has_bpm_ext) self.logger.debug('compute sky is needed: %s', compute_sky) if compute_sky: self.logger.info('compute sky simple') sky_result = self.compute_sky_simple(data_hdul, use_errors=False) self.save_intermediate_img(sky_result, 'sky_init.fits') sky_result.writeto('sky_init.fits', overwrite=True) sky_data = sky_result[0].data self.logger.debug('sky image has shape %s', sky_data.shape) self.logger.info('sky correction in individual images') corrector = proc.SkyCorrector( sky_data, self.datamodel, calibid=self.datamodel.get_imgid(sky_result)) # If we do not update keyword SKYADD # there is no sky subtraction for m in data_hdul: m[0].header['SKYADD'] = True # this is a little hackish # sky corrected data_hdul_s = [corrector(m) for m in data_hdul] base_header = data_hdul_s[0][0].header else: sky_result = None data_hdul_s = data_hdul self.logger.info('Computing offsets from WCS information') finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs( data_hdul_s, baseshape, subpixshape) self.logger.debug("Relative offsetsp %s", offsetsp) self.logger.info('Shape of resized array is %s', finalshape) # Resizing target imgs data_arr_sr, regions = narray.resize_arrays( [m[0].data for m in data_hdul_s], subpixshape, offsetsp, finalshape, fill=1) if has_num_ext: self.logger.debug('Using NUM extension') masks = [ numpy.where(m['NUM'].data, 0, 1).astype('int16') for m in data_hdul ] elif has_bpm_ext: self.logger.debug('Using BPM extension') # masks = [ numpy.where(m['BPM'].data, 1, 0).astype('int16') for m in data_hdul ] else: self.logger.warning('BPM missing, use zeros instead') false_mask = numpy.zeros(baseshape, dtype='int16') masks = [false_mask for _ in data_arr_sr] self.logger.debug('resize bad pixel masks') mask_arr_r, _ = narray.resize_arrays(masks, subpixshape, offsetsp, finalshape, fill=1) if self.intermediate_results: self.logger.debug('save resized intermediate img') for idx, arr_r in enumerate(data_arr_sr): self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx) hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp, use_errors) self.save_intermediate_img(hdulist, 'result_initial1.fits') compute_cross_offsets = True if compute_cross_offsets: self.logger.debug("Compute cross-correlation of images") # regions_c = self.compute_regions(finalshape, box=200, corners=True) # Regions frm bright objects regions_c = self.compute_regions_from_objs(hdulist[0].data, finalshape, box=20) try: offsets_xy_c = self.compute_offset_xy_crosscor_regions( data_arr_sr, regions_c, refine=True, tol=1) # # Combined offsets # Offsets in numpy order, swaping offsets_xy_t = offset_xy0 - offsets_xy_c offsets_fc = offsets_xy_t[:, ::-1] offsets_fc_t = numpy.round(offsets_fc).astype('int') self.logger.debug('Total offsets: %s', offsets_xy_t) self.logger.info('Computing relative offsets from cross-corr') finalshape, offsetsp = narray.combine_shape( subpixshape, offsets_fc_t) # self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp) self.logger.info('Shape of resized array (crosscorr) is %s', finalshape) # Resizing target imgs self.logger.debug("Resize to final offsets") data_arr_sr, regions = narray.resize_arrays( [m[0].data for m in data_hdul_s], subpixshape, offsetsp, finalshape, fill=1) if self.intermediate_results: self.logger.debug('save resized intermediate2 img') for idx, arr_r in enumerate(data_arr_sr): self.save_intermediate_array(arr_r, 'interm2_%03d.fits' % idx) self.logger.debug('resize bad pixel masks') mask_arr_r, _ = narray.resize_arrays(masks, subpixshape, offsetsp, finalshape, fill=1) hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp, use_errors) self.save_intermediate_img(hdulist, 'result_initial2.fits') except Exception as error: self.logger.warning('Error during cross-correlation, %s', error) catalog, objmask = self.create_object_catalog(hdulist[0].data, border=50) data_arr_sky = [sky_result[0].data for _ in data_arr_sr] data_arr_0 = [(d[r] + s) for d, r, s in zip(data_arr_sr, regions, data_arr_sky)] data_arr_r = [d.copy() for d in data_arr_sr] for inum in range(1, rinput.iterations + 1): # superflat sf_data = self.compute_superflat(data_arr_0, objmask, regions, channels) fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True) # apply superflat data_arr_rf = data_arr_r for base, arr, reg in zip(data_arr_rf, data_arr_0, regions): arr_f = arr / sf_data #arr_f = arr base[reg] = arr_f # compute sky advanced data_arr_sky = [] data_arr_rfs = [] self.logger.info('Step %d, SC: computing advanced sky', inum) scale = rinput.sky_images_sep_time * 60 tstamps = numpy.array([info['tstamp'] for info in img_info]) for idx, hdu in enumerate(data_hdul): diff1 = tstamps - tstamps[idx] idxs1 = (diff1 > 0) & (diff1 < scale) idxs2 = (diff1 < 0) & (diff1 > -scale) l1, = numpy.nonzero(idxs1) l2, = numpy.nonzero(idxs2) limit1 = l1[-rinput.sky_images:] limit2 = l2[:rinput.sky_images] len_l1 = len(limit1) len_l2 = len(limit2) self.logger.info('For image %s, using %d-%d images)', idx, len_l1, len_l2) if len_l1 + len_l2 == 0: self.logger.error('No sky image available for frame %d', idx) raise ValueError('No sky image') skydata = [] skymasks = [] skyscales = [] my_region = regions[idx] my_sky_scale = numpy.median(data_arr_rf[idx][my_region]) for i in numpy.concatenate((limit1, limit2)): region_s = regions[i] data_s = data_arr_rf[i][region_s] mask_s = objmask[region_s] scale_s = numpy.median(data_s) skydata.append(data_s) skymasks.append(mask_s) skyscales.append(scale_s) self.logger.debug('computing background with %d frames', len(skydata)) sky, _, num = nacom.median(skydata, skymasks, scales=skyscales) # rescale sky *= my_sky_scale binmask = num == 0 if numpy.any(binmask): # We have pixels without # sky background information self.logger.warn( 'pixels without sky information when correcting %d', idx) # FIXME: during development, this is faster # sky[binmask] = sky[num != 0].mean() # To continue we interpolate over the patches narray.fixpix2(sky, binmask, out=sky, iterations=1) name = 'sky_%d_%03d.fits' % (inum, idx) fits.writeto(name, sky, overwrite=True) name = 'sky_binmask_%d_%03d.fits' % (inum, idx) fits.writeto(name, binmask.astype('int16'), overwrite=True) data_arr_sky.append(sky) arr = numpy.copy(data_arr_rf[idx]) arr[my_region] = data_arr_rf[idx][my_region] - sky data_arr_rfs.append(arr) # subtract sky advanced if self.intermediate_results: self.logger.debug('save resized intermediate img') for idx, arr_r in enumerate(data_arr_rfs): self.save_intermediate_array( arr_r, 'interm_%d_%03d.fits' % (inum, idx)) hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul, offsetsp, use_errors) self.save_intermediate_img(hdulist, 'result_%d.fits' % inum) # For next step catalog, objmask = self.create_object_catalog(hdulist[0].data, border=50) data_arr_0 = [ (d[r] + s) for d, r, s in zip(data_arr_rfs, regions, data_arr_sky) ] data_arr_r = [d.copy() for d in data_arr_rfs] result = self.create_result(frame=hdulist) self.logger.info('end of dither recipe') return result def compute_sky_advanced(self, data_hdul, omasks, base_header, use_errors): method = narray.combine.mean self.logger.info('recombine images with segmentation mask') sky_data = method([m[0].data for m in data_hdul], masks=omasks, dtype='float32') hdu = fits.PrimaryHDU(sky_data[0], header=base_header) points_no_data = (sky_data[2] == 0).sum() self.logger.debug('update created sky image result header') skyid = str(uuid.uuid1()) hdu.header['UUID'] = skyid hdu.header['history'] = "Combined {} images using '{}'".format( len(data_hdul), method.__name__) hdu.header['history'] = 'Combination time {}'.format( datetime.datetime.utcnow().isoformat()) for img in data_hdul: hdu.header['history'] = "Image {}".format( self.datamodel.get_imgid(img)) msg = "missing pixels, total: {}, fraction: {:3.1f}".format( points_no_data, points_no_data / sky_data[2].size) hdu.header['history'] = msg self.logger.debug(msg) if use_errors: varhdu = fits.ImageHDU(sky_data[1], name='VARIANCE') num = fits.ImageHDU(sky_data[2], name='MAP') sky_result = fits.HDUList([hdu, varhdu, num]) else: sky_result = fits.HDUList([hdu]) return sky_result def combine_frames(self, frames, extinction, out=None, step=0): self.logger.debug('Step %d, opening sky-subtracted frames', step) def fits_open(name): """Open FITS with memmap in readonly mode""" return fits.open(name, mode='readonly', memmap=True) frameslll = [ fits_open(frame.lastname) for frame in frames if frame.valid_target ] self.logger.debug('Step %d, opening mask frames', step) mskslll = [ fits_open(frame.resized_mask) for frame in frames if frame.valid_target ] self.logger.debug('Step %d, combining %d frames', step, len(frameslll)) try: extinc = [ pow(10, -0.4 * frame.metadata['airmass'] * extinction) for frame in frames if frame.valid_target ] data = [i['primary'].data for i in frameslll] masks = [i['primary'].data for i in mskslll] headers = [i['primary'].header for i in frameslll] out = nacom.median(data, masks, scales=extinc, dtype='float32', out=out) base_header = headers[0] hdu = fits.PrimaryHDU(out[0], header=base_header) hdu.header['history'] = "Combined %d images using '%s'" % ( len(frameslll), 'median') hdu.header['history'] = 'Combination time {}'.format( datetime.datetime.utcnow().isoformat()) for img in frameslll: hdu.header['history'] = "Image {}".format( img[0].header['uuid']) prevnum = base_header.get('NUM-NCOM', 1) hdu.header['NUM-NCOM'] = prevnum * len(frameslll) hdu.header['NUMRNAM'] = 'FullDitheredImagesRecipe' hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = 'FULL_DITHERED_IMAGE' # Headers of last image hdu.header['TSUTC2'] = headers[-1]['TSUTC2'] varhdu = fits.ImageHDU(out[1], name='VARIANCE') num = fits.ImageHDU(out[2].astype('uint8'), name='MAP') result = fits.HDUList([hdu, varhdu, num]) # saving the three extensions fits.writeto('result_i%0d.fits' % step, out[0], overwrite=True) fits.writeto('result_i%0d_var.fits' % step, out[1], overwrite=True) fits.writeto('result_i%0d_npix.fits' % step, out[2], overwrite=True) result.writeto('result_i%0d_full.fits' % step, overwrite=True) return result finally: self.logger.debug('Step %d, closing sky-subtracted frames', step) for f in frameslll: f.close() self.logger.debug('Step %d, closing mask frames', step) for f in mskslll: f.close() def resize(self, frames, shape, offsetsp, finalshape, window=None, scale=1, step=0): self.logger.info('Resizing frames and masks') for frame, rel_offset in zip(frames, offsetsp): if frame.valid_target: region, _ = narray.subarray_match(finalshape, rel_offset, shape) # Valid region frame.valid_region = region # Relative offset frame.rel_offset = rel_offset # names of frame and mask framen, maskn = name_redimensioned_frames(frame.label, step) frame.resized_base = framen frame.resized_mask = maskn self.logger.debug( '%s, valid region is %s, relative offset is %s', frame.label, custom_region_to_str(region), rel_offset) self.resize_frame_and_mask(frame, finalshape, framen, maskn, window, scale) def resize_frame_and_mask(self, frame, finalshape, framen, maskn, window, scale): self.logger.info('Resizing frame %s', frame.label) with frame.origin.open() as hdul: baseshape = hdul[0].data.shape # FIXME: Resize_fits saves the resized image in framen resize_fits(hdul, framen, finalshape, frame.valid_region, window=window, scale=scale, dtype='float32') self.logger.info('Resizing mask %s', frame.label) # We don't conserve the sum of the values of the frame here, just # expand the mask if frame.mask is None: self.logger.warning('BPM missing, use zeros instead') false_mask = numpy.zeros(baseshape, dtype='int16') hdum = fits.HDUList(fits.PrimaryHDU(false_mask)) frame.mask = hdum #DataFrame(frame=hdum) elif isinstance(frame.mask, nfcom.Extension): ename = frame.mask.name with frame.origin.open() as hdul: frame.mask = fits.HDUList(hdul[ename].copy()) resize_fits(frame.mask, maskn, finalshape, frame.valid_region, fill=1, window=window, scale=scale, conserve=False) def create_mask(self, img, seeing_fwhm, step=0): # remove_border = True # sextractor takes care of bad pixels # if seeing_fwhm is not None and seeing_fwhm > 0: # sex.config['SEEING_FWHM'] = seeing_fwhm * sex.config['PIXEL_SCALE'] if remove_border: weigthmap = 'weights4rms.fits' # Create weight map, remove n pixs from either side # using a Hannig filter # npix = 90 # w1 = npix # w2 = npix # wmap = numpy.ones_like(sf_data[0]) # cos_win1 = numpy.hanning(2 * w1) # cos_win2 = numpy.hanning(2 * w2) # wmap[:,:w1] *= cos_win1[:w1] # wmap[:,-w1:] *= cos_win1[-w1:] # wmap[:w2,:] *= cos_win2[:w2, numpy.newaxis] # wmap[-w2:,:] *= cos_win2[-w2:, numpy.newaxis] # Take the number of combined images from the combined image wm = img[2].data.copy() # Dont search objects where nimages < lower # FIXME: this is a magic number # We ignore objects in regions where we have less # than 10% of the images lower = wm.max() // 10 border = (wm < lower) fits.writeto(weigthmap, border.astype('uint8'), overwrite=True) # sex.config['WEIGHT_TYPE'] = 'MAP_WEIGHT' # FIXME: this is a magic number # sex.config['WEIGHT_THRESH'] = 50 # sex.config['WEIGHT_IMAGE'] = weigthmap else: border = None data_res = img[0].data bkg = sep.Background(data_res) data_sub = data_res - bkg self.logger.info('Runing source extraction in previous result') objects, objmask = sep.extract(data_sub, 1.5, err=bkg.globalrms, mask=border, segmentation_map=True) fits.writeto(name_segmask(step), objmask, overwrite=True) # # Plot objects # # FIXME, plot sextractor objects on top of image # patches = [] # fwhms = [] # nfirst = 0 # catalog_f = sopen(sex.config['CATALOG_NAME']) # try: # star = catalog_f.readline() # while star: # flags = star['FLAGS'] # # ignoring those objects with corrupted apertures # if flags & sexcatalog.CORRUPTED_APER: # star = catalog_f.readline() # continue # center = (star['X_IMAGE'], star['Y_IMAGE']) # wd = 10 * star['A_IMAGE'] # hd = 10 * star['B_IMAGE'] # color = 'red' # e = Ellipse(center, wd, hd, star['THETA_IMAGE'], color=color) # patches.append(e) # fwhms.append(star['FWHM_IMAGE']) # nfirst += 1 # # FIXME Plot a ellipse # star = catalog_f.readline() # finally: # catalog_f.close() # # p = PatchCollection(patches, alpha=0.4) # ax = self._figure.gca() # ax.add_collection(p) # self._figure.canvas.draw() # self._figure.savefig('figure-segmentation-overlay_%01d.png' % step) # # self.figure_fwhm_histogram(fwhms, step=step) # # # mode with an histogram # hist, edges = numpy.histogram(fwhms, 50) # idx = hist.argmax() # # seeing_fwhm = 0.5 * (edges[idx] + edges[idx + 1]) # if seeing_fwhm <= 0: # _logger.warning( # 'Seeing FHWM %f pixels is negative, reseting', seeing_fwhm) # seeing_fwhm = None # else: # _logger.info('Seeing FHWM %f pixels (%f arcseconds)', # seeing_fwhm, seeing_fwhm * sex.config['PIXEL_SCALE']) # objmask = fits.getdata(name_segmask(step)) return objmask, seeing_fwhm def compute_advanced_sky(self, targetframes, objmask, skyframes=None, target_is_sky=False, maxsep=5.0, nframes=10, step=0, save=True): if target_is_sky: skyframes = targetframes # Each frame is its closest sky frame nframes += 1 elif skyframes is None: raise ValueError('skyframes not defined') # build kdtree sarray = numpy.array([frame.metadata['mjd'] for frame in skyframes]) # shape must be (n, 1) sarray = numpy.expand_dims(sarray, axis=1) # query tarray = numpy.array([frame.metadata['mjd'] for frame in targetframes]) # shape must be (n, 1) tarray = numpy.expand_dims(tarray, axis=1) kdtree = KDTree(sarray) # 1 / minutes in a Julian day SCALE = 60.0 # max_time_sep = ri.sky_images_sep_time / 1440.0 _dis, idxs = kdtree.query(tarray, k=nframes, distance_upper_bound=maxsep * SCALE) nsky = len(sarray) for tid, idss in enumerate(idxs): try: tf = targetframes[tid] self.logger.info('Step %d, SC: computing advanced sky for %s', step, tf.label) # filter(lambda x: x < nsky, idss) locskyframes = [] for si in idss: if tid == si: # this sky frame it is the current frame, reject continue if si < nsky: self.logger.debug('Step %d, SC: %s is a sky frame', step, skyframes[si].label) locskyframes.append(skyframes[si]) self.compute_advanced_sky_for_frame(tf, locskyframes, step=step, save=save) except IndexError: self.logger.error('No sky image available for frame %s', tf.lastname) raise def compute_advanced_sky_for_frame(self, frame, skyframes, step=0, save=True): self.logger.info('Correcting sky in frame %s', frame.lastname) self.logger.info('with sky computed from frames') for i in skyframes: self.logger.info('%s', i.flat_corrected) data = [] scales = [] masks = [] # handle the FITS file to close it finally desc = [] try: for i in skyframes: filename = i.flat_corrected hdulist = fits.open(filename, mode='readonly', memmap=True) data.append(hdulist['primary'].data[i.valid_region]) desc.append(hdulist) #scales.append(numpy.median(data[-1])) if i.objmask_data is not None: masks.append(i.objmask_data) self.logger.debug('object mask is shared') elif i.objmask is not None: hdulistmask = fits.open(i.objmask, mode='readonly', memmap=True) masks.append(hdulistmask['primary'].data) desc.append(hdulistmask) self.logger.debug('object mask is particular') else: self.logger.warn('no object mask for %s', filename) self.logger.debug('computing background with %d frames', len(data)) sky, _, num = nacom.median(data, masks) #, scales=scales) finally: # Closing all FITS files for hdl in desc: hdl.close() if numpy.any(num == 0): # We have pixels without # sky background information self.logger.warn( 'pixels without sky information when correcting %s', frame.flat_corrected) binmask = num == 0 # FIXME: during development, this is faster # sky[binmask] = sky[num != 0].mean() # To continue we interpolate over the patches narray.fixpix2(sky, binmask, out=sky, iterations=1) name = name_skybackgroundmask(frame.label, step) fits.writeto(name, binmask.astype('int16'), overwrite=True) name_sky = name_skybackground(frame.label, step) fits.writeto(name_sky, sky, overwrite=True) dst = name_skysub_proc(frame.label, step) prev = frame.lastname shutil.copyfile(prev, dst) frame.lastname = dst with fits.open(frame.lastname, mode='update') as hdulist: data = hdulist['primary'].data valid = data[frame.valid_region] valid -= sky def compute_regions_from_objs(self, arr, finalshape, box=50, corners=True): regions = [] catalog, mask = self.create_object_catalog(arr, border=300) self.save_intermediate_array(mask, 'objmask.fits') # with the catalog, compute 5 objects LIMIT_AREA = 5000 NKEEP = 1 idx_small = catalog['npix'] < LIMIT_AREA objects_small = catalog[idx_small] idx_flux = objects_small['flux'].argsort() objects_nth = objects_small[idx_flux][-NKEEP:] for obj in objects_nth: print('ref is', obj['x'], obj['y']) region = nautils.image_box2d(obj['x'], obj['y'], finalshape, (box, box)) print(region) regions.append(region) return regions def create_object_catalog(self, arr, threshold=3.0, border=0): if border > 0: wmap = numpy.ones_like(arr) wmap[border:-border, border:-border] = 0 else: wmap = None bkg = sep.Background(arr) data_sub = arr - bkg objects, objmask = sep.extract(data_sub, threshold, err=bkg.globalrms * numpy.ones_like(data_sub), mask=wmap, segmentation_map=True) return objects, objmask
class BarDetectionRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() bars_nominal_positions = Requirement(prods.CoordinateList2DType, 'Nominal positions of the bars') median_filter_size = Parameter(5, 'Size of the median box') canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm') canny_high_threshold = Parameter(0.04, 'High threshold for the canny algorithm') canny_low_threshold = Parameter(0.01, 'High threshold for the canny algorithm') # Recipe Results frame = Result(prods.ProcessedImage) positions = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) ROTANG = Result(float) csupos = Result(tarray.ArrayType) csusens = Result(tarray.ArrayType) param_median_filter_size = Result(float) param_canny_high_threshold = Result(float) param_canny_low_threshold = Result(float) def run(self, rinput): logger = logging.getLogger('numina.recipes.emir') logger.info('starting processing for bars detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) try: rotang = hdr['ROTANG'] dtub, dtur = datamodel.get_dtur_from_header(hdr) csupos = datamodel.get_csup_from_header(hdr) csusens = datamodel.get_cs_from_header(hdr) except KeyError as error: logger.error(error) raise numina.exceptions.RecipeError(error) logger.debug('finding bars') arr = hdulist[0].data # Median filter logger.debug('median filtering') mfilter_size = rinput.median_filter_size arr_median = median_filter(arr, size=mfilter_size) # Image is mapped between 0 and 1 # for the full range [0: 2**16] logger.debug('image scaling to 0-1') arr_grey = normalize_raw(arr_median) # Find borders logger.debug('find borders') canny_sigma = rinput.canny_sigma # These threshols corespond roughly with # value x (2**16 - 1) high_threshold = rinput.canny_high_threshold low_threshold = rinput.canny_low_threshold edges = canny(arr_grey, sigma=canny_sigma, high_threshold=high_threshold, low_threshold=low_threshold) # Number or rows used # These other parameters cab be tuned also total = 5 maxdist = 1.0 bstart = 100 bend = 1900 positions = [] nt = total // 2 xfac = dtur[0] / EMIR_PIXSCALE yfac = -dtur[1] / EMIR_PIXSCALE vec = [yfac, xfac] logger.debug('DTU shift is %s', vec) # Based on the 'edges image' # and the table of approx positions of the slits barstab = rinput.bars_nominal_positions # Currently, we only use fields 0 and 2 # of the nominal positions file for coords in barstab: lbarid = int(coords[0]) rbarid = lbarid + 55 ref_y_coor = coords[2] + vec[1] prow = coor_to_pix_1d(ref_y_coor) - 1 fits_row = prow + 1 # FITS pixel index logger.debug('looking for bars with ids %d - %d', lbarid, rbarid) logger.debug('reference y position is Y %7.2f', ref_y_coor) # Find the position of each bar bpos = find_position(edges, prow, bstart, bend, total) nbars_found = len(bpos) # If no bar is found, append and empty token if nbars_found == 0: logger.debug('bars %d, %d not found at row %d', lbarid, rbarid, fits_row) thisres1 = (lbarid, fits_row, 0, 0, 1) thisres2 = (rbarid, fits_row, 0, 0, 1) elif nbars_found == 2: # Order values by increasing X centl, centr = sorted(bpos, key=lambda cen: cen[0]) c1 = centl[0] c2 = centr[0] logger.debug('bars found at row %d between %7.2f - %7.2f', fits_row, c1, c2) # Compute FWHM of the collapsed profile cslit = arr_grey[prow - nt:prow + nt + 1, :] pslit = cslit.mean(axis=0) # Add 1 to return FITS coordinates epos, epos_f, error = locate_bar_l(pslit, c1) thisres1 = lbarid, fits_row, epos + 1, epos_f + 1, error epos, epos_f, error = locate_bar_r(pslit, c2) thisres2 = rbarid, fits_row, epos + 1, epos_f + 1, error elif nbars_found == 1: logger.warning( 'only 1 edge found at row %d, not yet implemented', fits_row) thisres1 = (lbarid, fits_row, 0, 0, 1) thisres2 = (rbarid, fits_row, 0, 0, 1) else: logger.warning( '3 or more edges found at row %d, not yet implemented', fits_row) thisres1 = (lbarid, fits_row, 0, 0, 1) thisres2 = (rbarid, fits_row, 0, 0, 1) positions.append(thisres1) positions.append(thisres2) logger.debug('end finding bars') result = self.create_result( frame=hdulist, positions=positions, DTU=dtub, ROTANG=rotang, csupos=csupos, csusens=csusens, param_median_filter_size=rinput.median_filter_size, param_canny_high_threshold=rinput.canny_high_threshold, param_canny_low_threshold=rinput.canny_low_threshold) return result
class ArcCalibrationRecipe(EmirRecipe): """Process arc images applying wavelength calibration""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() bound_param = reqs.RefinedBoundaryModelParamRequirement() lines_catalog = Requirement(LinesCatalog, 'Catalog of lines') reduced_image = Result(prods.ProcessedImage) rectwv_coeff = Result(prods.RectWaveCoeff) reduced_55sp = Result(prods.ProcessedMOS) reduced_arc = Result(prods.ProcessedMOS) @emirdrp.decorators.loginfo def run(self, rinput): self.logger.info('starting rect.+wavecal. reduction of arc spectra') # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header con additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object (with rectification and wavelength calibration # coefficients for the particular CSU configuration of the arc image) # and HDUList object with the FITS image corresponding to 55 median # spectra of each slitlet rectwv_coeff, reduced_55sp = rectwv_coeff_from_arc_image( reduced_image, rinput.bound_param, rinput.lines_catalog, ) # generate associated ds9 region files and save them in work directory if self.intermediate_results: save_four_ds9(rectwv_coeff) # apply rectification and wavelength calibration reduced_arc = apply_rectwv_coeff(reduced_image, rectwv_coeff) # save results in result directory self.logger.info('end rect.+wavecal. reduction of arc spectra') result = self.create_result(reduced_image=reduced_image, rectwv_coeff=rectwv_coeff, reduced_55sp=reduced_55sp, reduced_arc=reduced_arc) return result def set_base_headers(self, hdr): newhdr = super(ArcCalibrationRecipe, self).set_base_headers(hdr) # Update SEC to 0 newhdr['SEC'] = 0 return newhdr
class TestPinholeRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() pinhole_nominal_positions = Requirement( prods.CoordinateList2DType, 'Nominal positions of the pinholes') shift_coordinates = Parameter( True, 'Use header information to' ' shift the pinhole positions from (0,0) ' 'to X_DTU, Y_DTU') box_half_size = Parameter(4, 'Half of the computation box size in pixels') recenter = Parameter(True, 'Recenter the pinhole coordinates') max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering') # Recipe Results frame = Result(prods.ProcessedImage) positions = Result(tarray.ArrayType) positions_alt = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) filter = Result(str) readmode = Result(str) ROTANG = Result(float) DETPA = Result(float) DTUPA = Result(float) param_recenter = Result(bool) param_max_recenter_radius = Result(float) param_box_half_size = Result(float) def run(self, rinput): _logger.info('starting processing for slit detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) _logger.debug('finding pinholes') try: filtername = hdr['FILTER'] readmode = hdr['READMODE'] rotang = hdr['ROTANG'] detpa = hdr['DETPA'] dtupa = hdr['DTUPA'] dtub, dtur = datamodel.get_dtur_from_header(hdr) except KeyError as error: _logger.error(error) raise numina.exceptions.RecipeError(error) if rinput.shift_coordinates: xdtur, ydtur, zdtur = dtur xfac = xdtur / EMIR_PIXSCALE yfac = -ydtur / EMIR_PIXSCALE vec = numpy.array([yfac, xfac]) _logger.info('shift is %s', vec) ncenters = rinput.pinhole_nominal_positions + vec else: _logger.info('using pinhole coordinates as they are') ncenters = rinput.pinhole_nominal_positions _logger.info('pinhole characterization') positions = pinhole_char(hdulist[0].data, ncenters, box=rinput.box_half_size, recenter_pinhole=rinput.recenter, maxdist=rinput.max_recenter_radius) _logger.info('alternate pinhole characterization') positions_alt = pinhole_char2( hdulist[0].data, ncenters, recenter_pinhole=rinput.recenter, recenter_half_box=rinput.box_half_size, recenter_maxdist=rinput.max_recenter_radius) result = self.create_result( frame=hdulist, positions=positions, positions_alt=positions_alt, filter=filtername, DTU=dtub, readmode=readmode, ROTANG=rotang, DETPA=detpa, DTUPA=dtupa, param_recenter=rinput.recenter, param_max_recenter_radius=rinput.max_recenter_radius, param_box_half_size=rinput.box_half_size) return result