class SimpleSkyRecipe(EmirRecipe): """Recipe to process data taken in intensity flat-field mode. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() skyframe = Result(prods.MasterSky) def run(self, rinput): _logger.info('starting sky reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=True) hdr = hdulist[0].header self.set_base_headers(hdr) result = self.create_result(skyframe=hdulist) return result
class StareSpectraRecipe(EmirRecipe): """Process images in Stare spectra mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_sky = reqs.SpectralSkyRequirement(optional=True) stare = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting stare spectra reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median) hdr = hdulist[0].header self.set_base_headers(hdr) # Update EXP to 0 hdr['EXP'] = 0 self.logger.info('end stare spectra reduction') result = self.create_result(stare=hdulist) return result
class DitherSkyRecipe(EmirRecipe): """Recipe to process data taken in dither sky mode. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() skyframe = Result(prods.MasterSky) def run(self, rinput): _logger.debug('instrument %s, mode %s', rinput.obresult.instrument, rinput.obresult.mode) _logger.info('starting sky reduction with dither') flow = self.init_filters(rinput) hdulist = basic_processing_with_segmentation(rinput, flow, method=median, errors=True) hdr = hdulist[0].header self.set_base_headers(hdr) _logger.info('end sky reduction with dither') result = self.create_result(skyframe=hdulist) return result
class TestSkyCorrectRecipe(EmirRecipe): obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = Requirement(prods.MasterIntensityFlat, 'Master Sky calibration') frame = Result(prods.ProcessedImage) def run(self, rinput): self.logger.info('starting simple sky reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median) hdr = hdulist[0].header self.set_base_headers(hdr) # Update SEC to 0 hdr['SEC'] = 0 result = self.create_result(frame=hdulist) return result
class BiasRecipe(EmirRecipe): """ Recipe to process data taken in Bias image Mode. Bias images only appear in Simple Readout mode. **Observing modes:** * Bias Image (3.1) **Inputs:** **Outputs:** * A combined bias frame, with variance extension. * Statistics of the final image per channel (mean, median, variance) **Procedure:** The list of images can be readly processed by combining them with a median algorithm. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() biasframe = Result(prods.MasterBias) def run(self, rinput): _logger.info('starting bias reduction') iinfo = gather_info_frames(rinput.obresult.frames) if iinfo: mode = iinfo[0]['readmode'] if mode.lower() not in EMIR_BIAS_MODES: msg = 'readmode %s, is not a bias mode' % mode _logger.error(msg) raise RecipeError(msg) flow = lambda x: x hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=False) pdata = hdulist[0].data # update hdu header with # reduction keywords hdr = hdulist[0].header self.set_base_headers(hdr) hdr['CCDMEAN'] = pdata.mean() _logger.info('bias reduction ended') result = self.create_result(biasframe=DataFrame(hdulist)) return result
class IntensityFlatRecipe(EmirRecipe): """Recipe to process data taken in intensity flat-field mode. Recipe to process intensity flat-fields. The flat-on and flat-off images are combined (method?) separately and the subtracted to obtain a thermal subtracted flat-field. **Observing modes:** * Intensity Flat-Field **Inputs:** * A master dark frame * Non linearity * A model of the detector. **Outputs:** * TBD **Procedure:** * A combined thermal subtracted flat field, normalized to median 1, with with variance extension and quality flag. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() flatframe = Result(prods.MasterIntensityFlat) def run(self, rinput): _logger.info('starting flat reduction') errors = True flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=errors) hdr = hdulist[0].header self.set_base_headers(hdr) mm = hdulist[0].data.mean() hdr['CCDMEAN'] = mm hdulist[0].data /= mm if errors: hdulist['variance'].data /= (mm * mm) result = self.create_result(flatframe=hdulist) return result
class SkySpecRecipe(EmirRecipe): """Recipe to process data taken in spectral sky mode. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() skyspec = Result(prods.SkySpectrum) reduced_image = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting spectral sky reduction') flow = self.init_filters(rinput) reduced_image = basic_processing_with_combination( rinput, flow, method=median, errors=True ) hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength calibration # coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv ) # save as JSON file in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # generate associated ds9 region files and save them in work directory if self.intermediate_results: save_four_ds9(rectwv_coeff) # apply rectification and wavelength calibration skyspec = apply_rectwv_coeff( reduced_image, rectwv_coeff ) self.logger.info('end sky spectral reduction') result = self.create_result( reduced_image=reduced_image, skyspec=skyspec ) return result
class StareImageRecipe2(EmirRecipe): """Process images in Stare Image Mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() result_image = Result(prods.ProcessedImage) def run(self, rinput): self.logger.info('starting stare image reduction (offline)') frames = rinput.obresult.frames datamodel = self.datamodel with extra.manage_frame(frames) as list_of: c_img = extra.combine_frames(list_of, datamodel, method=combine.mean, errors=False) flow = self.init_filters(rinput) # Correct Bias if needed # Correct Dark if needed # Correct FF processed_img = flow(c_img) hdr = processed_img[0].header self.set_base_headers(hdr) self.logger.debug('append BPM') if rinput.master_bpm is not None: self.logger.debug('using BPM from inputs') hdul_bpm = rinput.master_bpm.open() hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0]) else: self.logger.debug('using empty BPM') hdu_bpm = extra.generate_empty_bpm_hdu(processed_img[0]) # Append the BPM to the result processed_img.append(hdu_bpm) self.logger.info('end stare image (off) reduction') result = self.create_result(result_image=processed_img) return result def set_base_headers(self, hdr): """Set metadata in FITS headers.""" hdr = super(StareImageRecipe2, self).set_base_headers(hdr) # Set EXP to 0 hdr['EXP'] = 0 return hdr
class DarkRecipe(EmirRecipe): """Recipe to process data taken in Dark current image Mode. Recipe to process dark images. The dark images will be combined using the median. They do have to be of the same exposure time t. **Observing mode:** * Dark current Image (3.2) **Inputs:** **Outputs:** * A combined dark frame, with variance extension. """ master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() darkframe = Result(prods.MasterDark) def run(self, rinput): _logger.info('starting dark reduction') flow = self.init_filters(rinput) iinfo = gather_info_frames(rinput.obresult.frames) ref_exptime = 0.0 for el in iinfo[1:]: if abs(el['texp'] - ref_exptime) > 1e-4: _logger.error('image with wrong exposure time') raise RecipeError('image with wrong exposure time') hdulist = basic_processing_with_combination(rinput, flow, method=median, errors=True) pdata = hdulist[0].data # update hdu header with # reduction keywords hdr = hdulist[0].header self.set_base_headers(hdr) hdr['CCDMEAN'] = pdata.mean() _logger.info('dark reduction ended') result = self.create_result(darkframe=hdulist) return result
class IntensityFlatRecipe2(EmirRecipe): obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flatframe = Result(prods.MasterIntensityFlat) def run(self, rinput): import emirdrp.core.extra as extra from numina.array import combine _logger.info('starting flat reduction') frames = rinput.obresult.frames datamodel = self.datamodel with extra.manage_frame(frames) as list_of: c_img = extra.combine_frames2(list_of, datamodel, method=combine.mean, errors=False) self.save_intermediate_img(c_img, 'p0.fits') flow = self.init_filters(rinput) processed_img = flow(c_img) self.save_intermediate_img(processed_img, 'p1.fits') hdr = processed_img[0].header self.set_base_headers(hdr) import scipy.ndimage.filters _logger.info('median filter') data_smooth = scipy.ndimage.filters.median_filter( processed_img[0].data, size=11) self.save_intermediate_array(data_smooth, 'smooth.fits') mm = processed_img[0].data.mean() hdr['CCDMEAN'] = mm processed_img[0].data /= data_smooth self.save_intermediate_img(processed_img, 'p2.fits') result = self.create_result(master_flatframe=processed_img) return result
class SpectralFlatRecipe(EmirRecipe): """Recipe to process data taken in intensity flat-field mode.""" master_bpm = reqs.MasterBadPixelMaskRequirement() obresult = reqs.ObservationResultRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() flatframe = Result(prods.MasterSpectralFlat) def run(self, rinput): return self.create_result(flatframe=prods.MasterSpectralFlat())
class StareImageBaseRecipe(EmirRecipe): """Process images in Stare Image Mode""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement(optional=True) frame = Result(prods.ProcessedImage) def __init__(self, *args, **kwargs): super(StareImageBaseRecipe, self).__init__(*args, **kwargs) if False: self.query_options['master_sky'] = Ignore() @emirdrp.decorators.loginfo @timeit def run(self, rinput): self.logger.info('starting stare image reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination( rinput, flow, method=combine.median ) hdr = hdulist[0].header self.set_base_headers(hdr) if rinput.master_bpm: hdul_bpm = rinput.master_bpm.open() hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0]) else: hdu_bpm = extra.generate_empty_bpm_hdu(hdulist[0]) # Append the BPM to the result hdulist.append(hdu_bpm) self.logger.info('end stare image reduction') result = self.create_result(frame=hdulist) return result def set_base_headers(self, hdr): """Set metadata in FITS headers.""" hdr = super(StareImageBaseRecipe, self).set_base_headers(hdr) # Update EXP to 0 hdr['EXP'] = 0 return hdr
class MaskCheckRecipe(EmirRecipe): """ Acquire a target. Recipe for the processing of multi-slit/long-slit check images. **Observing modes:** * MSM and LSM check """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() def run(self, rinput): self.logger.info("Start MaskCheckRecipe") self.logger.info("End MaskCheckRecipe") return self.create_result()
class BarDetectionRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() bars_nominal_positions = Requirement(prods.CoordinateList2DType, 'Nominal positions of the bars') median_filter_size = Parameter(5, 'Size of the median box') canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm') canny_high_threshold = Parameter(0.04, 'High threshold for the canny algorithm') canny_low_threshold = Parameter(0.01, 'High threshold for the canny algorithm') # Recipe Results frame = Result(prods.ProcessedImage) positions = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) ROTANG = Result(float) csupos = Result(tarray.ArrayType) csusens = Result(tarray.ArrayType) param_median_filter_size = Result(float) param_canny_high_threshold = Result(float) param_canny_low_threshold = Result(float) def run(self, rinput): logger = logging.getLogger('numina.recipes.emir') logger.info('starting processing for bars detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) try: rotang = hdr['ROTANG'] dtub, dtur = datamodel.get_dtur_from_header(hdr) csupos = datamodel.get_csup_from_header(hdr) csusens = datamodel.get_cs_from_header(hdr) except KeyError as error: logger.error(error) raise numina.exceptions.RecipeError(error) logger.debug('finding bars') arr = hdulist[0].data # Median filter logger.debug('median filtering') mfilter_size = rinput.median_filter_size arr_median = median_filter(arr, size=mfilter_size) # Image is mapped between 0 and 1 # for the full range [0: 2**16] logger.debug('image scaling to 0-1') arr_grey = normalize_raw(arr_median) # Find borders logger.debug('find borders') canny_sigma = rinput.canny_sigma # These threshols corespond roughly with # value x (2**16 - 1) high_threshold = rinput.canny_high_threshold low_threshold = rinput.canny_low_threshold edges = canny(arr_grey, sigma=canny_sigma, high_threshold=high_threshold, low_threshold=low_threshold) # Number or rows used # These other parameters cab be tuned also total = 5 maxdist = 1.0 bstart = 100 bend = 1900 positions = [] nt = total // 2 xfac = dtur[0] / EMIR_PIXSCALE yfac = -dtur[1] / EMIR_PIXSCALE vec = [yfac, xfac] logger.debug('DTU shift is %s', vec) # Based on the 'edges image' # and the table of approx positions of the slits barstab = rinput.bars_nominal_positions # Currently, we only use fields 0 and 2 # of the nominal positions file for coords in barstab: lbarid = int(coords[0]) rbarid = lbarid + 55 ref_y_coor = coords[2] + vec[1] prow = coor_to_pix_1d(ref_y_coor) - 1 fits_row = prow + 1 # FITS pixel index logger.debug('looking for bars with ids %d - %d', lbarid, rbarid) logger.debug('reference y position is Y %7.2f', ref_y_coor) # Find the position of each bar bpos = find_position(edges, prow, bstart, bend, total) nbars_found = len(bpos) # If no bar is found, append and empty token if nbars_found == 0: logger.debug('bars %d, %d not found at row %d', lbarid, rbarid, fits_row) thisres1 = (lbarid, fits_row, 0, 0, 1) thisres2 = (rbarid, fits_row, 0, 0, 1) elif nbars_found == 2: # Order values by increasing X centl, centr = sorted(bpos, key=lambda cen: cen[0]) c1 = centl[0] c2 = centr[0] logger.debug('bars found at row %d between %7.2f - %7.2f', fits_row, c1, c2) # Compute FWHM of the collapsed profile cslit = arr_grey[prow - nt:prow + nt + 1, :] pslit = cslit.mean(axis=0) # Add 1 to return FITS coordinates epos, epos_f, error = locate_bar_l(pslit, c1) thisres1 = lbarid, fits_row, epos + 1, epos_f + 1, error epos, epos_f, error = locate_bar_r(pslit, c2) thisres2 = rbarid, fits_row, epos + 1, epos_f + 1, error elif nbars_found == 1: logger.warning( 'only 1 edge found at row %d, not yet implemented', fits_row) thisres1 = (lbarid, fits_row, 0, 0, 1) thisres2 = (rbarid, fits_row, 0, 0, 1) else: logger.warning( '3 or more edges found at row %d, not yet implemented', fits_row) thisres1 = (lbarid, fits_row, 0, 0, 1) thisres2 = (rbarid, fits_row, 0, 0, 1) positions.append(thisres1) positions.append(thisres2) logger.debug('end finding bars') result = self.create_result( frame=hdulist, positions=positions, DTU=dtub, ROTANG=rotang, csupos=csupos, csusens=csusens, param_median_filter_size=rinput.median_filter_size, param_canny_high_threshold=rinput.canny_high_threshold, param_canny_low_threshold=rinput.canny_low_threshold) return result
class TestSlitDetectionRecipe(EmirRecipe): # Recipe Requirements obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() median_filter_size = Parameter(5, 'Size of the median box') canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm') canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm') canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm') # Recipe Results frame = Result(prods.ProcessedImage) slitstable = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) ROTANG = Result(float) DETPA = Result(float) DTUPA = Result(float) def run(self, rinput): self.logger.info('starting slit processing') self.logger.info('basic image reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) try: rotang = hdr['ROTANG'] detpa = hdr['DETPA'] dtupa = hdr['DTUPA'] dtub, dtur = datamodel.get_dtur_from_header(hdr) except KeyError as error: self.logger.error(error) raise RecipeError(error) self.logger.debug('finding slits') # Filter values below 0.0 self.logger.debug('Filter values below 0') data1 = hdulist[0].data[:] data1[data1 < 0.0] = 0.0 # First, prefilter with median median_filter_size = rinput.median_filter_size canny_sigma = rinput.canny_sigma self.logger.debug('Median filter with box %d', median_filter_size) data2 = median_filter(data1, size=median_filter_size) # Grey level image img_grey = normalize_raw(data2) # Find edges with Canny self.logger.debug('Find edges, Canny sigma %f', canny_sigma) # These thresholds corespond roughly with # value x (2**16 - 1) high_threshold = rinput.canny_high_threshold low_threshold = rinput.canny_low_threshold self.logger.debug('Find edges, Canny high threshold %f', high_threshold) self.logger.debug('Find edges, Canny low threshold %f', low_threshold) edges = canny(img_grey, sigma=canny_sigma, high_threshold=high_threshold, low_threshold=low_threshold) # Fill edges self.logger.debug('Fill holes') # I do a dilation and erosion to fill # possible holes in 'edges' fill = ndimage.binary_dilation(edges) fill2 = ndimage.binary_fill_holes(fill) fill_slits = ndimage.binary_erosion(fill2) self.logger.debug('Label objects') label_objects, nb_labels = ndimage.label(fill_slits) self.logger.debug('%d objects found', nb_labels) ids = list(six.moves.range(1, nb_labels + 1)) self.logger.debug('Find regions and centers') regions = ndimage.find_objects(label_objects) centers = ndimage.center_of_mass(data2, labels=label_objects, index=ids ) table = char_slit(data2, regions, slit_size_ratio=-1.0 ) result = self.create_result(frame=hdulist, slitstable=table, DTU=dtub, ROTANG=rotang, DETPA=detpa, DTUPA=dtupa ) return result
class TestPinholeRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() pinhole_nominal_positions = Requirement( prods.CoordinateList2DType, 'Nominal positions of the pinholes') shift_coordinates = Parameter( True, 'Use header information to' ' shift the pinhole positions from (0,0) ' 'to X_DTU, Y_DTU') box_half_size = Parameter(4, 'Half of the computation box size in pixels') recenter = Parameter(True, 'Recenter the pinhole coordinates') max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering') # Recipe Results frame = Result(prods.ProcessedImage) positions = Result(tarray.ArrayType) positions_alt = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) filter = Result(str) readmode = Result(str) ROTANG = Result(float) DETPA = Result(float) DTUPA = Result(float) param_recenter = Result(bool) param_max_recenter_radius = Result(float) param_box_half_size = Result(float) def run(self, rinput): _logger.info('starting processing for slit detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) _logger.debug('finding pinholes') try: filtername = hdr['FILTER'] readmode = hdr['READMODE'] rotang = hdr['ROTANG'] detpa = hdr['DETPA'] dtupa = hdr['DTUPA'] dtub, dtur = datamodel.get_dtur_from_header(hdr) except KeyError as error: _logger.error(error) raise numina.exceptions.RecipeError(error) if rinput.shift_coordinates: xdtur, ydtur, zdtur = dtur xfac = xdtur / EMIR_PIXSCALE yfac = -ydtur / EMIR_PIXSCALE vec = numpy.array([yfac, xfac]) _logger.info('shift is %s', vec) ncenters = rinput.pinhole_nominal_positions + vec else: _logger.info('using pinhole coordinates as they are') ncenters = rinput.pinhole_nominal_positions _logger.info('pinhole characterization') positions = pinhole_char(hdulist[0].data, ncenters, box=rinput.box_half_size, recenter_pinhole=rinput.recenter, maxdist=rinput.max_recenter_radius) _logger.info('alternate pinhole characterization') positions_alt = pinhole_char2( hdulist[0].data, ncenters, recenter_pinhole=rinput.recenter, recenter_half_box=rinput.box_half_size, recenter_maxdist=rinput.max_recenter_radius) result = self.create_result( frame=hdulist, positions=positions, positions_alt=positions_alt, filter=filtername, DTU=dtub, readmode=readmode, ROTANG=rotang, DETPA=detpa, DTUPA=dtupa, param_recenter=rinput.recenter, param_max_recenter_radius=rinput.max_recenter_radius, param_box_half_size=rinput.box_half_size) return result
class ArcCalibrationRecipe(EmirRecipe): """Process arc images applying wavelength calibration""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() bound_param = reqs.RefinedBoundaryModelParamRequirement() lines_catalog = Requirement(LinesCatalog, 'Catalog of lines') reduced_image = Result(prods.ProcessedImage) rectwv_coeff = Result(prods.RectWaveCoeff) reduced_55sp = Result(prods.ProcessedMOS) reduced_arc = Result(prods.ProcessedMOS) @emirdrp.decorators.loginfo def run(self, rinput): self.logger.info('starting rect.+wavecal. reduction of arc spectra') # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header con additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object (with rectification and wavelength calibration # coefficients for the particular CSU configuration of the arc image) # and HDUList object with the FITS image corresponding to 55 median # spectra of each slitlet rectwv_coeff, reduced_55sp = rectwv_coeff_from_arc_image( reduced_image, rinput.bound_param, rinput.lines_catalog, ) # generate associated ds9 region files and save them in work directory if self.intermediate_results: save_four_ds9(rectwv_coeff) # apply rectification and wavelength calibration reduced_arc = apply_rectwv_coeff(reduced_image, rectwv_coeff) # save results in result directory self.logger.info('end rect.+wavecal. reduction of arc spectra') result = self.create_result(reduced_image=reduced_image, rectwv_coeff=rectwv_coeff, reduced_55sp=reduced_55sp, reduced_arc=reduced_arc) return result def set_base_headers(self, hdr): newhdr = super(ArcCalibrationRecipe, self).set_base_headers(hdr) # Update SEC to 0 newhdr['SEC'] = 0 return newhdr
class SpecFlatLowFreq(EmirRecipe): """Process continuum exposures of continuum lamp (lamp ON-OFF) """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True) master_rectwv = reqs.MasterRectWaveRequirement(optional=True) # note that 'sum' is not allowed as combination method method = Parameter('sigmaclip', description='Combination method', choices=['mean', 'median', 'sigmaclip']) method_kwargs = Parameter(dict(), description='Arguments for combination method', optional=True) minimum_slitlet_width_mm = Parameter( float(EMIR_MINIMUM_SLITLET_WIDTH_MM), description='Minimum width (mm) for a valid slitlet', optional=True) maximum_slitlet_width_mm = Parameter( float(EMIR_MAXIMUM_SLITLET_WIDTH_MM), description='Maximum width (mm) for a valid slitlet', optional=True) global_integer_offset_x_pix = Parameter( 0, description='Global offset (pixels) in wavelength direction (integer)', optional=True) global_integer_offset_y_pix = Parameter( 0, description='Global offset (pixels) in spatial direction (integer)', optional=True) nwindow_x_median = Parameter( 301, description='Window size to smooth the flatfield in the spectral ' 'direction', optional=True) nwindow_y_median = Parameter( 11, description='Window size to smooth the flatfield in the spatial ' 'direction', optional=True) minimum_fraction = Parameter( 0.01, description='Minimum useful flatfielding value (fraction of maximum)', optional=True) minimum_value_in_output = Parameter( 0.01, description='Minimum value allowed in output: pixels below this value' ' are set to 1.0', optional=True) maximum_value_in_output = Parameter( 10.0, description='Maximum value allowed in output: pixels above this value' ' are set to 1.0', optional=True) debugplot = Parameter(0, description='Debugger parameter', optional=True) reduced_flatlowfreq = Result(prods.MasterSpectralFlat) def run(self, rinput): self.logger.info('starting generation of flatlowfreq') self.logger.info('rectwv_coeff..........................: {}'.format( rinput.rectwv_coeff)) self.logger.info('master_rectwv.........................: {}'.format( rinput.master_rectwv)) self.logger.info('Minimum slitlet width (mm)............: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info('Maximum slitlet width (mm)............: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info('Global offset X direction (pixels)....: {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info('Global offset Y direction (pixels)....: {}'.format( rinput.global_integer_offset_y_pix)) self.logger.info('nwindow_x_median......................: {}'.format( rinput.nwindow_x_median)) self.logger.info('nwindow_y_median......................: {}'.format( rinput.nwindow_y_median)) self.logger.info('Minimum fraction......................: {}'.format( rinput.minimum_fraction)) self.logger.info('Minimum value in output...............: {}'.format( rinput.minimum_value_in_output)) self.logger.info('Maximum value in output...............: {}'.format( rinput.maximum_value_in_output)) # check rectification and wavelength calibration information if rinput.master_rectwv is None and rinput.rectwv_coeff is None: raise ValueError('No master_rectwv nor rectwv_coeff data have ' 'been provided') elif rinput.master_rectwv is not None and \ rinput.rectwv_coeff is not None: self.logger.warning('rectwv_coeff will be used instead of ' 'master_rectwv') if rinput.rectwv_coeff is not None and \ (rinput.global_integer_offset_x_pix != 0 or rinput.global_integer_offset_y_pix != 0): raise ValueError('global_integer_offsets cannot be used ' 'simultaneously with rectwv_coeff') # check headers to detect lamp status (on/off) list_lampincd = [] for fname in rinput.obresult.frames: with fname.open() as f: list_lampincd.append(f[0].header['lampincd']) # check number of images nimages = len(rinput.obresult.frames) n_on = list_lampincd.count(1) n_off = list_lampincd.count(0) self.logger.info( 'Number of images with lamp ON.........: {}'.format(n_on)) self.logger.info( 'Number of images with lamp OFF........: {}'.format(n_off)) self.logger.info( 'Total number of images................: {}'.format(nimages)) if n_on == 0: raise ValueError('Insufficient number of images with lamp ON') if n_on + n_off != nimages: raise ValueError('Number of images does not match!') # check combination method if rinput.method_kwargs == {}: method_kwargs = None else: if rinput.method == 'sigmaclip': method_kwargs = rinput.method_kwargs else: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # build object to proceed with bpm, bias, and dark (not flat) flow = self.init_filters(rinput) # available combination methods method = getattr(combine, rinput.method) # basic reduction of images with lamp ON or OFF lampmode = {0: 'off', 1: 'on'} reduced_image_on = None reduced_image_off = None for imode in lampmode.keys(): self.logger.info('starting basic reduction of images with' ' lamp {}'.format(lampmode[imode])) tmplist = [ rinput.obresult.frames[i] for i, lampincd in enumerate(list_lampincd) if lampincd == imode ] if len(tmplist) > 0: with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in tmplist ] reduced_image = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) if imode == 0: reduced_image_off = flow(reduced_image) hdr = reduced_image_off[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced_image_off, 'reduced_image_off.fits') elif imode == 1: reduced_image_on = flow(reduced_image) hdr = reduced_image_on[0].header self.set_base_headers(hdr) self.save_intermediate_img(reduced_image_on, 'reduced_image_on.fits') else: raise ValueError('Unexpected imode={}'.format(imode)) # computation of ON-OFF header_on = reduced_image_on[0].header data_on = reduced_image_on[0].data.astype('float32') if n_off > 0: header_off = reduced_image_off[0].header data_off = reduced_image_off[0].data.astype('float32') else: header_off = None data_off = np.zeros_like(data_on) reduced_data = data_on - data_off # update reduced image header reduced_image = self.create_reduced_image(rinput, reduced_data, header_on, header_off, list_lampincd) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # define rectification and wavelength calibration coefficients if rinput.rectwv_coeff is None: rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv) # set global offsets rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix else: rectwv_coeff = rinput.rectwv_coeff # save as JSON in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # apply global offsets (to both, the original and the cleaned version) image2d = apply_integer_offsets( image2d=reduced_data, offx=rectwv_coeff.global_integer_offset_x_pix, offy=rectwv_coeff.global_integer_offset_y_pix) # load CSU configuration csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header) # determine (pseudo) longslits dict_longslits = csu_conf.pseudo_longslits() # valid slitlet numbers list_valid_islitlets = list(range(1, EMIR_NBARS + 1)) for idel in rectwv_coeff.missing_slitlets: self.logger.info('-> Removing slitlet (not defined): ' + str(idel)) list_valid_islitlets.remove(idel) # filter out slitlets with widths outside valid range list_outside_valid_width = [] for islitlet in list_valid_islitlets: slitwidth = csu_conf.csu_bar_slit_width(islitlet) if (slitwidth < rinput.minimum_slitlet_width_mm) or \ (slitwidth > rinput.maximum_slitlet_width_mm): list_outside_valid_width.append(islitlet) self.logger.info('-> Removing slitlet (width out of range): ' + str(islitlet)) if len(list_outside_valid_width) > 0: for idel in list_outside_valid_width: list_valid_islitlets.remove(idel) # initialize rectified image image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1)) # main loop grism_name = rectwv_coeff.tags['grism'] filter_name = rectwv_coeff.tags['filter'] cout = '0' debugplot = rinput.debugplot for islitlet in list(range(1, EMIR_NBARS + 1)): if islitlet in list_valid_islitlets: # define Slitlet2D object slt = Slitlet2D(islitlet=islitlet, rectwv_coeff=rectwv_coeff, debugplot=debugplot) if abs(slt.debugplot) > 10: print(slt) # extract (distorted) slitlet from the initial image slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d, subtitle='original image') # rectify slitlet slitlet2d_rect = slt.rectify( slitlet2d=slitlet2d, resampling=2, subtitle='original (cleaned) rectified') naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape if naxis1_slitlet2d != EMIR_NAXIS1: print('naxis1_slitlet2d: ', naxis1_slitlet2d) print('EMIR_NAXIS1.....: ', EMIR_NAXIS1) raise ValueError("Unexpected naxis1_slitlet2d") # get useful slitlet region (use boundaries) spectrail = slt.list_spectrails[0] yy0 = slt.corr_yrect_a + \ slt.corr_yrect_b * spectrail(slt.x0_reference) ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig spectrail = slt.list_spectrails[2] yy0 = slt.corr_yrect_a + \ slt.corr_yrect_b * spectrail(slt.x0_reference) ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig # median spectrum sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0) # smooth median spectrum along the spectral direction sp_median = ndimage.median_filter(sp_collapsed, 5, mode='nearest') ymax_spmedian = sp_median.max() y_threshold = ymax_spmedian * rinput.minimum_fraction lremove = np.where(sp_median < y_threshold) sp_median[lremove] = 0.0 if abs(slt.debugplot) % 10 != 0: xaxis1 = np.arange(1, naxis1_slitlet2d + 1) title = 'Slitlet#' + str(islitlet) + ' (median spectrum)' ax = ximplotxy(xaxis1, sp_collapsed, title=title, show=False, **{'label': 'collapsed spectrum'}) ax.plot(xaxis1, sp_median, label='fitted spectrum') ax.plot([1, naxis1_slitlet2d], 2 * [y_threshold], label='threshold') # ax.plot(xknots, yknots, 'o', label='knots') ax.legend() ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian) pause_debugplot(slt.debugplot, pltshow=True, tight_layout=True) # generate rectified slitlet region filled with the # median spectrum slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1)) if abs(slt.debugplot) % 10 != 0: slt.ximshow_rectified( slitlet2d_rect=slitlet2d_rect_spmedian, subtitle='rectified, filled with median spectrum') # compute smooth surface # clipped region slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy() slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0 slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0 # unrectified clipped image slitlet2d_unrect_clipped = slt.rectify( slitlet2d=slitlet2d_rect_clipped, resampling=2, inverse=True, subtitle='unrectified, filled with median spectrum ' '(clipped)') # normalize initial slitlet image (avoid division by zero) slitlet2d_norm_clipped = np.zeros_like(slitlet2d) for j in range(naxis1_slitlet2d): for i in range(naxis2_slitlet2d): den = slitlet2d_unrect_clipped[i, j] if den == 0: slitlet2d_norm_clipped[i, j] = 1.0 else: slitlet2d_norm_clipped[i, j] = \ slitlet2d[i, j] / den # set to 1.0 one additional pixel at each side (since # 'den' above is small at the borders and generates wrong # bright pixels) slitlet2d_norm_clipped = fix_pix_borders( image2d=slitlet2d_norm_clipped, nreplace=1, sought_value=1.0, replacement_value=1.0) slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose() slitlet2d_norm_clipped = fix_pix_borders( image2d=slitlet2d_norm_clipped, nreplace=1, sought_value=1.0, replacement_value=1.0) slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose() slitlet2d_norm_smooth = ndimage.median_filter( slitlet2d_norm_clipped, size=(rinput.nwindow_y_median, rinput.nwindow_x_median), mode='nearest') if abs(slt.debugplot) % 10 != 0: slt.ximshow_unrectified( slitlet2d=slitlet2d_norm_clipped, subtitle='unrectified, pixel-to-pixel (clipped)') slt.ximshow_unrectified( slitlet2d=slitlet2d_norm_smooth, subtitle='unrectified, pixel-to-pixel (smoothed)') # --- # check for (pseudo) longslit with previous and next slitlet imin = dict_longslits[islitlet].imin() imax = dict_longslits[islitlet].imax() if islitlet > 1: same_slitlet_below = (islitlet - 1) >= imin else: same_slitlet_below = False if islitlet < EMIR_NBARS: same_slitlet_above = (islitlet + 1) <= imax else: same_slitlet_above = False for j in range(EMIR_NAXIS1): xchannel = j + 1 y0_lower = slt.list_frontiers[0](xchannel) y0_upper = slt.list_frontiers[1](xchannel) n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower, y0_frontier_upper=y0_upper, resize=True) # note that n1 and n2 are scans (ranging from 1 to NAXIS2) nn1 = n1 - slt.bb_ns1_orig + 1 nn2 = n2 - slt.bb_ns1_orig + 1 image2d_flatfielded[(n1 - 1):n2, j] = \ slitlet2d_norm_smooth[(nn1 - 1):nn2, j] # force to 1.0 region around frontiers if not same_slitlet_below: image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1 if not same_slitlet_above: image2d_flatfielded[(n2 - 5):n2, j] = 1 cout += '.' else: cout += 'i' if islitlet % 10 == 0: if cout != 'i': cout = str(islitlet // 10) self.logger.info(cout) # restore global offsets image2d_flatfielded = apply_integer_offsets( image2d=image2d_flatfielded, offx=-rectwv_coeff.global_integer_offset_x_pix, offy=-rectwv_coeff.global_integer_offset_y_pix) # set pixels below minimum value to 1.0 filtered = np.where( image2d_flatfielded < rinput.minimum_value_in_output) image2d_flatfielded[filtered] = 1.0 # set pixels above maximum value to 1.0 filtered = np.where( image2d_flatfielded > rinput.maximum_value_in_output) image2d_flatfielded[filtered] = 1.0 # update image header reduced_flatlowfreq = self.create_reduced_image( rinput, image2d_flatfielded, header_on, header_off, list_lampincd) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # save results in results directory self.logger.info('end of flatlowfreq generation') result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq) return result def create_reduced_image(self, rinput, reduced_data, header_on, header_off, list_lampincd): with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] # Copy header of first image base_header = hduls[0][0].header.copy() hdu = fits.PrimaryHDU(reduced_data, header=base_header) self.set_base_headers(hdu.header) self.logger.debug('update result header') # update additional keywords hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = 'flatlowfreq' hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2'] hdu.header['history'] = "Processed flatlowfreq" hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames') # update history dm = emirdrp.datamodel.EmirDataModel() for img, lampincd in zip(hduls, list_lampincd): imgid = dm.get_imgid(img) hdu.header['HISTORY'] = "Image '{}' has lampincd='{}'".format( imgid, lampincd) hdu.header['HISTORY'] = "Processed flatlowfreq" hdu.header['HISTORY'] = '--- Reduction of images with lamp ON ---' for line in header_on['HISTORY']: hdu.header['HISTORY'] = line if header_off is not None: hdu.header['HISTORY'] = '--- Reduction of images with lamp OFF ---' for line in header_off['HISTORY']: hdu.header['HISTORY'] = line hdu.header.add_history('--- rinput.stored() (BEGIN) ---') for item in rinput.stored(): value = getattr(rinput, item) cline = '{}: {}'.format(item, value) hdu.header.add_history(cline) hdu.header.add_history('--- rinput.stored() (END) ---') result = fits.HDUList([hdu]) return result def set_base_headers(self, hdr): newhdr = super(SpecFlatLowFreq, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class ABBASpectraFastRectwv(EmirRecipe): """Process AB or ABBA images applying a single wavelength calibration Note that the wavelength calibration has already been determined. """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement() pattern = Parameter('ABBA', description='Observation pattern', choices=['AB', 'ABBA']) # do not allow 'sum' as combination method in order to avoid # confusion with number of counts in resulting image method = Parameter('mean', description='Combination method', choices=['mean', 'median', 'sigmaclip']) method_kwargs = Parameter(dict(), description='Arguments for combination method') voffset_pix = Parameter(0.0, description='Shift (pixels) to move A into B', optional=True) reduced_mos_abba = Result(prods.ProcessedMOS) reduced_mos_abba_combined = Result(prods.ProcessedMOS) def run(self, rinput): nimages = len(rinput.obresult.frames) pattern = rinput.pattern pattern_length = len(pattern) # check combination method if rinput.method != 'sigmaclip': if rinput.method_kwargs != {}: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # check pattern sequence matches number of images if nimages % pattern_length != 0: raise ValueError('Number of images is not a multiple of pattern ' 'length: {}, {}'.format(nimages, pattern_length)) nsequences = nimages // pattern_length rectwv_coeff = rinput.rectwv_coeff self.logger.info(rectwv_coeff) self.logger.info('observation pattern: {}'.format(pattern)) self.logger.info('nsequences.........: {}'.format(nsequences)) full_set = pattern * nsequences self.logger.info('full set of images.: {}'.format(full_set)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # available combination methods method = getattr(combine, rinput.method) method_kwargs = rinput.method_kwargs # basic reduction of A images list_a = [ rinput.obresult.frames[i] for i, char in enumerate(full_set) if char == 'A' ] with contextlib.ExitStack() as stack: self.logger.info('starting basic reduction of A images') hduls = [stack.enter_context(fname.open()) for fname in list_a] reduced_image_a = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) reduced_image_a = flow(reduced_image_a) hdr = reduced_image_a[0].header self.set_base_headers(hdr) # basic reduction of B images list_b = [ rinput.obresult.frames[i] for i, char in enumerate(full_set) if char == 'B' ] with contextlib.ExitStack() as stack: self.logger.info('starting basic reduction of B images') hduls = [stack.enter_context(fname.open()) for fname in list_b] reduced_image_b = combine_imgs(hduls, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) reduced_image_b = flow(reduced_image_b) hdr = reduced_image_b[0].header self.set_base_headers(hdr) # save intermediate reduced_image_a and reduced_image_b self.save_intermediate_img(reduced_image_a, 'reduced_image_a.fits') self.save_intermediate_img(reduced_image_b, 'reduced_image_b.fits') # computation of A-B header_a = reduced_image_a[0].header header_b = reduced_image_b[0].header data_a = reduced_image_a[0].data.astype('float32') data_b = reduced_image_b[0].data.astype('float32') reduced_data = data_a - data_b # update reduced image header reduced_image = self.create_reduced_image( rinput, reduced_data, header_a, header_b, rinput.pattern, full_set, voffset_pix=0, header_mos_abba=None, ) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # apply rectification and wavelength calibration self.logger.info('begin rect.+wavecal. reduction of ABBA spectra') reduced_mos_abba = apply_rectwv_coeff(reduced_image, rectwv_coeff) header_mos_abba = reduced_mos_abba[0].header # combine A and B data by shifting B on top of A voffset_pix = rinput.voffset_pix if voffset_pix is not None and voffset_pix != 0: self.logger.info( 'correcting vertical offset (pixesl): {}'.format(voffset_pix)) reduced_mos_abba_data = reduced_mos_abba[0].data.astype('float32') shifted_a_minus_b_data = shift_image2d( reduced_mos_abba_data, yoffset=-voffset_pix, ).astype('float32') reduced_mos_abba_combined_data = \ reduced_mos_abba_data - shifted_a_minus_b_data # scale signal to exposure of a single image reduced_mos_abba_combined_data /= 2.0 else: reduced_mos_abba_combined_data = None # update reduced combined image header reduced_mos_abba_combined = self.create_reduced_image( rinput, reduced_mos_abba_combined_data, header_a, header_b, rinput.pattern, full_set, voffset_pix, header_mos_abba=header_mos_abba) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos_abba, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of ABBA spectra') result = self.create_result( reduced_mos_abba=reduced_mos_abba, reduced_mos_abba_combined=reduced_mos_abba_combined) return result def create_reduced_image(self, rinput, reduced_data, header_a, header_b, pattern, full_set, voffset_pix, header_mos_abba): with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] # Copy header of first image base_header = hduls[0][0].header.copy() hdu = fits.PrimaryHDU(reduced_data, header=base_header) self.set_base_headers(hdu.header) self.logger.debug('update result header') if header_mos_abba is not None: self.logger.debug('update result header') crpix1 = header_mos_abba['crpix1'] crval1 = header_mos_abba['crval1'] cdelt1 = header_mos_abba['cdelt1'] # update wavelength calibration in FITS header for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']: if keyword in hdu.header: hdu.header.remove(keyword) hdu.header['crpix1'] = (crpix1, 'reference pixel') hdu.header['crval1'] = (crval1, 'central wavelength at crpix1') hdu.header['cdelt1'] = \ (cdelt1, 'linear dispersion (Angstrom/pixel)') hdu.header['cunit1'] = ('Angstrom', 'units along axis1') hdu.header['ctype1'] = 'WAVELENGTH' hdu.header['crpix2'] = (0.0, 'reference pixel') hdu.header['crval2'] = (0.0, 'central value at crpix2') hdu.header['cdelt2'] = (1.0, 'increment') hdu.header['ctype2'] = 'PIXEL' hdu.header['cunit2'] = ('Pixel', 'units along axis2') for keyword in [ 'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2', 'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2' ]: if keyword in hdu.header: hdu.header.remove(keyword) # update additional keywords hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = pattern + ' pattern' hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2'] hdu.header['history'] = "Processed " + pattern + " pattern" hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames') # update history dm = emirdrp.datamodel.EmirDataModel() for img, key in zip(hduls, full_set): imgid = dm.get_imgid(img) hdu.header['HISTORY'] = "Image '{}' is '{}'".format(imgid, key) hdu.header['HISTORY'] = "Processed " + pattern + " pattern" hdu.header['HISTORY'] = '--- Reduction of A images ---' for line in header_a['HISTORY']: hdu.header['HISTORY'] = line hdu.header['HISTORY'] = '--- Reduction of B images ---' for line in header_b['HISTORY']: hdu.header['HISTORY'] = line if voffset_pix is not None and voffset_pix != 0: hdu.header['HISTORY'] = '--- Combination of AB spectra ---' hdu.header['HISTORY'] = "voffset_pix between A and B {}".format( voffset_pix) hdu.header.add_history('--- rinput.stored() (BEGIN) ---') for item in rinput.stored(): value = getattr(rinput, item) cline = '{}: {}'.format(item, value) hdu.header.add_history(cline) hdu.header.add_history('--- rinput.stored() (END) ---') result = fits.HDUList([hdu]) return result def set_base_headers(self, hdr): newhdr = super(ABBASpectraFastRectwv, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr def extract_tags_from_obsres(self, obsres, tag_keys): # this function is necessary to make use of recipe requirements # that are provided as lists final_tags = [] for frame in obsres.frames: ref_img = frame.open() tag = self.extract_tags_from_ref(ref_img, tag_keys, base=obsres.labels) final_tags.append(tag) return final_tags
class StareSpectraRectwv(EmirRecipe): """Process images in Stare spectra mode applying wavelength calibration Note that in this case the wavelength calibration has already been determined. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement() reduced_mos = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('applying existing rect.+wavecal. calibration of ' 'stare spectra') # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rinput.rectwv_coeff) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rinput.rectwv_coeff) save_spectral_lines_ds9(rinput.rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of stare spectra') result = self.create_result(reduced_mos=reduced_mos) return result def set_base_headers(self, hdr): newhdr = super(StareSpectraRectwv, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class GenerateRectwvCoeff(EmirRecipe): """Process images in Stare spectra. This recipe generates a rectified and wavelength calibrated image after applying a model (master_rectwv). This calibration can be refined (using refine_wavecalib_mode != 0). """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() refine_wavecalib_mode = Parameter( 0, description='Apply wavelength calibration refinement', choices=[0, 1, 2, 11, 12]) minimum_slitlet_width_mm = Parameter( float(EMIR_MINIMUM_SLITLET_WIDTH_MM), description='Minimum width (mm) for a valid slitlet', ) maximum_slitlet_width_mm = Parameter( float(EMIR_MAXIMUM_SLITLET_WIDTH_MM), description='Maximum width (mm) for a valid slitlet', ) global_integer_offset_x_pix = Parameter( 0, description='Global offset (pixels) in wavelength direction (integer)', ) global_integer_offset_y_pix = Parameter( 0, description='Global offset (pixels) in spatial direction (integer)', ) reduced_mos = Result(prods.ProcessedMOS) rectwv_coeff = Result(RectWaveCoeff) def run(self, rinput): self.logger.info('starting rect.+wavecal. reduction of stare spectra') self.logger.info(rinput.master_rectwv) self.logger.info('Wavelength calibration refinement mode: {}'.format( rinput.refine_wavecalib_mode)) self.logger.info('Minimum slitlet width (mm)............: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info('Maximum slitlet width (mm)............: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info('Global offset X direction (pixels)....: {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info('Global offset Y direction (pixels)....: {}'.format( rinput.global_integer_offset_y_pix)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength # calibration coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image, rinput.master_rectwv) # set global offsets rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix # apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) # wavelength calibration refinement # 0 -> no refinement # 1 -> apply global offset to all the slitlets (using ARC lines) # 2 -> apply individual offset to each slitlet (using ARC lines) # 11 -> apply global offset to all the slitlets (using OH lines) # 12 -> apply individual offset to each slitlet (using OH lines) if rinput.refine_wavecalib_mode != 0: self.logger.info( 'Refining wavelength calibration (mode={})'.format( rinput.refine_wavecalib_mode)) # refine RectWaveCoeff object rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff( reduced_mos, rectwv_coeff, rinput.refine_wavecalib_mode, rinput.minimum_slitlet_width_mm, rinput.maximum_slitlet_width_mm, save_intermediate_results=self.intermediate_results) self.save_intermediate_img(expected_catalog_lines, 'expected_catalog_lines.fits') # re-apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of stare spectra') result = self.create_result(reduced_mos=reduced_mos, rectwv_coeff=rectwv_coeff) return result def set_base_headers(self, hdr): newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class ABBASpectraRectwv(EmirRecipe): """Process images in AB or ABBA mode applying wavelength calibration Note that in this case the wavelength calibration has already been determined. """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True) list_rectwv_coeff = reqs.ListOfRectWaveCoeffRequirement(optional=True) pattern = Parameter('ABBA', description='Observation pattern', choices=['AB', 'ABBA']) # do not allow 'sum' as combination method in order to avoid # confusion with number of counts in resulting image method = Parameter('mean', description='Combination method', choices=['mean', 'median', 'sigmaclip']) method_kwargs = Parameter(dict(), description='Arguments for combination method', optional=True) refine_target_along_slitlet = Parameter( dict(), description='Parameters to refine location of target along the slitlet' ) reduced_mos_abba = Result(prods.ProcessedMOS) reduced_mos_abba_combined = Result(prods.ProcessedMOS) def run(self, rinput): nimages = len(rinput.obresult.frames) pattern = rinput.pattern pattern_length = len(pattern) # check combination method if rinput.method != 'sigmaclip': if rinput.method_kwargs != {}: raise ValueError('Unexpected method_kwargs={}'.format( rinput.method_kwargs)) # check pattern sequence matches number of images if nimages % pattern_length != 0: raise ValueError('Number of images is not a multiple of pattern ' 'length: {}, {}'.format(nimages, pattern_length)) nsequences = nimages // pattern_length # check rectification and wavelength calibration information if rinput.rectwv_coeff is None and rinput.list_rectwv_coeff is None: raise ValueError('No rectwv_coeff nor list_rectwv_coeff data have ' 'been provided') elif rinput.rectwv_coeff is not None and \ rinput.list_rectwv_coeff is not None: raise ValueError("rectwv_coeff and list_rectwv_coeff cannot be " "used simultaneously") elif rinput.rectwv_coeff is not None: list_rectwv_coeff = [rinput.rectwv_coeff] * nimages elif rinput.list_rectwv_coeff is not None: if len(rinput.list_rectwv_coeff) != nimages: raise ValueError("Unexpected number of rectwv_coeff files " "in list_rectwv_coeff") else: list_rectwv_coeff = rinput.list_rectwv_coeff # check filter and grism are the same in all JSON files for item in ['grism', 'filter']: list_values = [] for calib in list_rectwv_coeff: list_values.append(calib.tags[item]) if len(set(list_values)) != 1: raise ValueError( 'list_rectwv_coeff contains coefficients for ' 'different {}s'.format(item)) else: raise ValueError("Unexpected error!") # grism and filter names grism_name = list_rectwv_coeff[0].tags['grism'] filter_name = list_rectwv_coeff[0].tags['filter'] # compute offsets from WCS info in image headers with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] sep_arcsec, spatial_scales = compute_wcs_offsets(hduls) sep_pixel = np.round(sep_arcsec / spatial_scales, 6) for i in range(nimages): self.logger.info(list_rectwv_coeff[i]) self.logger.info( 'observation pattern..................: {}'.format(pattern)) self.logger.info( 'nsequences...........................: {}'.format(nsequences)) self.logger.info( 'offsets (arcsec, from WCS)...........: {}'.format(sep_arcsec)) self.logger.info( 'spatial scales (arcsec/pix, from WCS): {}'.format(spatial_scales)) self.logger.info( 'spatial scales (pixels, from WCS)....: {}'.format(sep_pixel)) full_set = pattern * nsequences self.logger.info( 'full set of images...................: {}'.format(full_set)) # basic parameters to determine useful pixels in the wavelength # direction dict_rtas = rinput.refine_target_along_slitlet valid_keys = [ 'npix_removed_near_ohlines', 'nwidth_medfilt', 'save_individual_images', 'ab_different_target', 'vpix_region_a_target', 'vpix_region_a_sky', 'vpix_region_b_target', 'vpix_region_b_sky', 'list_valid_wvregions_a', 'list_valid_wvregions_b' ] for dumkey in dict_rtas.keys(): if dumkey not in valid_keys: raise ValueError('Unexpected key={}'.format(dumkey)) if 'vpix_region_a_target' in dict_rtas.keys(): vpix_region_a_target = dict_rtas['vpix_region_a_target'] else: vpix_region_a_target = None if 'vpix_region_a_sky' in dict_rtas.keys(): vpix_region_a_sky = dict_rtas['vpix_region_a_sky'] else: vpix_region_a_sky = None if 'vpix_region_b_target' in dict_rtas.keys(): vpix_region_b_target = dict_rtas['vpix_region_b_target'] else: vpix_region_b_target = None if 'vpix_region_b_sky' in dict_rtas.keys(): vpix_region_b_sky = dict_rtas['vpix_region_b_sky'] else: vpix_region_b_sky = None if 'ab_different_target' in dict_rtas.keys(): ab_different_target = int(dict_rtas['ab_different_target']) if ab_different_target not in [-1, 0, 1, 9]: raise ValueError('Invalid ab_different_target={} value'.format( ab_different_target)) else: raise ValueError('Missing ab_different_target value') try: npix_removed_near_ohlines = \ int(dict_rtas['npix_removed_near_ohlines']) except KeyError: npix_removed_near_ohlines = 0 except ValueError: raise ValueError('wrong value: npix_removed_near_ohlines=' '{}'.format( dict_rtas['npix_removed_near_ohlines'])) if 'list_valid_wvregions_a' in dict_rtas.keys(): if vpix_region_a_target is None: raise ValueError('Unexpected list_valid_wvregions_a when ' 'vpix_region_a_target is not set') list_valid_wvregions_a = dict_rtas['list_valid_wvregions_a'] else: list_valid_wvregions_a = None if 'list_valid_wvregions_b' in dict_rtas.keys(): if vpix_region_b_target is None: raise ValueError('Unexpected list_valid_wvregions_b when ' 'vpix_region_b_target is not set') list_valid_wvregions_b = dict_rtas['list_valid_wvregions_b'] else: list_valid_wvregions_b = None try: nwidth_medfilt = int(dict_rtas['nwidth_medfilt']) except KeyError: nwidth_medfilt = 0 except ValueError: raise ValueError('wrong value: nwidth_medfilt={}'.format( dict_rtas['nwidth_medfilt'])) try: save_individual_images = int(dict_rtas['save_individual_images']) except KeyError: save_individual_images = 0 except ValueError: raise ValueError('wrong value: save_individual_images={}'.format( dict_rtas['save_individual_images'])) self.logger.info( 'npix_removed_near_ohlines: {}'.format(npix_removed_near_ohlines)) self.logger.info('nwidth_medfilt: {}'.format(nwidth_medfilt)) self.logger.info( 'vpix_region_a_target: {}'.format(vpix_region_a_target)) self.logger.info( 'vpix_region_b_target: {}'.format(vpix_region_b_target)) self.logger.info( 'list_valid_wvregions_a: {}'.format(list_valid_wvregions_a)) self.logger.info( 'list_valid_wvregions_b: {}'.format(list_valid_wvregions_b)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # basic reduction, rectification and wavelength calibration of # all the individual images list_reduced_mos_images = [] self.logger.info('starting reduction of individual images') for i, char in enumerate(full_set): frame = rinput.obresult.frames[i] self.logger.info('image {} ({} of {})'.format( char, i + 1, nimages)) self.logger.info('image: {}'.format(frame.filename)) with frame.open() as f: base_header = f[0].header.copy() data = f[0].data.astype('float32') grism_name_ = base_header['grism'] if grism_name_ != grism_name: raise ValueError('Incompatible grism name in ' 'rectwv_coeff.json file and FITS image') filter_name_ = base_header['filter'] if filter_name_ != filter_name: raise ValueError('Incompatible filter name in ' 'rectwv_coeff.json file and FITS image') hdu = fits.PrimaryHDU(data, header=base_header) hdu.header['UUID'] = str(uuid.uuid1()) hdul = fits.HDUList([hdu]) # basic reduction reduced_image = flow(hdul) hdr = reduced_image[0].header self.set_base_headers(hdr) # rectification and wavelength calibration reduced_mos_image = apply_rectwv_coeff(reduced_image, list_rectwv_coeff[i]) if save_individual_images != 0: self.save_intermediate_img( reduced_mos_image, 'reduced_mos_image_' + char + '_' + frame.filename[:10] + '.fits') list_reduced_mos_images.append(reduced_mos_image) # intermediate PDF file with crosscorrelation plots if self.intermediate_results: from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages('crosscorrelation_ab.pdf') else: pdf = None # compute offsets between images self.logger.info('computing offsets between individual images') first_a = True first_b = True xisok_a = None xisok_b = None reference_profile_a = None reference_profile_b = None refine_a = vpix_region_a_target is not None refine_b = vpix_region_b_target is not None list_offsets = [] for i, char in enumerate(full_set): if char == 'A' and refine_a: refine_image = True elif char == 'B' and refine_b: refine_image = True else: refine_image = False if refine_image: frame = rinput.obresult.frames[i] isky = get_isky(i, pattern) frame_sky = rinput.obresult.frames[isky] self.logger.info('image {} ({} of {})'.format( char, i + 1, nimages)) self.logger.info('image: {}'.format(frame.filename)) self.logger.info('(sky): {}'.format(frame_sky.filename)) data = list_reduced_mos_images[i][0].data.copy() data_sky = list_reduced_mos_images[isky][0].data data -= data_sky base_header = list_reduced_mos_images[i][0].header # get useful pixels in the wavelength direction if char == 'A' and first_a: xisok_a = useful_mos_xpixels( data, base_header, vpix_region=vpix_region_a_target, npix_removed_near_ohlines=npix_removed_near_ohlines, list_valid_wvregions=list_valid_wvregions_a, debugplot=0) elif char == 'B' and first_b: xisok_b = useful_mos_xpixels( data, base_header, vpix_region=vpix_region_b_target, npix_removed_near_ohlines=npix_removed_near_ohlines, list_valid_wvregions=list_valid_wvregions_b, debugplot=0) if char == 'A': nsmin = vpix_region_a_target[0] nsmax = vpix_region_a_target[1] xisok = xisok_a if vpix_region_a_sky is not None: nsmin_sky = vpix_region_a_sky[0] nsmax_sky = vpix_region_a_sky[1] skysubtraction = True else: nsmin_sky = None nsmax_sky = None skysubtraction = False elif char == 'B': nsmin = vpix_region_b_target[0] nsmax = vpix_region_b_target[1] xisok = xisok_b if vpix_region_b_sky is not None: nsmin_sky = vpix_region_b_sky[0] nsmax_sky = vpix_region_b_sky[1] skysubtraction = True else: nsmin_sky = None nsmax_sky = None skysubtraction = False else: raise ValueError('Unexpected char value: {}'.format(char)) # initial slitlet region slitlet2d = data[(nsmin - 1):nsmax, :].copy() if skysubtraction: slitlet2d_sky = data[(nsmin_sky - 1):nsmax_sky, :].copy() median_sky = np.median(slitlet2d_sky, axis=0) slitlet2d -= median_sky # selected wavelength regions after blocking OH lines slitlet2d_blocked = slitlet2d[:, xisok] # apply median filter in the X direction if nwidth_medfilt > 1: slitlet2d_blocked_smoothed = ndimage.filters.median_filter( slitlet2d_blocked, size=(1, nwidth_medfilt)) else: slitlet2d_blocked_smoothed = slitlet2d_blocked # --- # convert to 1D series of spatial profiles (note: this # option does not work very well when the signal is low; # a simple mean profile does a better job) # profile = slitlet2d_blocked_smoothed.transpose().ravel() # --- profile = np.mean(slitlet2d_blocked_smoothed, axis=1) if char == 'A' and first_a: reference_profile_a = profile.copy() elif char == 'B' and first_b: reference_profile_b = profile.copy() # --- # # To write pickle file # import pickle # with open(frame.filename[:10] + '_profile.pkl', 'wb') as f: # pickle.dump(profile, f) # # To read pickle file # with open('test.pkl', 'rb') as f: # x = pickle.load(f) # --- # crosscorrelation to find offset naround_zero = (nsmax - nsmin) // 3 if char == 'A': reference_profile = reference_profile_a elif char == 'B': reference_profile = reference_profile_b else: raise ValueError('Unexpected char value: {}'.format(char)) offset, fpeak = periodic_corr1d( sp_reference=reference_profile, sp_offset=profile, remove_mean=False, frac_cosbell=0.10, zero_padding=11, fminmax=None, nfit_peak=5, naround_zero=naround_zero, sp_label='spatial profile', plottitle='Image #{} (type {}), {}'.format( i + 1, char, frame.filename[:10]), pdf=pdf) # round to 4 decimal places if abs(offset) < 1E-4: offset = 0.0 # avoid -0.0 else: offset = round(offset, 4) list_offsets.append(offset) # end of loop if char == 'A' and first_a: first_a = False elif char == 'B' and first_b: first_b = False else: list_offsets.append(0.0) self.logger.info('computed offsets: {}'.format(list_offsets)) self.logger.info('correcting vertical offsets between individual ' 'images') list_a = [] list_b = [] for i, (char, offset) in enumerate(zip(full_set, list_offsets)): frame = rinput.obresult.frames[i] self.logger.info('image {} ({} of {})'.format( char, i + 1, nimages)) self.logger.info('image: {}'.format(frame.filename)) reduced_mos_image = list_reduced_mos_images[i] data = reduced_mos_image[0].data base_header = reduced_mos_image[0].header self.logger.info( 'correcting vertical offset (pixesl): {}'.format(offset)) if offset != 0: reduced_mos_image[0].data = shift_image2d( data, yoffset=-offset).astype('float32') base_header['HISTORY'] = 'Applying voffset_pix {}'.format(offset) if save_individual_images != 0: self.save_intermediate_img( reduced_mos_image, 'reduced_mos_image_refined_' + char + '_' + frame.filename[:10] + '.fits') # store reduced_mos_image if char == 'A': list_a.append(reduced_mos_image) elif char == 'B': list_b.append(reduced_mos_image) else: raise ValueError('Unexpected char value: {}'.format(char)) # combination method method = getattr(combine, rinput.method) method_kwargs = rinput.method_kwargs # final combination of A images self.logger.info('combining individual A images') reduced_mos_image_a = combine_imgs(list_a, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) self.save_intermediate_img(reduced_mos_image_a, 'reduced_mos_image_a.fits') # final combination of B images self.logger.info('combining individual B images') reduced_mos_image_b = combine_imgs(list_b, method=method, method_kwargs=method_kwargs, errors=False, prolog=None) self.save_intermediate_img(reduced_mos_image_b, 'reduced_mos_image_b.fits') self.logger.info('mixing A and B spectra') header_a = reduced_mos_image_a[0].header header_b = reduced_mos_image_b[0].header data_a = reduced_mos_image_a[0].data.astype('float32') data_b = reduced_mos_image_b[0].data.astype('float32') reduced_mos_abba_data = data_a - data_b # update reduced mos image header reduced_mos_abba = self.create_mos_abba_image(rinput, dict_rtas, reduced_mos_abba_data, header_a, header_b, pattern, full_set, list_offsets, voffset_pix=0) # combine A and B data by shifting B on top of A if abs(ab_different_target) == 0: len_prof_a = len(reference_profile_a) len_prof_b = len(reference_profile_b) if len_prof_a == len_prof_b: reference_profile = reference_profile_a profile = reference_profile_b naround_zero = len_prof_a // 3 elif len_prof_a > len_prof_b: ndiff = len_prof_a - len_prof_b reference_profile = reference_profile_a profile = np.concatenate( (reference_profile_b, np.zeros(ndiff, dtype='float'))) naround_zero = len_prof_a // 3 else: ndiff = len_prof_b - len_prof_a reference_profile = np.concatenate( (reference_profile_a, np.zeros(ndiff, dtype='float'))) profile = reference_profile_b naround_zero = len_prof_b // 3 offset, fpeak = periodic_corr1d( sp_reference=reference_profile, sp_offset=profile, remove_mean=False, frac_cosbell=0.10, zero_padding=11, fminmax=None, nfit_peak=5, naround_zero=naround_zero, sp_label='spatial profile', plottitle='Comparison of A and B profiles', pdf=pdf) voffset_pix = vpix_region_b_target[0] - vpix_region_a_target[0] voffset_pix += offset elif abs(ab_different_target) == 1: # apply nominal offset (from WCS info) between first A # and first B voffset_pix = sep_pixel[1] * ab_different_target elif ab_different_target == 9: voffset_pix = None else: raise ValueError( 'Invalid ab_different_target={}'.format(ab_different_target)) # close output PDF file if pdf is not None: pdf.close() if voffset_pix is not None: self.logger.info( 'correcting vertical offset (pixesl): {}'.format(voffset_pix)) shifted_a_minus_b_data = shift_image2d( reduced_mos_abba_data, yoffset=-voffset_pix, ).astype('float32') reduced_mos_abba_combined_data = \ reduced_mos_abba_data - shifted_a_minus_b_data # scale signal to exposure of a single image reduced_mos_abba_combined_data /= 2.0 else: reduced_mos_abba_combined_data = None # update reduced mos combined image header reduced_mos_abba_combined = self.create_mos_abba_image( rinput, dict_rtas, reduced_mos_abba_combined_data, header_a, header_b, pattern, full_set, list_offsets, voffset_pix) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(list_rectwv_coeff[0]) save_spectral_lines_ds9(list_rectwv_coeff[0]) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos_abba, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of ABBA spectra') result = self.create_result( reduced_mos_abba=reduced_mos_abba, reduced_mos_abba_combined=reduced_mos_abba_combined) return result def create_mos_abba_image(self, rinput, dict_rtas, reduced_mos_abba_data, header_a, header_b, pattern, full_set, list_offsets, voffset_pix): with contextlib.ExitStack() as stack: hduls = [ stack.enter_context(fname.open()) for fname in rinput.obresult.frames ] # Copy header of first image base_header = hduls[0][0].header.copy() hdu = fits.PrimaryHDU(reduced_mos_abba_data, header=base_header) self.set_base_headers(hdu.header) # check consistency of wavelength calibration paramenters for param in ['crpix1', 'crval1', 'cdelt1']: if header_a[param] != header_b[param]: raise ValueError( 'Headers of A and B images have different ' 'values of {}'.format(param)) self.logger.debug('update result header') crpix1 = header_a['crpix1'] crval1 = header_a['crval1'] cdelt1 = header_a['cdelt1'] # update wavelength calibration in FITS header for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']: if keyword in hdu.header: hdu.header.remove(keyword) hdu.header['crpix1'] = (crpix1, 'reference pixel') hdu.header['crval1'] = (crval1, 'central wavelength at crpix1') hdu.header['cdelt1'] = (cdelt1, 'linear dispersion (Angstrom/pixel)') hdu.header['cunit1'] = ('Angstrom', 'units along axis1') hdu.header['ctype1'] = 'WAVELENGTH' hdu.header['crpix2'] = (0.0, 'reference pixel') hdu.header['crval2'] = (0.0, 'central value at crpix2') hdu.header['cdelt2'] = (1.0, 'increment') hdu.header['ctype2'] = 'PIXEL' hdu.header['cunit2'] = ('Pixel', 'units along axis2') for keyword in [ 'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2', 'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2' ]: if keyword in hdu.header: hdu.header.remove(keyword) # update additional keywords hdu.header['UUID'] = str(uuid.uuid1()) hdu.header['OBSMODE'] = pattern + ' pattern' hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2'] hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames') # update history hdu.header['HISTORY'] = "Processed " + pattern + " pattern" hdu.header['HISTORY'] = '--- Reduction of A images ---' for line in header_a['HISTORY']: hdu.header['HISTORY'] = line hdu.header['HISTORY'] = '--- Reduction of B images ---' for line in header_b['HISTORY']: hdu.header['HISTORY'] = line hdu.header['HISTORY'] = '--- Combination of ABBA images ---' for key in dict_rtas: hdu.header['HISTORY'] = '{}: {}'.format(key, dict_rtas[key]) dm = emirdrp.datamodel.EmirDataModel() for img, key, offset in zip(hduls, full_set, list_offsets): imgid = dm.get_imgid(img) hdu.header['HISTORY'] = \ "Image '{}' is '{}', with voffset_pix {}".format( imgid, key, offset) if voffset_pix is not None and voffset_pix != 0: hdu.header['HISTORY'] = '--- Combination of AB spectra ---' hdu.header['HISTORY'] = "voffset_pix between A and B {}".format( voffset_pix) hdu.header.add_history('--- rinput.stored() (BEGIN) ---') for item in rinput.stored(): value = getattr(rinput, item) cline = '{}: {}'.format(item, value) hdu.header.add_history(cline) hdu.header.add_history('--- rinput.stored() (END) ---') result = fits.HDUList([hdu]) return result def set_base_headers(self, hdr): newhdr = super(ABBASpectraRectwv, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr def extract_tags_from_obsres(self, obsres, tag_keys): # this function is necessary to make use of recipe requirements # that are provided as lists final_tags = [] for frame in obsres.frames: ref_img = frame.open() tag = self.extract_tags_from_ref(ref_img, tag_keys, base=obsres.labels) final_tags.append(tag) return final_tags
class StareSpectraWaveRecipe(EmirRecipe): """Process images in Stare spectra at the GTC. This recipe is intended to be used at GTC. The rectification and wavelength calibration can computed from a model if this model (master_rectwv) is provided as input. """ obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement(optional=True) master_sky = reqs.SpectralSkyRequirement(optional=True) reduced_image = Result(prods.ProcessedImage) stare = Result(prods.ProcessedMOS) def run(self, rinput): self.logger.info('starting reduction of stare spectra') self.logger.info(rinput.master_rectwv) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=median) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # rectification and wavelength calibration (if a model has # been provided) if rinput.master_rectwv: # RectWaveCoeff object with rectification and wavelength # calibration coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library( reduced_image, rinput.master_rectwv) # apply rectification and wavelength calibration stare_image = apply_rectwv_coeff(reduced_image, rectwv_coeff) # save as JSON file in work directory self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json') # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(stare_image, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # image_wl_calibrated = True else: stare_image = reduced_image self.logger.info('No wavelength calibration provided') grism_value = hdr.get('GRISM', 'unknown') self.logger.debug('GRISM is %s', grism_value) if grism_value.lower() == 'open': self.logger.debug('GRISM is %s, so this seems OK', grism_value) # image_wl_calibrated = False if rinput.master_sky: # Sky subtraction after rectification msky = rinput.master_sky.open() # Check if images have the same size. # if so, go ahead if msky[0].data.shape != stare_image[0].data.shape: self.logger.warning( "sky and current image don't have the same shape") else: sky_corrector = proc.SkyCorrector( msky[0].data, datamodel=self.datamodel, calibid=self.datamodel.get_imgid(msky)) stare_image = sky_corrector(stare_image) else: self.logger.info('No sky image provided') # save results in results directory self.logger.info('end reduction of stare spectra') result = self.create_result(reduced_image=reduced_image, stare=stare_image) return result def set_base_headers(self, hdr): newhdr = super(StareSpectraWaveRecipe, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class TestMaskRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() pinhole_nominal_positions = Requirement( prods.CoordinateList2DType, 'Nominal positions of the pinholes') shift_coordinates = Parameter( True, 'Use header information to' ' shift the pinhole positions from (0,0) ' 'to X_DTU, Y_DTU') box_half_size = Parameter(4, 'Half of the computation box size in pixels') recenter = Parameter(True, 'Recenter the pinhole coordinates') max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering') median_filter_size = Parameter(5, 'Size of the median box') canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm') obj_min_size = Parameter(200, 'Minimum size of the slit') obj_max_size = Parameter(3000, 'Maximum size of the slit') slit_size_ratio = Parameter( 4.0, 'Minimum ratio between height and width for slits') # Recipe Results frame = Result(prods.ProcessedImage) positions = Result(tarray.ArrayType) positions_alt = Result(tarray.ArrayType) slitstable = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) filter = Result(str) readmode = Result(str) ROTANG = Result(float) DETPA = Result(float) DTUPA = Result(float) param_recenter = Result(bool) param_max_recenter_radius = Result(float) param_box_half_size = Result(float) def run(self, rinput): _logger.info('starting processing for slit detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) _logger.debug('finding pinholes') try: filtername = hdr['FILTER'] readmode = hdr['READMODE'] rotang = hdr['ROTANG'] detpa = hdr['DETPA'] dtupa = hdr['DTUPA'] dtub, dtur = datamodel.get_dtur_from_header(hdr) except KeyError as error: _logger.error(error) raise numina.exceptions.RecipeError(error) if rinput.shift_coordinates: xdtur, ydtur, zdtur = dtur xfac = xdtur / EMIR_PIXSCALE yfac = -ydtur / EMIR_PIXSCALE vec = numpy.array([yfac, xfac]) _logger.info('shift is %s', vec) ncenters = rinput.pinhole_nominal_positions + vec else: _logger.info('using pinhole coordinates as they are') ncenters = rinput.pinhole_nominal_positions _logger.info('pinhole characterization') positions = pinhole_char(hdulist[0].data, ncenters, box=rinput.box_half_size, recenter_pinhole=rinput.recenter, maxdist=rinput.max_recenter_radius) _logger.info('alternate pinhole characterization') positions_alt = pinhole_char2( hdulist[0].data, ncenters, recenter_pinhole=rinput.recenter, recenter_half_box=rinput.box_half_size, recenter_maxdist=rinput.max_recenter_radius) _logger.debug('finding slits') # First, prefilter with median median_filter_size = rinput.median_filter_size canny_sigma = rinput.canny_sigma obj_min_size = rinput.obj_min_size obj_max_size = rinput.obj_max_size data1 = hdulist[0].data _logger.debug('Median filter with box %d', median_filter_size) data2 = median_filter(data1, size=median_filter_size) # Grey level image img_grey = normalize(data2) # Find edges with canny _logger.debug('Find edges with canny, sigma %d', canny_sigma) edges = canny(img_grey, sigma=canny_sigma) # Fill edges _logger.debug('Fill holes') fill_slits = ndimage.binary_fill_holes(edges) _logger.debug('Label objects') label_objects, nb_labels = ndimage.label(fill_slits) _logger.debug('%d objects found', nb_labels) # Filter on the area of the labeled region # Perhaps we could ignore this filtering and # do it later? _logger.debug('Filter objects by size') # Sizes of regions sizes = numpy.bincount(label_objects.ravel()) _logger.debug('Min size is %d', obj_min_size) _logger.debug('Max size is %d', obj_max_size) mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size) # Filter out regions nids, = numpy.where(mask_sizes) mm = numpy.in1d(label_objects, nids) mm.shape = label_objects.shape fill_slits_clean = numpy.where(mm, 1, 0) # and relabel _logger.debug('Label filtered objects') relabel_objects, nb_labels = ndimage.label(fill_slits_clean) _logger.debug('%d objects found after filtering', nb_labels) ids = list(six.moves.range(1, nb_labels + 1)) _logger.debug('Find regions and centers') regions = ndimage.find_objects(relabel_objects) centers = ndimage.center_of_mass(data2, labels=relabel_objects, index=ids) table = char_slit(data2, regions, slit_size_ratio=rinput.slit_size_ratio) result = self.create_result( frame=hdulist, positions=positions, positions_alt=positions_alt, slitstable=table, filter=filtername, DTU=dtub, readmode=readmode, ROTANG=rotang, DETPA=detpa, DTUPA=dtupa, param_recenter=rinput.recenter, param_max_recenter_radius=rinput.max_recenter_radius, param_box_half_size=rinput.box_half_size) return result
class BarDetectionRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() bars_nominal_positions = Requirement(prods.NominalPositions, 'Nominal positions of the bars') median_filter_size = Parameter(5, 'Size of the median box') average_box_row_size = Parameter( 7, 'Number of rows to average for fine centering (odd)') average_box_col_size = Parameter( 21, 'Number of columns to extract for fine centering (odd)') fit_peak_npoints = Parameter( 3, 'Number of points to use for fitting the peak (odd)') # Recipe Products frame = Result(prods.ProcessedImage) # derivative = Result(prods.ProcessedImage) slits = Result(tarray.ArrayType) positions3 = Result(tarray.ArrayType) positions5 = Result(tarray.ArrayType) positions7 = Result(tarray.ArrayType) positions9 = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) ROTANG = Result(float) TSUTC1 = Result(float) csupos = Result(tarray.ArrayType) csusens = Result(tarray.ArrayType) def run(self, rinput): self.logger.info('starting processing for bars detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) self.save_intermediate_img(hdulist, 'reduced_image.fits') try: rotang = hdr['ROTANG'] tsutc1 = hdr['TSUTC1'] dtub, dtur = datamodel.get_dtur_from_header(hdr) csupos = datamodel.get_csup_from_header(hdr) if len(csupos) != 2 * EMIR_NBARS: raise RecipeError('Number of CSUPOS != 2 * NBARS') csusens = datamodel.get_cs_from_header(hdr) except KeyError as error: self.logger.error(error) raise RecipeError(error) self.logger.debug('start finding bars') allpos, slits = find_bars( hdulist, rinput.bars_nominal_positions, csupos, dtur, average_box_row_size=rinput.average_box_row_size, average_box_col_size=rinput.average_box_col_size, fit_peak_npoints=rinput.fit_peak_npoints, median_filter_size=rinput.median_filter_size, logger=self.logger) self.logger.debug('end finding bars') if self.intermediate_results: with open('ds9.reg', 'w') as ds9reg: slits_to_ds9_reg(ds9reg, slits) result = self.create_result( frame=hdulist, slits=slits, positions9=allpos[9], positions7=allpos[7], positions5=allpos[5], positions3=allpos[3], DTU=dtub, ROTANG=rotang, TSUTC1=tsutc1, csupos=csupos, csusens=csusens, ) return result
class GenerateRectwvCoeff(EmirRecipe): """Process images in Stare spectra. This recipe generates a rectified and wavelength calibrated image after applying a model (master_rectwv). This calibration can be refined (using refine_wavecalib_mode != 0). """ logger = logging.getLogger(__name__) obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterSpectralFlatFieldRequirement() master_rectwv = reqs.MasterRectWaveRequirement() refine_wavecalib_mode = Parameter( 0, description='Apply wavelength calibration refinement', optional=True, choices=[0, 1, 2, 11, 12]) minimum_slitlet_width_mm = Parameter( float(EMIR_MINIMUM_SLITLET_WIDTH_MM), description='Minimum width (mm) for a valid slitlet', optional=True) maximum_slitlet_width_mm = Parameter( float(EMIR_MAXIMUM_SLITLET_WIDTH_MM), description='Maximum width (mm) for a valid slitlet', optional=True) global_integer_offsets_mode = Parameter( 'fixed', description='Global integer offsets computation', choices=['auto', 'fixed']) global_integer_offset_x_pix = Parameter( 0, description='Global integer offset (pixels) in wavelength direction', optional=True) global_integer_offset_y_pix = Parameter( 0, description='Global integer offset (pixels) in spatial direction', optional=True) reduced_mos = Result(prods.ProcessedMOS) rectwv_coeff = Result(RectWaveCoeff) def run(self, rinput): self.logger.info('starting rect.+wavecal. reduction of stare spectra') self.logger.info(rinput.master_rectwv) self.logger.info( 'Wavelength calibration refinement mode....: {}'.format( rinput.refine_wavecalib_mode)) self.logger.info( 'Minimum slitlet width (mm)................: {}'.format( rinput.minimum_slitlet_width_mm)) self.logger.info( 'Maximum slitlet width (mm)................: {}'.format( rinput.maximum_slitlet_width_mm)) self.logger.info( 'Global integer offsets mode...............: {}'.format( rinput.global_integer_offsets_mode)) self.logger.info( 'Global integer offset X direction (pixels): {}'.format( rinput.global_integer_offset_x_pix)) self.logger.info( 'Global integer offset Y direction (pixels): {}'.format( rinput.global_integer_offset_y_pix)) # build object to proceed with bpm, bias, dark and flat flow = self.init_filters(rinput) # apply bpm, bias, dark and flat reduced_image = basic_processing_with_combination(rinput, flow, method=sigmaclip) # update header with additional info hdr = reduced_image[0].header self.set_base_headers(hdr) # save intermediate image in work directory self.save_intermediate_img(reduced_image, 'reduced_image.fits') # RectWaveCoeff object with rectification and wavelength # calibration coefficients for the particular CSU configuration rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image, rinput.master_rectwv) # wavelength calibration refinement # 0 -> no refinement # 1 -> apply global offset to all the slitlets (using ARC lines) # 2 -> apply individual offset to each slitlet (using ARC lines) # 11 -> apply global offset to all the slitlets (using OH lines) # 12 -> apply individual offset to each slitlet (using OH lines) if rinput.refine_wavecalib_mode != 0: main_header = reduced_image[0].header # determine useful slitlets csu_config = CsuConfiguration.define_from_header(main_header) # segregate slitlets list_useful_slitlets = csu_config.widths_in_range_mm( minwidth=rinput.minimum_slitlet_width_mm, maxwidth=rinput.maximum_slitlet_width_mm) # remove missing slitlets if len(rectwv_coeff.missing_slitlets) > 0: for iremove in rectwv_coeff.missing_slitlets: if iremove in list_useful_slitlets: list_useful_slitlets.remove(iremove) list_not_useful_slitlets = [ i for i in list(range(1, EMIR_NBARS + 1)) if i not in list_useful_slitlets ] self.logger.info( 'list of useful slitlets: {}'.format(list_useful_slitlets)) self.logger.info('list of unusable slitlets: {}'.format( list_not_useful_slitlets)) # retrieve arc/OH lines catlines_all_wave, catlines_all_flux = retrieve_catlines( rinput.refine_wavecalib_mode, main_header['grism']) # global integer offsets if rinput.global_integer_offsets_mode == 'auto': if (rinput.global_integer_offset_x_pix != 0) or \ (rinput.global_integer_offset_y_pix != 0): raise ValueError('Global integer offsets must be zero when' ' mode=auto') # ToDo: include additional airglow emission lines self.logger.info('computing synthetic image') # generate synthetic image synthetic_raw_data = synthetic_lines_rawdata( catlines_all_wave, catlines_all_flux, list_useful_slitlets, rectwv_coeff) synthetic_raw_header = main_header.copy() synthetic_raw_header['DATE-OBS'] = \ datetime.now().strftime('%Y-%m-%dT%H:%M:%S') chistory = 'Synthetic image' synthetic_raw_header.add_history(chistory) hdu = fits.PrimaryHDU(synthetic_raw_data.astype('float32'), header=synthetic_raw_header) synthetic_raw_image = fits.HDUList([hdu]) if self.intermediate_results: self.save_intermediate_img(synthetic_raw_image, 'synthetic_raw_image.fits') # cross-correlation to determine global integer offsets # (rescaling data arrays to [0, 1] before using skimage # function) data1_rs, coef1_rs = rescale_array_to_z1z2( reduced_image[0].data, (0, 1)) data2_rs, coef2_rs = rescale_array_to_z1z2( synthetic_raw_data, (0, 1)) shifts, error, diffphase = register_translation( data1_rs, data2_rs, 100) self.logger.info( 'global_float_offset_x_pix..: {}'.format(-shifts[1])) self.logger.info( 'global_float_offset_y_pix..: {}'.format(-shifts[0])) rectwv_coeff.global_integer_offset_x_pix = \ -int(round(shifts[1])) rectwv_coeff.global_integer_offset_y_pix = \ -int(round(shifts[0])) self.logger.info('global_integer_offset_x_pix: {}'.format( rectwv_coeff.global_integer_offset_x_pix)) self.logger.info('global_integer_offset_y_pix: {}'.format( rectwv_coeff.global_integer_offset_y_pix)) if self.intermediate_results: data_product = np.fft.fft2(data1_rs) * \ np.fft.fft2(data2_rs).conj() cc_image = np.fft.fftshift(np.fft.ifft2(data_product)) power = np.log10(cc_image.real) hdu_power = fits.PrimaryHDU(power) hdul_power = fits.HDUList([hdu_power]) hdul_power.writeto('power.fits', overwrite=True) else: rectwv_coeff.global_integer_offset_x_pix = \ rinput.global_integer_offset_x_pix rectwv_coeff.global_integer_offset_y_pix = \ rinput.global_integer_offset_y_pix # apply initial rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) self.logger.info( 'Refining wavelength calibration (mode={})'.format( rinput.refine_wavecalib_mode)) # refine RectWaveCoeff object rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff( reduced_mos, rectwv_coeff, catlines_all_wave, catlines_all_flux, rinput.refine_wavecalib_mode, list_useful_slitlets, save_intermediate_results=self.intermediate_results) self.save_intermediate_img(expected_catalog_lines, 'expected_catalog_lines.fits') # apply rectification and wavelength calibration reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff) # ds9 region files (to be saved in the work directory) if self.intermediate_results: save_four_ds9(rectwv_coeff) save_spectral_lines_ds9(rectwv_coeff) # compute median spectra employing the useful region of the # rectified image if self.intermediate_results: for imode, outfile in enumerate([ 'median_spectra_full', 'median_spectra_slitlets', 'median_spectrum_slitlets' ]): median_image = median_slitlets_rectified(reduced_mos, mode=imode) self.save_intermediate_img(median_image, outfile + '.fits') # save results in results directory self.logger.info('end rect.+wavecal. reduction of stare spectra') result = self.create_result(reduced_mos=reduced_mos, rectwv_coeff=rectwv_coeff) return result def set_base_headers(self, hdr): newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr) # Update EXP to 0 newhdr['EXP'] = 0 return newhdr
class MaskImagingRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() bars_nominal_positions = Requirement(prods.CoordinateList2DType, 'Nominal positions of the bars') median_filter_size = Parameter(5, 'Size of the median box') average_box_row_size = Parameter( 7, 'Number of rows to average for fine centering (odd)') average_box_col_size = Parameter( 21, 'Number of columns to extract for fine centering (odd)') fit_peak_npoints = Parameter( 3, 'Number of points to use for fitting the peak (odd)') # Recipe Products frame = Result(prods.ProcessedImage) # derivative = Result(prods.ProcessedImage) slits = Result(tarray.ArrayType) positions3 = Result(tarray.ArrayType) positions5 = Result(tarray.ArrayType) positions7 = Result(tarray.ArrayType) positions9 = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) ROTANG = Result(float) TSUTC1 = Result(float) csupos = Result(tarray.ArrayType) csusens = Result(tarray.ArrayType) def run(self, rinput): self.logger.info('starting processing for bars detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) try: rotang = hdr['ROTANG'] tsutc1 = hdr['TSUTC1'] dtub, dtur = datamodel.get_dtur_from_header(hdr) csupos = datamodel.get_csup_from_header(hdr) csusens = datamodel.get_cs_from_header(hdr) except KeyError as error: self.logger.error(error) raise numina.exceptions.RecipeError(error) self.logger.debug('finding bars') # Processed array arr = hdulist[0].data # Median filter of processed array (two times) mfilter_size = rinput.median_filter_size self.logger.debug('median filtering X, %d columns', mfilter_size) arr_median = median_filter(arr, size=(1, mfilter_size)) self.logger.debug('median filtering X, %d rows', mfilter_size) arr_median = median_filter(arr_median, size=(mfilter_size, 1)) # Median filter of processed array (two times) in the other direction # for Y coordinates self.logger.debug('median filtering Y, %d rows', mfilter_size) arr_median_alt = median_filter(arr, size=(mfilter_size, 1)) self.logger.debug('median filtering Y, %d columns', mfilter_size) arr_median_alt = median_filter(arr_median_alt, size=(1, mfilter_size)) xfac = dtur[0] / EMIR_PIXSCALE yfac = -dtur[1] / EMIR_PIXSCALE vec = [yfac, xfac] self.logger.debug('DTU shift is %s', vec) # and the table of approx positions of the slits barstab = rinput.bars_nominal_positions # Currently, we only use fields 0 and 2 # of the nominal positions file # Number or rows used # These other parameters cab be tuned also bstart = 1 bend = 2047 self.logger.debug('ignoring columns outside %d - %d', bstart, bend - 1) # extract a region to average wy = (rinput.average_box_row_size // 2) wx = (rinput.average_box_col_size // 2) self.logger.debug('extraction window is %d rows, %d cols', 2 * wy + 1, 2 * wx + 1) # Fit the peak with these points wfit = 2 * (rinput.fit_peak_npoints // 2) + 1 self.logger.debug('fit with %d points', wfit) # Minimum threshold threshold = 5 * EMIR_RON # Savitsky and Golay (1964) filter to compute the X derivative # scipy >= xx has a savgol_filter function # for compatibility we do it manually allpos = {} ypos3_kernel = None slits = numpy.zeros((EMIR_NBARS, 8), dtype='float') self.logger.info('start finding bars') for ks in [3, 5, 7, 9]: self.logger.debug('kernel size is %d', ks) # S and G kernel for derivative kw = ks * (ks * ks - 1) / 12.0 coeffs_are = -numpy.arange((1 - ks) // 2, (ks - 1) // 2 + 1) / kw if ks == 3: ypos3_kernel = coeffs_are self.logger.debug('kernel weights are %s', coeffs_are) self.logger.debug('derive image in X direction') arr_deriv = convolve1d(arr_median, coeffs_are, axis=-1) # Axis 0 is # self.logger.debug('derive image in Y direction (with kernel=3)') arr_deriv_alt = convolve1d(arr_median_alt, ypos3_kernel, axis=0) positions = [] for coords in barstab: lbarid = int(coords[0]) rbarid = lbarid + EMIR_NBARS ref_y_coor = coords[1] + vec[1] poly_coeffs = coords[2:] prow = coor_to_pix_1d(ref_y_coor) - 1 fits_row = prow + 1 # FITS pixel index # A function that returns the center of the bar # given its X position def center_of_bar(x): # Pixel values are 0-based return polyval(x + 1 - vec[0], poly_coeffs) + vec[1] - 1 self.logger.debug('looking for bars with ids %d - %d', lbarid, rbarid) self.logger.debug('reference y position is Y %7.2f', ref_y_coor) # if ref_y_coor is outlimits, skip this bar # ref_y_coor is in FITS format if (ref_y_coor >= 2047) or (ref_y_coor <= 1): self.logger.debug( 'reference y position is outlimits, skipping') positions.append([lbarid, fits_row, fits_row, 1, 0, 3]) positions.append([rbarid, fits_row, fits_row, 1, 0, 3]) continue # Left bar self.logger.debug('measure left border (%d)', lbarid) centery, xpos, fwhm, st = char_bar_peak_l(arr_deriv, prow, bstart, bend, threshold, center_of_bar, wx=wx, wy=wy, wfit=wfit) xpos1 = xpos positions.append( [lbarid, centery + 1, fits_row, xpos + 1, fwhm, st]) # Right bar self.logger.debug('measure rigth border (%d)', rbarid) centery, xpos, fwhm, st = char_bar_peak_r(arr_deriv, prow, bstart, bend, threshold, center_of_bar, wx=wx, wy=wy, wfit=wfit) positions.append( [rbarid, centery + 1, fits_row, xpos + 1, fwhm, st]) xpos2 = xpos # if st == 0: self.logger.debug('measure top-bottom borders') try: y1, y2, statusy = char_bar_height(arr_deriv_alt, xpos1, xpos2, centery, threshold, wh=35, wfit=wfit) except Exception as error: self.logger.warning('Error computing height: %s', error) statusy = 44 if statusy in [0, 40]: # Main border is detected positions[-1][1] = y2 + 1 positions[-2][1] = y2 + 1 else: # Update status positions[-1][-1] = 4 positions[-2][-1] = 4 else: self.logger.debug('slit is not complete') y1, y2 = 0, 0 # Update positions self.logger.debug( 'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d', *positions[-2]) self.logger.debug( 'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d', *positions[-1]) if ks == 5: slits[lbarid - 1] = [xpos1, y2, xpos2, y2, xpos2, y1, xpos1, y1] # FITS coordinates slits[lbarid - 1] += 1.0 self.logger.debug('inserting bars %d-%d into "slits"', lbarid, rbarid) allpos[ks] = numpy.asarray( positions, dtype='float') # GCS doesn't like lists of lists self.logger.debug('end finding bars') result = self.create_result( frame=hdulist, slits=slits, positions9=allpos[9], positions7=allpos[7], positions5=allpos[5], positions3=allpos[3], DTU=dtub, ROTANG=rotang, TSUTC1=tsutc1, csupos=csupos, csusens=csusens, ) return result
class TestSlitMaskDetectionRecipe(EmirRecipe): # Recipe Requirements obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() median_filter_size = Parameter(5, 'Size of the median box') canny_sigma = Parameter(3.0, 'Sigma for the Canny algorithm') canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm') canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm') obj_min_size = Parameter(200, 'Minimum size of the slit') obj_max_size = Parameter(3000, 'Maximum size of the slit') slit_size_ratio = Parameter(4.0, 'Minimum ratio between height and width for slits') # Recipe Results frame = Result(prods.DataFrameType) slitstable = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) ROTANG = Result(float) DETPA = Result(float) DTUPA = Result(float) def run(self, rinput): self.logger.info('starting slit processing') self.logger.info('basic image reduction') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) try: rotang = hdr['ROTANG'] detpa = hdr['DETPA'] dtupa = hdr['DTUPA'] dtub, dtur = datamodel.get_dtur_from_header(hdr) except KeyError as error: self.logger.error(error) raise RecipeError(error) self.logger.debug('finding slits') # First, prefilter with median median_filter_size = rinput.median_filter_size canny_sigma = rinput.canny_sigma obj_min_size = rinput.obj_min_size obj_max_size = rinput.obj_max_size data1 = hdulist[0].data self.logger.debug('Median filter with box %d', median_filter_size) data2 = median_filter(data1, size=median_filter_size) # Grey level image img_grey = normalize_raw(data2) # Find edges with Canny self.logger.debug('Find edges with Canny, sigma %f', canny_sigma) # These thresholds corespond roughly with # value x (2**16 - 1) high_threshold = rinput.canny_high_threshold low_threshold = rinput.canny_low_threshold self.logger.debug('Find edges, Canny high threshold %f', high_threshold) self.logger.debug('Find edges, Canny low threshold %f', low_threshold) edges = canny(img_grey, sigma=canny_sigma, high_threshold=high_threshold, low_threshold=low_threshold) # Fill edges self.logger.debug('Fill holes') fill_slits = ndimage.binary_fill_holes(edges) self.logger.debug('Label objects') label_objects, nb_labels = ndimage.label(fill_slits) self.logger.debug('%d objects found', nb_labels) # Filter on the area of the labeled region # Perhaps we could ignore this filtering and # do it later? self.logger.debug('Filter objects by size') # Sizes of regions sizes = numpy.bincount(label_objects.ravel()) self.logger.debug('Min size is %d', obj_min_size) self.logger.debug('Max size is %d', obj_max_size) mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size) # Filter out regions nids, = numpy.where(mask_sizes) mm = numpy.in1d(label_objects, nids) mm.shape = label_objects.shape fill_slits_clean = numpy.where(mm, 1, 0) #plt.imshow(fill_slits_clean) # and relabel self.logger.debug('Label filtered objects') relabel_objects, nb_labels = ndimage.label(fill_slits_clean) self.logger.debug('%d objects found after filtering', nb_labels) ids = list(six.moves.range(1, nb_labels + 1)) self.logger.debug('Find regions and centers') regions = ndimage.find_objects(relabel_objects) centers = ndimage.center_of_mass(data2, labels=relabel_objects, index=ids ) table = char_slit(data2, regions, slit_size_ratio=rinput.slit_size_ratio ) result = self.create_result(frame=hdulist, slitstable=table, DTU=dtub, ROTANG=rotang, DETPA=detpa, DTUPA=dtupa ) return result
class TestPointSourceRecipe(EmirRecipe): # Recipe Requirements # obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() master_flat = reqs.MasterIntensityFlatFieldRequirement() master_sky = reqs.MasterSkyRequirement() shift_coordinates = Parameter( True, 'Use header information to' ' shift the pinhole positions from (0,0) ' 'to X_DTU, Y_DTU') box_half_size = Parameter(4, 'Half of the computation box size in pixels') recenter = Parameter(True, 'Recenter the pinhole coordinates') max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering') # Recipe Results frame = Result(prods.ProcessedImage) positions = Result(tarray.ArrayType) positions_alt = Result(tarray.ArrayType) DTU = Result(tarray.ArrayType) filter = Result(str) readmode = Result(str) ROTANG = Result(float) DETPA = Result(float) DTUPA = Result(float) param_recenter = Result(bool) param_max_recenter_radius = Result(float) param_box_half_size = Result(float) def run(self, rinput): self.logger.info('starting processing for object detection') flow = self.init_filters(rinput) hdulist = basic_processing_with_combination(rinput, flow=flow) hdr = hdulist[0].header self.set_base_headers(hdr) self.logger.debug('finding point sources') try: filtername = hdr['FILTER'] readmode = hdr['READMODE'] rotang = hdr['ROTANG'] detpa = hdr['DETPA'] dtupa = hdr['DTUPA'] dtub, dtur = datamodel.get_dtur_from_header(hdr) except KeyError as error: self.logger.error(error) raise RecipeError(error) data = hdulist[0].data # Copy needed in numpy 1.7 # This seems already bitswapped?? # FIXME: check this works offline/online # ndata = data.byteswap().newbyteorder() # data = data.byteswap(inplace=True).newbyteorder() snr_detect = 5.0 fwhm = 4.0 npixels = 15 box_shape = [64, 64] self.logger.info('point source detection2') self.logger.info('using internal mask to remove corners') # Corners mask = numpy.zeros_like(data, dtype='int32') mask[2000:, 0:80] = 1 mask[2028:, 2000:] = 1 mask[:50, 1950:] = 1 mask[:100, :50] = 1 # Remove corner regions self.logger.info('compute background map, %s', box_shape) bkg = sep.Background(data) self.logger.info('reference fwhm is %5.1f pixels', fwhm) self.logger.info('detect threshold, %3.1f over background', snr_detect) self.logger.info('convolve with gaussian kernel, FWHM %3.1f pixels', fwhm) sigma = fwhm * gaussian_fwhm_to_sigma # kernel = Gaussian2DKernel(sigma) kernel.normalize() thresh = snr_detect * bkg.globalrms data_s = data - bkg.back() objects, segmap = sep.extract(data - bkg.back(), thresh, minarea=npixels, filter_kernel=kernel.array, segmentation_map=True, mask=mask) fits.writeto('segmap.fits', segmap) self.logger.info('detected %d objects', len(objects)) # Hardcoded values rs2 = 15.0 fit_rad = 10.0 flux_min = 1000.0 flux_max = 30000.0 self.logger.debug('Flux limit is %6.1f %6.1f', flux_min, flux_max) # FIXME: this should be a view, not a copy xall = objects['x'] yall = objects['y'] mm = numpy.array([xall, yall]).T self.logger.info('computing FWHM') # Find objects with pairs inside fit_rad kdtree = KDTree(mm) nearobjs = (kdtree.query_ball_tree(kdtree, r=fit_rad)) positions = [] for idx, obj in enumerate(objects): x0 = obj['x'] y0 = obj['y'] sl = image_box2d(x0, y0, data.shape, (fit_rad, fit_rad)) # sl_sky = image_box2d(x0, y0, data.shape, (rs2, rs2)) part_s = data_s[sl] # Logical coordinates xx0 = x0 - sl[1].start yy0 = y0 - sl[0].start _, fwhm_x, fwhm_y = compute_fwhm_2d_simple(part_s, xx0, yy0) if min(fwhm_x, fwhm_x) < 3: continue if flux_min > obj['peak'] or flux_max < obj['peak']: continue # nobjs is the number of object inside fit_rad nobjs = len(nearobjs[idx]) flag = 0 if nobjs == 1 else 1 positions.append([idx, x0, y0, obj['peak'], fwhm_x, fwhm_y, flag]) self.logger.info('saving photometry') positions = numpy.array(positions) positions_alt = positions self.logger.info('end processing for object detection') result = self.create_result( frame=hdulist, positions=positions_alt, positions_alt=positions_alt, filter=filtername, DTU=dtub, readmode=readmode, ROTANG=rotang, DETPA=detpa, DTUPA=dtupa, param_recenter=rinput.recenter, param_max_recenter_radius=rinput.max_recenter_radius, param_box_half_size=rinput.box_half_size) return result
class MultiTwilightFlatRecipe(EmirRecipe): """Create a list of twilight flats""" obresult = reqs.ObservationResultRequirement() master_bpm = reqs.MasterBadPixelMaskRequirement() master_bias = reqs.MasterBiasRequirement() master_dark = reqs.MasterDarkRequirement() twflatframes = Result(dt.ListOfType(prods.MasterIntensityFlat)) def run(self, rinput): results = [] self.logger.info('starting multiflat flat reduction') # Uncomment this line # to revert to non-ramp # flow = self.init_filters(rinput) saturation = 45000.0 iinfo = gather_info_frames(rinput.obresult.frames) image_groups = {} self.logger.info('group images by filter') for idx, info in enumerate(iinfo): filt = info['filter'] if filt not in image_groups: self.logger.debug('new filter %s', filt) image_groups[filt] = [] img = rinput.obresult.frames[idx] self.logger.debug('image %s in group %s', img, filt) image_groups[filt].append(img) for filt, frames in image_groups.items(): self.logger.info('processing filter %s', filt) # Uncomment this line and comment the following # to revert to non-ramp # res = self.run_per_filter(frames, flow) try: res = self.run_per_filter_ramp(frames, saturation=saturation) results.append(res) except ValueError: self.logger.info('filter %s cannot be processed', filt) self.logger.info('end multiflat flat reduction') result = self.create_result(twflatframes=results) return result def run_per_filter(self, frames, flow): errors = True self.logger.debug('using errors: %s', errors) hdulist = basic_processing_with_combination_frames(frames, flow, method=median, errors=errors) hdr = hdulist[0].header self.set_base_headers(hdr) mm = hdulist[0].data.mean() self.logger.info('mean value of flat is: %f', mm) hdr['CCDMEAN'] = mm self.logger.debug('normalize image') hdulist[0].data /= mm if errors: self.logger.debug('normalize VAR extension') hdulist['variance'].data /= (mm * mm) return hdulist def run_per_filter_ramp(self, frames, saturation, errors=False): imgs = [frame.open() for frame in frames] return self.run_img_per_filter_ramp(imgs, saturation, errors) def run_img_per_filter_ramp(self, imgs, saturation, errors=False): nimages = len(imgs) if nimages == 0: raise ValueError('len(images) == 0') median_frames = numpy.empty((nimages, )) exptime_frames = [] utc_frames = [] # generate 3D cube bshape = self.datamodel.shape flat_frames = numpy.empty((bshape[0], bshape[1], nimages)) for idx, image in enumerate(imgs): flat_frames[:, :, idx] = image['primary'].data exptime_frames.append(image[0].header['EXPTIME']) median_frames[idx] = numpy.median(image['primary'].data) utc_frames.append(image[0].header['UTC']) self.logger.debug('image %d exptime %f median %f UTC %s', idx, exptime_frames[idx], median_frames[idx], utc_frames[-1]) # filter saturated images good_images = median_frames < saturation ngood_images = good_images.sum() slope_scaled_var = None slope_scaled_num = None if ngood_images == 0: self.logger.warning('We have only %d good images', ngood_images) raise ValueError('No images under saturation') elif ngood_images < 2: self.logger.warning('We have only %d good images', ngood_images) # Reference image ref_image = imgs[0] slope_scaled = numpy.ones(bshape) * exptime_frames[0] if errors: slope_scaled_var = numpy.zeros_like(slope_scaled) slope_scaled_num = numpy.zeros_like( slope_scaled, dtype='int16') + ngood_images else: nsaturated = nimages - good_images.sum() if nsaturated > 0: self.logger.debug( 'we have %d images with median value over saturation (%f)', nsaturated, saturation) m = flat_frames[:, :, good_images] # Reshape array to obtain a 2D array m_r = m.reshape((bshape[0] * bshape[1], ngood_images)) self.logger.debug('fitting slopes with Theil-Sen') # self.logger.debug('fitting slopes with mean-squares') # ll = nppol.polyfit(median_frames[good_images], m_r.T, deg=1) ll = self.filter_nsigma(median_frames[good_images], m_r.T) slope = ll[1].reshape(bshape) base = ll[0].reshape(bshape) # First good frame index_of_first_good = numpy.nonzero(good_images)[0][0] slope_scaled = slope * exptime_frames[index_of_first_good] if errors: slope_scaled_var = numpy.zeros_like(slope_scaled) slope_scaled_num = numpy.zeros_like( slope_scaled, dtype='int16') + ngood_images cdata = [] for idx, img in enumerate(imgs): if good_images[idx]: cdata.append(img) result = self.compose_result(cdata, slope_scaled, errors, slope_scaled_var, slope_scaled_num) return result def filter_nsigma(self, median_val, image_val, nsigma=10.0, nloop=1): # Initial estimation ll = fit_theil_sen(median_val, image_val) ni = 0 self.logger.debug('initial estimation') while ni < nloop: # Prediction self.logger.debug('loop %d', ni + 1) base, slope = ll image_val_pred = base + median_val[:, numpy.newaxis] * slope image_diff = image_val - image_val_pred # Compute MAD mad = compute_mad(image_diff) sigma_robust = nsigma * 1.4826 * mad self.logger.debug('compute robust std deviation') self.logger.debug('min %7.1f max %7.1f mean %7.1f', sigma_robust.min(), sigma_robust.max(), sigma_robust.mean()) # Check values over sigma mask_over = numpy.abs(image_diff) >= sigma_robust[:, numpy.newaxis] self.logger.debug('values over sigma: %d', mask_over.sum()) # Insert expected values in image # instead of masking image_val[mask_over] = image_val_pred[mask_over] # self.logger.debug('Theil-Sen fit') ll = fit_theil_sen(median_val, image_val) ni += 1 return ll def compose_result(self, imgs, slope_scaled, errors=False, slope_scaled_var=None, slope_scaled_num=None): self.logger.debug('update result header') cnum = len(imgs) method_name = 'Theil-Sen' base_header = imgs[0][0].header cdata = imgs hdu = fits.PrimaryHDU(data=slope_scaled, header=base_header) self.set_base_headers(hdu.header) hdu.header['history'] = "Combined %d images using '%s'" % (cnum, method_name) hdu.header['history'] = 'Combination time {}'.format( datetime.datetime.utcnow().isoformat()) for img in cdata: hdu.header['history'] = "Image {}".format( self.datamodel.get_imgid(img)) prevnum = base_header.get('NUM-NCOM', 1) hdu.header['NUM-NCOM'] = prevnum * cnum hdu.header['UUID'] = str(uuid.uuid1()) # Headers of last image hdu.header['TSUTC2'] = cdata[-1][0].header['TSUTC2'] # TODO: use BPM to compute mean mm = hdu.data.mean() self.logger.info('mean value of flat is: %f', mm) hdu.header['CCDMEAN'] = mm self.logger.debug('normalize image') hdu.data /= mm if errors: varhdu = fits.ImageHDU(slope_scaled_var, name='VARIANCE') num = fits.ImageHDU(slope_scaled_num, name='MAP') self.logger.debug('normalize VAR extension') varhdu.data /= (mm * mm) result = fits.HDUList([hdu, varhdu, num]) else: result = fits.HDUList([hdu]) return result