class DTUFocusRecipe(EmirRecipe):
"""
Recipe to compute the DTU focus.
**Observing modes:**
* EMIR focus control
**Inputs:**
* A list of images
* A list of sky images
* Bias, dark, flat
* A model of the detector
* List of focii
**Outputs:**
* Best focus
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
objects = Parameter([], 'List of x-y pair of object coordinates'),
msm_pattern = Parameter([], 'List of x-y pair of slit coordinates'),
dtu_focus_range = Parameter(
'dtu_focus_range', [], 'Focus range of the DTU: begin, end and step')
focus = Product(DTUFocus)
def run(self, rinput):
return self.create_result(focus=DTUFocus())
class TelescopeRoughFocusRecipe(EmirRecipe):
"""Recipe to compute the telescope focus.
**Observing modes:**
* Telescope rough focus
* Emir focus control
**Inputs:**
* A list of images
* A list of sky images
* Bias, dark, flat
* A model of the detector
* List of focii
**Outputs:**
* Best focus
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
objects = Parameter([], 'List of x-y pair of object coordinates'),
focus_range = Parameter([], 'Focus range: begin, end and step')
focus = Product(TelescopeFocus)
def run(self, rinput):
return self.create_result(focus=TelescopeFocus())
class MosaicRecipe(EmirRecipe):
"""
The effect of recording a series of stare images, with the same
acquisition parameters, and taken by pointing to a number of
sky positions, with separations of the order of the EMIR FOV.
This command is designed to fully cover a given area on the
sky, but can also be used to point to a number of sky positions
on which acquisition is only made at the beginning. Supersky
frame(s) can be built from the image series.
**Observing modes:**
* Mosaic images
"""
obresult = ObservationResultRequirement()
# FIXME: this parameter is optional
sources = Parameter([], 'List of x, y coordinates to measure FWHM')
frame = Product(DataFrameType)
catalog = Product(SourcesCatalog)
def run(self, ri):
return self.create_result(frame=DataFrame(None),
catalog=SourcesCatalog())
class TelescopeFineFocusRecipe(EmirRecipe):
"""
Recipe to compute the telescope focus.
**Observing modes:**
* Telescope fine focus
**Inputs:**
* A list of images
* A list of sky images
* Bias, dark, flat
* A model of the detector
* List of focii
**Outputs:**
* Best focus
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
objects = Parameter([], 'List of x-y pair of object coordinates'),
focus = Result(prods.TelescopeFocus)
def run(self, rinput):
return self.create_result(focus=prods.TelescopeFocus())
class Flat(BaseRecipe):
obresult = Requirement(ObservationResultType, "Observation Result")
master_bias = Requirement(MasterBias, "Master Bias")
polynomial_degree = Parameter(5,
'Polynomial degree of arc calibration',
as_list=True,
nelem='+',
validator=range_validator(minval=1))
@master_bias.validator
def custom_validator(self, value):
print('func', self, value)
return True
master_flat = Result(MasterFlat)
def run(self, rinput):
# Here the raw images are processed
# and a final image myframe is created
obresult = rinput.obresult
fr0 = obresult.frames[0].open()
data = numpy.ones_like(fr0[0].data)
hdu = fits.PrimaryHDU(data, header=fr0[0].header)
myframe = fits.HDUList([hdu])
#
result = self.create_result(master_flat=myframe)
return result
class ArcCalibrationRecipe(EmirRecipe):
obresult = ObservationResultRequirement()
master_bpm = MasterBadPixelMaskRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
lines_catalog = Requirement(LinesCatalog, "Catalog of lines")
polynomial_degree = Parameter(2,
'Polynomial degree of the arc calibration')
polynomial_coeffs = Product(ArrayType)
def run(self, rinput):
_logger.info('starting arc calibration')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
nslits = len(rinput.slits_catalog)
coeff_table = numpy.zeros((nslits, rinput.polynomial_degree + 1))
result = self.create_result(polynomial_coeffs=coeff_table)
return result
class OffsetSpectraRecipe(EmirRecipe):
"""
Observing mode:
Offset spectra beyond the slit
"""
obresult = ObservationResultRequirement()
master_bpm = MasterBadPixelMaskRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
master_flat = MasterIntensityFlatFieldRequirement()
master_spectral_ff = Requirement(prods.MasterSpectralFlat,
'Master spectral flatfield')
st_calibration = Requirement(prods.SlitTransmissionCalibration,
'Slit tranmision calibration')
w_calibration = Requirement(prods.WavelengthCalibration,
'Wavelength calibration')
lines = Parameter('lines', None,
'List of x-lambda pairs of line coordinates')
spectra = Product(prods.Spectra)
catalog = Product(prods.LinesCatalog)
def run(self, rinput):
return self.create_result(spectra=prods.Spectra(),
catalog=prods.LinesCatalog())
class NBImageRecipeInput(RecipeInput):
obresult = ObservationResultRequirement()
master_bpm = MasterBadPixelMaskRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
master_flat = MasterIntensityFlatFieldRequirement()
extinction = Parameter(0.0, 'Mean atmospheric extinction')
sources = Parameter(
[], 'List of x, y coordinates to measure FWHM', optional=True)
offsets = Offsets_Requirement()
sky_images = Parameter(5, 'Images used to estimate the background'
' before and after current image')
sky_images_sep_time = SkyImageSepTime_Requirement()
check_photometry_levels = Parameter(
[0.5, 0.8], 'Levels to check the flux of the objects')
check_photometry_actions = Parameter(
['warn', 'warn', 'default'], 'Actions to take on images')
class DTU_XY_CalibrationRecipe(EmirRecipe):
"""
**Observing modes:**
* DTU X_Y calibration
"""
obresult = ObservationResultRequirement()
slit_pattern = Parameter([], 'Slit pattern'),
dtu_range = Parameter([], 'DTU range: begin, end and step')
calibration = Product(DTU_XY_Calibration)
def run(self, rinput):
return self.create_result(calibration=DTU_XY_Calibration())
class CSUSpectraExtractionRecipe(EmirRecipe):
"""Extract spectra in image taken with the CSU configured"""
# Recipe Requirements
obresult = ObservationResultRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
master_flat = MasterIntensityFlatFieldRequirement()
master_sky = MasterSkyRequirement()
nrows_side = Parameter(5, 'Number of rows to extract around the center')
slits_positions = Requirement(ArrayType,
'Positions and widths of the slits')
# Recipe products
frame = Product(DataFrameType)
rss = Product(DataFrameType)
def run(self, rinput):
_logger.info('starting extraction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
data1 = hdulist[0].data
_logger.info('Create output images')
rssdata = numpy.zeros(
(rinput.slits_positions.shape[0], data1.shape[1]), dtype='float32')
nrows = rinput.nrows_side
# Loop over slits
for idx, slit_coords in enumerate(rinput.slits_positions):
x, y, ax, ay = slit_coords # Coords in FITS coordinates
ref_col = wc_to_pix(x - 1)
ref_row = wc_to_pix(y - 1)
_logger.info('Processing slit in column %i, row=%i', ref_col,
ref_row)
# Simple extraction
_logger.info('Extract %i rows around center', nrows)
region = data1[ref_row - nrows:ref_row + nrows + 1, :]
rssdata[idx, :] = region.mean(axis=0)
hdurss = fits.PrimaryHDU(rssdata)
result = self.create_result(frame=hdulist, rss=hdurss)
return result
class SpectroPhotometricCalibrationRecipe(EmirRecipe):
"""
**Observing modes:**
* Spectrophotometric calibration
"""
obresult = ObservationResultRequirement()
sphot = Parameter([], 'Information about standard stars')
calibration = Product(SpectroPhotometricCalibration)
def run(self, rinput):
return self.create_result(calibration=SpectroPhotometricCalibration())
class CSU2DetectorRecipe(EmirRecipe):
"""
**Observing modes:**
* CSU2Detector calibration
"""
obresult = ObservationResultRequirement()
dtu_range = Parameter([], 'DTU range: begin, end and step')
calibration = Product(DTU_XY_Calibration)
def run(self, rinput):
return self.create_result(calibration=DTU_XY_Calibration())
class MicroditheredImageRecipeInput(RecipeInput):
obresult = ObservationResultRequirement()
master_bpm = MasterBadPixelMaskRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
master_flat = MasterIntensityFlatFieldRequirement()
extinction = Parameter(0.0, 'Mean atmospheric extinction')
sources = Parameter([],
'List of x, y coordinates to measure FWHM',
optional=True)
offsets = Offsets_Requirement()
iterations = Parameter(4, 'Iterations of the recipe')
sky_images = Parameter(
5, 'Images used to estimate the background before '
'and after current image')
sky_images_sep_time = SkyImageSepTime_Requirement()
check_photometry_levels = Parameter(
[0.5, 0.8], 'Levels to check the flux of the objects')
check_photometry_actions = Parameter(['warn', 'warn', 'default'],
'Actions to take on images')
subpixelization = Parameter(4, 'Number of subdivisions in each pixel side')
window = Parameter([], 'Region of interesting data', optional=True)
class TestSlitMaskDetectionRecipe(EmirRecipe):
# Recipe Requirements
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the Canny algorithm')
canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm')
canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm')
obj_min_size = Parameter(200, 'Minimum size of the slit')
obj_max_size = Parameter(3000, 'Maximum size of the slit')
slit_size_ratio = Parameter(4.0, 'Minimum ratio between height and width for slits')
# Recipe Results
frame = Result(prods.DataFrameType)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
def run(self, rinput):
self.logger.info('starting slit processing')
self.logger.info('basic image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('finding slits')
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
obj_min_size = rinput.obj_min_size
obj_max_size = rinput.obj_max_size
data1 = hdulist[0].data
self.logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize_raw(data2)
# Find edges with Canny
self.logger.debug('Find edges with Canny, sigma %f', canny_sigma)
# These thresholds corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
self.logger.debug('Find edges, Canny high threshold %f', high_threshold)
self.logger.debug('Find edges, Canny low threshold %f', low_threshold)
edges = canny(img_grey, sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Fill edges
self.logger.debug('Fill holes')
fill_slits = ndimage.binary_fill_holes(edges)
self.logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
self.logger.debug('%d objects found', nb_labels)
# Filter on the area of the labeled region
# Perhaps we could ignore this filtering and
# do it later?
self.logger.debug('Filter objects by size')
# Sizes of regions
sizes = numpy.bincount(label_objects.ravel())
self.logger.debug('Min size is %d', obj_min_size)
self.logger.debug('Max size is %d', obj_max_size)
mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size)
# Filter out regions
nids, = numpy.where(mask_sizes)
mm = numpy.in1d(label_objects, nids)
mm.shape = label_objects.shape
fill_slits_clean = numpy.where(mm, 1, 0)
#plt.imshow(fill_slits_clean)
# and relabel
self.logger.debug('Label filtered objects')
relabel_objects, nb_labels = ndimage.label(fill_slits_clean)
self.logger.debug('%d objects found after filtering', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
self.logger.debug('Find regions and centers')
regions = ndimage.find_objects(relabel_objects)
centers = ndimage.center_of_mass(data2, labels=relabel_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=rinput.slit_size_ratio
)
result = self.create_result(frame=hdulist, slitstable=table,
DTU=dtub,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa
)
return result
class TestPinholeRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
pinhole_nominal_positions = Requirement(
prods.CoordinateList2DType, 'Nominal positions of the pinholes')
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
_logger.info('starting processing for slit detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.debug('finding pinholes')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
_logger.error(error)
raise numina.exceptions.RecipeError(error)
if rinput.shift_coordinates:
xdtur, ydtur, zdtur = dtur
xfac = xdtur / EMIR_PIXSCALE
yfac = -ydtur / EMIR_PIXSCALE
vec = numpy.array([yfac, xfac])
_logger.info('shift is %s', vec)
ncenters = rinput.pinhole_nominal_positions + vec
else:
_logger.info('using pinhole coordinates as they are')
ncenters = rinput.pinhole_nominal_positions
_logger.info('pinhole characterization')
positions = pinhole_char(hdulist[0].data,
ncenters,
box=rinput.box_half_size,
recenter_pinhole=rinput.recenter,
maxdist=rinput.max_recenter_radius)
_logger.info('alternate pinhole characterization')
positions_alt = pinhole_char2(
hdulist[0].data,
ncenters,
recenter_pinhole=rinput.recenter,
recenter_half_box=rinput.box_half_size,
recenter_maxdist=rinput.max_recenter_radius)
result = self.create_result(
frame=hdulist,
positions=positions,
positions_alt=positions_alt,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class GenerateRectwvCoeff(EmirRecipe):
"""Process images in Stare spectra.
This recipe generates a rectified and wavelength calibrated
image after applying a model (master_rectwv). This calibration
can be refined (using refine_wavecalib_mode != 0).
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
master_rectwv = reqs.MasterRectWaveRequirement()
refine_wavecalib_mode = Parameter(
0,
description='Apply wavelength calibration refinement',
choices=[0, 1, 2, 11, 12])
minimum_slitlet_width_mm = Parameter(
float(EMIR_MINIMUM_SLITLET_WIDTH_MM),
description='Minimum width (mm) for a valid slitlet',
)
maximum_slitlet_width_mm = Parameter(
float(EMIR_MAXIMUM_SLITLET_WIDTH_MM),
description='Maximum width (mm) for a valid slitlet',
)
global_integer_offset_x_pix = Parameter(
0,
description='Global offset (pixels) in wavelength direction (integer)',
)
global_integer_offset_y_pix = Parameter(
0,
description='Global offset (pixels) in spatial direction (integer)',
)
reduced_mos = Result(prods.ProcessedMOS)
rectwv_coeff = Result(RectWaveCoeff)
def run(self, rinput):
self.logger.info('starting rect.+wavecal. reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
self.logger.info('Wavelength calibration refinement mode: {}'.format(
rinput.refine_wavecalib_mode))
self.logger.info('Minimum slitlet width (mm)............: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info('Maximum slitlet width (mm)............: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info('Global offset X direction (pixels)....: {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info('Global offset Y direction (pixels)....: {}'.format(
rinput.global_integer_offset_y_pix))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=median)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image,
rinput.master_rectwv)
# set global offsets
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
# apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# wavelength calibration refinement
# 0 -> no refinement
# 1 -> apply global offset to all the slitlets (using ARC lines)
# 2 -> apply individual offset to each slitlet (using ARC lines)
# 11 -> apply global offset to all the slitlets (using OH lines)
# 12 -> apply individual offset to each slitlet (using OH lines)
if rinput.refine_wavecalib_mode != 0:
self.logger.info(
'Refining wavelength calibration (mode={})'.format(
rinput.refine_wavecalib_mode))
# refine RectWaveCoeff object
rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff(
reduced_mos,
rectwv_coeff,
rinput.refine_wavecalib_mode,
rinput.minimum_slitlet_width_mm,
rinput.maximum_slitlet_width_mm,
save_intermediate_results=self.intermediate_results)
self.save_intermediate_img(expected_catalog_lines,
'expected_catalog_lines.fits')
# re-apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of stare spectra')
result = self.create_result(reduced_mos=reduced_mos,
rectwv_coeff=rectwv_coeff)
return result
def set_base_headers(self, hdr):
newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class FullDitheredImagesRecipe(EmirRecipe):
"""Recipe for the reduction of imaging mode observations.
Recipe to reduce observations obtained in imaging mode, considering
different possibilities depending on the size of the offsets
between individual images.
In particular, the following observing modes are considered: stare imaging,
nodded beamswitched imaging, and dithered imaging.
A critical piece of information here is a table that clearly specifies
which images can be labeled as *science*, and which ones as *sky*.
Note that some images are used both as *science* and *sky*
(when the size of the targets is small compared to the offsets).
**Observing modes:**
* StareImage
* Nodded/Beam-switched images
* Dithered images
**Inputs:**
* Science frames + [Sky Frames]
* Observing mode name: **stare image**, **nodded beamswitched image**,
or **dithered imaging**
* A table relating each science image with its sky image(s) (TBD if
it's in the FITS header and/or in other format)
* Offsets between them (Offsets must be integer)
* Master Dark
* Bad pixel mask (BPM)
* Non-linearity correction polynomials
* Master flat (twilight/dome flats)
* Master background (thermal background, only in K band)
* Exposure Time (must be the same in all the frames)
* Airmass for each frame
* Detector model (gain, RN, lecture mode)
* Average extinction in the filter
* Astrometric calibration (TBD)
**Outputs:**
* Image with three extensions: final image scaled to the individual
exposure time, variance and exposure time map OR number of images
combined (TBD)
**Procedure:**
Images are corrected from dark, non-linearity and flat. Then, an iterative
process starts:
* Sky is computed from each frame, using the list of sky images of each
science frame. The objects are avoided using a mask (from the second
iteration on).
* The relative offsets are the nominal from the telescope. From the second
iteration on, we refine them using objects of appropriate brightness
(not too bright, not to faint).
* We combine the sky-subtracted images, output is: a new image, a variance
image and a exposure map/number of images used map.
* An object mask is generated.
* We recompute the sky map, using the object mask as an additional input.
From here we iterate (typically 4 times).
* Finally, the images are corrected from atmospheric extinction and flux
calibrated.
* A preliminary astrometric calibration can always be used (using
the central coordinates of the pointing and the plate scale
in the detector).
A better calibration might be computed using available stars (TBD).
"""
obresult = ObservationResultRequirement(
query_opts=ResultOf('result_image', node='children'))
master_bpm = reqs.MasterBadPixelMaskRequirement()
offsets = Requirement(prods.CoordinateList2DType,
'List of pairs of offsets',
optional=True)
refine_offsets = Parameter(False, 'Refine offsets by cross-correlation')
iterations = Parameter(2, 'Iterations of the recipe')
extinction = Parameter(0.0, 'Mean atmospheric extinction')
sky_images = Parameter(
5, 'Images used to estimate the '
'background before and after current image')
sky_images_sep_time = Parameter(
10, 'Maximum time interval between target and sky images [minutes]')
result_image = Result(prods.ProcessedImage)
result_sky = Result(prods.ProcessedImage, optional=True)
def run(self, rinput):
target_is_sky = True
obresult = rinput.obresult
sky_images = rinput.sky_images
sky_images_sep_time = rinput.sky_images_sep_time
baseshape = (2048, 2048)
user_offsets = rinput.offsets
extinction = rinput.extinction
images_info = self.initial_classification(obresult, target_is_sky)
# Resizing target frames
target_info = [iinfo for iinfo in images_info if iinfo.valid_target]
finalshape, offsetsp, refpix, offset_fc0 = self.compute_size(
target_info, baseshape, user_offsets)
self.resize(target_info, baseshape, offsetsp, finalshape)
result = self.process_basic(images_info,
target_is_sky=target_is_sky,
extinction=extinction)
if rinput.refine_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions_c = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions_c = self.compute_regions_from_objs(result[0].data,
finalshape,
box=40)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
images_info, regions_c, refine=True, tol=1)
#
# Combined offsets
# Offsets in numpy order, swaping
offset_xy0 = numpy.fliplr(offset_fc0)
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = numpy.fliplr(offsets_xy_t)
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape2, offsetsp2 = narray.combine_shape(
baseshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s",
offsetsp2)
self.logger.info('Shape of resized array (crosscorr) is %s',
finalshape2)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
self.resize(target_info, baseshape, offsetsp2, finalshape2)
except Exception as error:
self.logger.warning('Error during cross-correlation, %s',
error)
result = self.process_basic(images_info,
target_is_sky=target_is_sky,
extinction=extinction)
step = 1
while step <= rinput.iterations:
result = self.process_advanced(images_info,
result,
step,
target_is_sky,
maxsep=sky_images_sep_time,
nframes=sky_images,
extinction=extinction)
step += 1
return self.create_result(result_image=result)
def compute_offset_xy_crosscor_regions(self,
iinfo,
regions,
refine=False,
tol=0.5):
names = [frame.lastname for frame in iinfo]
print(names)
print(regions)
with nfcom.manage_fits(names) as imgs:
arrs = [img[0].data for img in imgs]
offsets_xy = offsets_from_crosscor_regions(arrs,
regions,
refine=refine,
order='xy',
tol=tol)
self.logger.debug("offsets_xy cross-corr %s", offsets_xy)
# Offsets in numpy order, swaping
return offsets_xy
def compute_size(self, images_info, baseshape, user_offsets=None):
# Reference pixel in the center of the frame
refpix = numpy.array([[baseshape[0] / 2.0, baseshape[1] / 2.0]])
target_info = [iinfo for iinfo in images_info if iinfo.valid_target]
if user_offsets is not None:
self.logger.info('Using offsets from parameters')
base_ref = numpy.asarray(user_offsets)
list_of_offsets = -(base_ref - base_ref[0])
else:
self.logger.debug('Computing offsets from WCS information')
with nfcom.manage_fits(img.origin
for img in target_info) as images:
list_of_offsets = offsets_from_wcs_imgs(images, refpix)
# FIXME: I am using offsets in row/columns
# the values are provided in XY so flip-lr
list_of_offsets = numpy.fliplr(list_of_offsets)
# Insert pixel offsets between frames
for iinfo, off in zip(target_info, list_of_offsets):
# Insert pixel offsets between frames
iinfo.pix_offset = off
self.logger.debug('Frame %s, offset=%s', iinfo.label, off)
self.logger.info('Computing relative offsets')
offsets = [iinfo.pix_offset for iinfo in target_info]
offsets = numpy.round(offsets).astype('int')
finalshape, offsetsp = narray.combine_shape(baseshape, offsets)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
return finalshape, offsetsp, refpix, list_of_offsets
def process_basic(self, images_info, target_is_sky=True, extinction=0.0):
step = 0
target_info = [iinfo for iinfo in images_info if iinfo.valid_target]
sky_info = [iinfo for iinfo in images_info if iinfo.valid_sky]
self.logger.info("Step %d, SF: compute superflat", step)
sf_arr = self.compute_superflat(images_info)
# Apply superflat
self.logger.info("Step %d, SF: apply superflat", step)
for iinfo in images_info:
self.correct_superflat(iinfo, sf_arr, step=step, save=True)
self.logger.info('Simple sky correction')
if target_is_sky:
# Each frame is the closest sky frame available
for iinfo in images_info:
self.compute_simple_sky_for_frame(iinfo, iinfo)
else:
# Not implemented
self.compute_simple_sky(target_info, sky_info)
# Combining the frames
self.logger.info("Step %d, Combining target frames", step)
result = self.combine_frames(target_info, extinction=extinction)
self.logger.info('Step %d, finished', step)
return result
def process_advanced(self,
images_info,
result,
step,
target_is_sky=True,
maxsep=5.0,
nframes=6,
extinction=0):
seeing_fwhm = None
baseshape = (2048, 2048)
target_info = [iinfo for iinfo in images_info if iinfo.valid_target]
sky_info = [iinfo for iinfo in images_info if iinfo.valid_sky]
self.logger.info('Step %d, generating segmentation image', step)
objmask, seeing_fwhm = self.create_mask(result, seeing_fwhm, step=step)
for frame in target_info:
frame.objmask = name_object_mask(frame.label, step)
self.logger.info('Step %d, create object mask %s', step,
frame.objmask)
frame.objmask_data = objmask[frame.valid_region]
fits.writeto(frame.objmask, frame.objmask_data, overwrite=True)
if not target_is_sky:
# Empty object mask for sky frames
bogus_objmask = numpy.zeros(baseshape, dtype='uint8')
for frame in sky_info:
frame.objmask_data = bogus_objmask
self.logger.info("Step %d, SF: compute superflat", step)
sf_arr = self.compute_superflat(sky_info, segmask=objmask, step=step)
# Apply superflat
self.logger.info("Step %d, SF: apply superflat", step)
for iinfo in images_info:
self.correct_superflat(iinfo, sf_arr, step=step, save=True)
self.logger.info('Step %d, advanced sky correction (SC)', step)
self.compute_advanced_sky(target_info,
objmask,
skyframes=sky_info,
target_is_sky=target_is_sky,
maxsep=maxsep,
nframes=nframes,
step=step)
# Combining the images
self.logger.info("Step %d, Combining the images", step)
# FIXME: only for science
result = self.combine_frames(target_info, extinction, step=step)
return result
def compute_simple_sky_for_frame(self, frame, skyframe, step=0, save=True):
self.logger.info('Correcting sky in frame %s', frame.lastname)
self.logger.info('with sky computed from frame %s', skyframe.lastname)
if hasattr(skyframe, 'median_sky'):
sky = skyframe.median_sky
else:
with fits.open(skyframe.lastname, mode='readonly') as hdulist:
data = hdulist['primary'].data
valid = data[frame.valid_region]
if skyframe.objmask_data is not None:
self.logger.debug('object mask defined')
msk = frame.objmask_data
sky = numpy.median(valid[msk == 0])
else:
self.logger.debug('object mask empty')
sky = numpy.median(valid)
self.logger.debug('median sky value is %f', sky)
skyframe.median_sky = sky
dst = name_skysub_proc(frame.label, step)
prev = frame.lastname
if save:
shutil.copyfile(prev, dst)
else:
os.rename(prev, dst)
frame.lastname = dst
with fits.open(frame.lastname, mode='update') as hdulist:
data = hdulist['primary'].data
valid = data[frame.valid_region]
valid -= sky
def compute_simple_sky(self, frame, skyframe, step=0, save=True):
raise NotImplementedError
def correct_superflat(self, frame, fitted, step=0, save=True):
frame.flat_corrected = name_skyflat_proc(frame.label, step)
if save:
shutil.copyfile(frame.resized_base, frame.flat_corrected)
else:
os.rename(frame.resized_base, frame.flat_corrected)
self.logger.info("Step %d, SF: apply superflat to frame %s", step,
frame.flat_corrected)
with fits.open(frame.flat_corrected, mode='update') as hdulist:
data = hdulist['primary'].data
datar = data[frame.valid_region]
data[frame.valid_region] = narray.correct_flatfield(datar, fitted)
frame.lastname = frame.flat_corrected
def initial_classification(self, obresult, target_is_sky=False):
"""Classify input frames, """
# lists of targets and sky frames
with obresult.frames[0].open() as baseimg:
# Initial checks
has_bpm_ext = 'BPM' in baseimg
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
images_info = []
for f in obresult.frames:
with f.open() as img:
# Getting some metadata from FITS header
hdr = img[0].header
iinfo = ImageInfo(f)
finfo = {}
iinfo.metadata = finfo
finfo['uuid'] = hdr['UUID']
finfo['exposure'] = hdr['EXPTIME']
# frame.baseshape = get_image_shape(hdr)
finfo['airmass'] = hdr['airmass']
finfo['mjd'] = hdr['tstamp']
iinfo.label = 'result_image_{}'.format(finfo['uuid'])
iinfo.mask = nfcom.Extension("BPM")
# Insert pixel offsets between frames
iinfo.objmask_data = None
iinfo.valid_target = False
iinfo.valid_sky = False
# FIXME: hardcode itype for the moment
iinfo.itype = 'TARGET'
if iinfo.itype == 'TARGET':
iinfo.valid_target = True
#targetframes.append(iinfo)
if target_is_sky:
iinfo.valid_sky = True
#skyframes.append(iinfo)
if iinfo.itype == 'SKY':
iinfo.valid_sky = True
#skyframes.append(iinfo)
images_info.append(iinfo)
return images_info
def compute_superflat(self, images_info, segmask=None, step=0):
self.logger.info("Step %d, SF: combining the frames without offsets",
step)
base_imgs = [img.resized_base for img in images_info]
with nfcom.manage_fits(base_imgs) as imgs:
data = []
masks = []
for img, img_info in zip(imgs, images_info):
self.logger.debug('Step %d, opening resized frame %s', step,
img_info.resized_base)
data.append(img['primary'].data[img_info.valid_region])
scales = [numpy.median(d) for d in data]
if segmask is not None:
masks = [segmask[frame.valid_region] for frame in images_info]
else:
for frame in images_info:
self.logger.debug('Step %d, opening resized mask %s', step,
frame.resized_mask)
hdulist = fits.open(frame.resized_mask,
memmap=True,
mode='readonly')
#filelist.append(hdulist)
masks.append(hdulist['primary'].data[frame.valid_region])
masks = None
self.logger.debug('Step %d, combining %d frames', step, len(data))
sf_data, _sf_var, sf_num = nacom.median(
data,
masks,
scales=scales,
dtype='float32',
#blank=1.0 / scales[0]
)
# Normalize, flat has mean = 1
sf_data[sf_data == 0] = 1e-5
sf_data /= sf_data.mean()
#sf_data[sf_data <= 0] = 1.0
# Auxiliary data
sfhdu = fits.PrimaryHDU(sf_data)
self.save_intermediate_img(sfhdu, name_skyflat('comb', step))
return sf_data
def run_single(self, rinput):
# FIXME: remove this, is deprecated
obresult = rinput.obresult
# just in case images are in result, instead of frames
if not obresult.frames:
frames = obresult.results
else:
frames = obresult.frames
img_info = []
data_hdul = []
for f in frames:
img = f.open()
data_hdul.append(img)
info = {}
info['tstamp'] = img[0].header['tstamp']
info['airmass'] = img[0].header['airmass']
img_info.append(info)
channels = FULL
use_errors = True
# Initial checks
baseimg = data_hdul[0]
has_num_ext = 'NUM' in baseimg
has_bpm_ext = 'BPM' in baseimg
baseshape = baseimg[0].shape
subpixshape = baseshape
base_header = baseimg[0].header
compute_sky = 'NUM-SK' not in base_header
compute_sky_advanced = False
self.logger.debug('base image is: %s',
self.datamodel.get_imgid(baseimg))
self.logger.debug('images have NUM extension: %s', has_num_ext)
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
self.logger.debug('compute sky is needed: %s', compute_sky)
if compute_sky:
self.logger.info('compute sky simple')
sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
self.save_intermediate_img(sky_result, 'sky_init.fits')
sky_result.writeto('sky_init.fits', overwrite=True)
sky_data = sky_result[0].data
self.logger.debug('sky image has shape %s', sky_data.shape)
self.logger.info('sky correction in individual images')
corrector = proc.SkyCorrector(
sky_data,
self.datamodel,
calibid=self.datamodel.get_imgid(sky_result))
# If we do not update keyword SKYADD
# there is no sky subtraction
for m in data_hdul:
m[0].header['SKYADD'] = True
# this is a little hackish
# sky corrected
data_hdul_s = [corrector(m) for m in data_hdul]
base_header = data_hdul_s[0][0].header
else:
sky_result = None
data_hdul_s = data_hdul
self.logger.info('Computing offsets from WCS information')
finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
data_hdul_s, baseshape, subpixshape)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
# Resizing target imgs
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if has_num_ext:
self.logger.debug('Using NUM extension')
masks = [
numpy.where(m['NUM'].data, 0, 1).astype('int16')
for m in data_hdul
]
elif has_bpm_ext:
self.logger.debug('Using BPM extension')
#
masks = [
numpy.where(m['BPM'].data, 1, 0).astype('int16')
for m in data_hdul
]
else:
self.logger.warning('BPM missing, use zeros instead')
false_mask = numpy.zeros(baseshape, dtype='int16')
masks = [false_mask for _ in data_arr_sr]
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp,
use_errors)
self.save_intermediate_img(hdulist, 'result_initial1.fits')
compute_cross_offsets = True
if compute_cross_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions_c = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions_c = self.compute_regions_from_objs(hdulist[0].data,
finalshape,
box=20)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
data_arr_sr, regions_c, refine=True, tol=1)
#
# Combined offsets
# Offsets in numpy order, swaping
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = offsets_xy_t[:, ::-1]
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape, offsetsp = narray.combine_shape(
subpixshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
self.logger.info('Shape of resized array (crosscorr) is %s',
finalshape)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate2 img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r,
'interm2_%03d.fits' % idx)
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_initial2.fits')
except Exception as error:
self.logger.warning('Error during cross-correlation, %s',
error)
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
data_arr_0 = [(d[r] + s)
for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
data_arr_r = [d.copy() for d in data_arr_sr]
for inum in range(1, rinput.iterations + 1):
# superflat
sf_data = self.compute_superflat(data_arr_0, objmask, regions,
channels)
fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
# apply superflat
data_arr_rf = data_arr_r
for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
arr_f = arr / sf_data
#arr_f = arr
base[reg] = arr_f
# compute sky advanced
data_arr_sky = []
data_arr_rfs = []
self.logger.info('Step %d, SC: computing advanced sky', inum)
scale = rinput.sky_images_sep_time * 60
tstamps = numpy.array([info['tstamp'] for info in img_info])
for idx, hdu in enumerate(data_hdul):
diff1 = tstamps - tstamps[idx]
idxs1 = (diff1 > 0) & (diff1 < scale)
idxs2 = (diff1 < 0) & (diff1 > -scale)
l1, = numpy.nonzero(idxs1)
l2, = numpy.nonzero(idxs2)
limit1 = l1[-rinput.sky_images:]
limit2 = l2[:rinput.sky_images]
len_l1 = len(limit1)
len_l2 = len(limit2)
self.logger.info('For image %s, using %d-%d images)', idx,
len_l1, len_l2)
if len_l1 + len_l2 == 0:
self.logger.error('No sky image available for frame %d',
idx)
raise ValueError('No sky image')
skydata = []
skymasks = []
skyscales = []
my_region = regions[idx]
my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
for i in numpy.concatenate((limit1, limit2)):
region_s = regions[i]
data_s = data_arr_rf[i][region_s]
mask_s = objmask[region_s]
scale_s = numpy.median(data_s)
skydata.append(data_s)
skymasks.append(mask_s)
skyscales.append(scale_s)
self.logger.debug('computing background with %d frames',
len(skydata))
sky, _, num = nacom.median(skydata, skymasks, scales=skyscales)
# rescale
sky *= my_sky_scale
binmask = num == 0
if numpy.any(binmask):
# We have pixels without
# sky background information
self.logger.warn(
'pixels without sky information when correcting %d',
idx)
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
narray.fixpix2(sky, binmask, out=sky, iterations=1)
name = 'sky_%d_%03d.fits' % (inum, idx)
fits.writeto(name, sky, overwrite=True)
name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
data_arr_sky.append(sky)
arr = numpy.copy(data_arr_rf[idx])
arr[my_region] = data_arr_rf[idx][my_region] - sky
data_arr_rfs.append(arr)
# subtract sky advanced
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_rfs):
self.save_intermediate_array(
arr_r, 'interm_%d_%03d.fits' % (inum, idx))
hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)
# For next step
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_0 = [
(d[r] + s)
for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)
]
data_arr_r = [d.copy() for d in data_arr_rfs]
result = self.create_result(frame=hdulist)
self.logger.info('end of dither recipe')
return result
def compute_sky_advanced(self, data_hdul, omasks, base_header, use_errors):
method = narray.combine.mean
self.logger.info('recombine images with segmentation mask')
sky_data = method([m[0].data for m in data_hdul],
masks=omasks,
dtype='float32')
hdu = fits.PrimaryHDU(sky_data[0], header=base_header)
points_no_data = (sky_data[2] == 0).sum()
self.logger.debug('update created sky image result header')
skyid = str(uuid.uuid1())
hdu.header['UUID'] = skyid
hdu.header['history'] = "Combined {} images using '{}'".format(
len(data_hdul), method.__name__)
hdu.header['history'] = 'Combination time {}'.format(
datetime.datetime.utcnow().isoformat())
for img in data_hdul:
hdu.header['history'] = "Image {}".format(
self.datamodel.get_imgid(img))
msg = "missing pixels, total: {}, fraction: {:3.1f}".format(
points_no_data, points_no_data / sky_data[2].size)
hdu.header['history'] = msg
self.logger.debug(msg)
if use_errors:
varhdu = fits.ImageHDU(sky_data[1], name='VARIANCE')
num = fits.ImageHDU(sky_data[2], name='MAP')
sky_result = fits.HDUList([hdu, varhdu, num])
else:
sky_result = fits.HDUList([hdu])
return sky_result
def combine_frames(self, frames, extinction, out=None, step=0):
self.logger.debug('Step %d, opening sky-subtracted frames', step)
def fits_open(name):
"""Open FITS with memmap in readonly mode"""
return fits.open(name, mode='readonly', memmap=True)
frameslll = [
fits_open(frame.lastname) for frame in frames if frame.valid_target
]
self.logger.debug('Step %d, opening mask frames', step)
mskslll = [
fits_open(frame.resized_mask) for frame in frames
if frame.valid_target
]
self.logger.debug('Step %d, combining %d frames', step, len(frameslll))
try:
extinc = [
pow(10, -0.4 * frame.metadata['airmass'] * extinction)
for frame in frames if frame.valid_target
]
data = [i['primary'].data for i in frameslll]
masks = [i['primary'].data for i in mskslll]
headers = [i['primary'].header for i in frameslll]
out = nacom.median(data,
masks,
scales=extinc,
dtype='float32',
out=out)
base_header = headers[0]
hdu = fits.PrimaryHDU(out[0], header=base_header)
hdu.header['history'] = "Combined %d images using '%s'" % (
len(frameslll), 'median')
hdu.header['history'] = 'Combination time {}'.format(
datetime.datetime.utcnow().isoformat())
for img in frameslll:
hdu.header['history'] = "Image {}".format(
img[0].header['uuid'])
prevnum = base_header.get('NUM-NCOM', 1)
hdu.header['NUM-NCOM'] = prevnum * len(frameslll)
hdu.header['NUMRNAM'] = 'FullDitheredImagesRecipe'
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = 'FULL_DITHERED_IMAGE'
# Headers of last image
hdu.header['TSUTC2'] = headers[-1]['TSUTC2']
varhdu = fits.ImageHDU(out[1], name='VARIANCE')
num = fits.ImageHDU(out[2].astype('uint8'), name='MAP')
result = fits.HDUList([hdu, varhdu, num])
# saving the three extensions
fits.writeto('result_i%0d.fits' % step, out[0], overwrite=True)
fits.writeto('result_i%0d_var.fits' % step, out[1], overwrite=True)
fits.writeto('result_i%0d_npix.fits' % step,
out[2],
overwrite=True)
result.writeto('result_i%0d_full.fits' % step, overwrite=True)
return result
finally:
self.logger.debug('Step %d, closing sky-subtracted frames', step)
for f in frameslll:
f.close()
self.logger.debug('Step %d, closing mask frames', step)
for f in mskslll:
f.close()
def resize(self,
frames,
shape,
offsetsp,
finalshape,
window=None,
scale=1,
step=0):
self.logger.info('Resizing frames and masks')
for frame, rel_offset in zip(frames, offsetsp):
if frame.valid_target:
region, _ = narray.subarray_match(finalshape, rel_offset,
shape)
# Valid region
frame.valid_region = region
# Relative offset
frame.rel_offset = rel_offset
# names of frame and mask
framen, maskn = name_redimensioned_frames(frame.label, step)
frame.resized_base = framen
frame.resized_mask = maskn
self.logger.debug(
'%s, valid region is %s, relative offset is %s',
frame.label, custom_region_to_str(region), rel_offset)
self.resize_frame_and_mask(frame, finalshape, framen, maskn,
window, scale)
def resize_frame_and_mask(self, frame, finalshape, framen, maskn, window,
scale):
self.logger.info('Resizing frame %s', frame.label)
with frame.origin.open() as hdul:
baseshape = hdul[0].data.shape
# FIXME: Resize_fits saves the resized image in framen
resize_fits(hdul,
framen,
finalshape,
frame.valid_region,
window=window,
scale=scale,
dtype='float32')
self.logger.info('Resizing mask %s', frame.label)
# We don't conserve the sum of the values of the frame here, just
# expand the mask
if frame.mask is None:
self.logger.warning('BPM missing, use zeros instead')
false_mask = numpy.zeros(baseshape, dtype='int16')
hdum = fits.HDUList(fits.PrimaryHDU(false_mask))
frame.mask = hdum #DataFrame(frame=hdum)
elif isinstance(frame.mask, nfcom.Extension):
ename = frame.mask.name
with frame.origin.open() as hdul:
frame.mask = fits.HDUList(hdul[ename].copy())
resize_fits(frame.mask,
maskn,
finalshape,
frame.valid_region,
fill=1,
window=window,
scale=scale,
conserve=False)
def create_mask(self, img, seeing_fwhm, step=0):
#
remove_border = True
# sextractor takes care of bad pixels
# if seeing_fwhm is not None and seeing_fwhm > 0:
# sex.config['SEEING_FWHM'] = seeing_fwhm * sex.config['PIXEL_SCALE']
if remove_border:
weigthmap = 'weights4rms.fits'
# Create weight map, remove n pixs from either side
# using a Hannig filter
# npix = 90
# w1 = npix
# w2 = npix
# wmap = numpy.ones_like(sf_data[0])
# cos_win1 = numpy.hanning(2 * w1)
# cos_win2 = numpy.hanning(2 * w2)
# wmap[:,:w1] *= cos_win1[:w1]
# wmap[:,-w1:] *= cos_win1[-w1:]
# wmap[:w2,:] *= cos_win2[:w2, numpy.newaxis]
# wmap[-w2:,:] *= cos_win2[-w2:, numpy.newaxis]
# Take the number of combined images from the combined image
wm = img[2].data.copy()
# Dont search objects where nimages < lower
# FIXME: this is a magic number
# We ignore objects in regions where we have less
# than 10% of the images
lower = wm.max() // 10
border = (wm < lower)
fits.writeto(weigthmap, border.astype('uint8'), overwrite=True)
# sex.config['WEIGHT_TYPE'] = 'MAP_WEIGHT'
# FIXME: this is a magic number
# sex.config['WEIGHT_THRESH'] = 50
# sex.config['WEIGHT_IMAGE'] = weigthmap
else:
border = None
data_res = img[0].data
bkg = sep.Background(data_res)
data_sub = data_res - bkg
self.logger.info('Runing source extraction in previous result')
objects, objmask = sep.extract(data_sub,
1.5,
err=bkg.globalrms,
mask=border,
segmentation_map=True)
fits.writeto(name_segmask(step), objmask, overwrite=True)
# # Plot objects
# # FIXME, plot sextractor objects on top of image
# patches = []
# fwhms = []
# nfirst = 0
# catalog_f = sopen(sex.config['CATALOG_NAME'])
# try:
# star = catalog_f.readline()
# while star:
# flags = star['FLAGS']
# # ignoring those objects with corrupted apertures
# if flags & sexcatalog.CORRUPTED_APER:
# star = catalog_f.readline()
# continue
# center = (star['X_IMAGE'], star['Y_IMAGE'])
# wd = 10 * star['A_IMAGE']
# hd = 10 * star['B_IMAGE']
# color = 'red'
# e = Ellipse(center, wd, hd, star['THETA_IMAGE'], color=color)
# patches.append(e)
# fwhms.append(star['FWHM_IMAGE'])
# nfirst += 1
# # FIXME Plot a ellipse
# star = catalog_f.readline()
# finally:
# catalog_f.close()
#
# p = PatchCollection(patches, alpha=0.4)
# ax = self._figure.gca()
# ax.add_collection(p)
# self._figure.canvas.draw()
# self._figure.savefig('figure-segmentation-overlay_%01d.png' % step)
#
# self.figure_fwhm_histogram(fwhms, step=step)
#
# # mode with an histogram
# hist, edges = numpy.histogram(fwhms, 50)
# idx = hist.argmax()
#
# seeing_fwhm = 0.5 * (edges[idx] + edges[idx + 1])
# if seeing_fwhm <= 0:
# _logger.warning(
# 'Seeing FHWM %f pixels is negative, reseting', seeing_fwhm)
# seeing_fwhm = None
# else:
# _logger.info('Seeing FHWM %f pixels (%f arcseconds)',
# seeing_fwhm, seeing_fwhm * sex.config['PIXEL_SCALE'])
# objmask = fits.getdata(name_segmask(step))
return objmask, seeing_fwhm
def compute_advanced_sky(self,
targetframes,
objmask,
skyframes=None,
target_is_sky=False,
maxsep=5.0,
nframes=10,
step=0,
save=True):
if target_is_sky:
skyframes = targetframes
# Each frame is its closest sky frame
nframes += 1
elif skyframes is None:
raise ValueError('skyframes not defined')
# build kdtree
sarray = numpy.array([frame.metadata['mjd'] for frame in skyframes])
# shape must be (n, 1)
sarray = numpy.expand_dims(sarray, axis=1)
# query
tarray = numpy.array([frame.metadata['mjd'] for frame in targetframes])
# shape must be (n, 1)
tarray = numpy.expand_dims(tarray, axis=1)
kdtree = KDTree(sarray)
# 1 / minutes in a Julian day
SCALE = 60.0
# max_time_sep = ri.sky_images_sep_time / 1440.0
_dis, idxs = kdtree.query(tarray,
k=nframes,
distance_upper_bound=maxsep * SCALE)
nsky = len(sarray)
for tid, idss in enumerate(idxs):
try:
tf = targetframes[tid]
self.logger.info('Step %d, SC: computing advanced sky for %s',
step, tf.label)
# filter(lambda x: x < nsky, idss)
locskyframes = []
for si in idss:
if tid == si:
# this sky frame it is the current frame, reject
continue
if si < nsky:
self.logger.debug('Step %d, SC: %s is a sky frame',
step, skyframes[si].label)
locskyframes.append(skyframes[si])
self.compute_advanced_sky_for_frame(tf,
locskyframes,
step=step,
save=save)
except IndexError:
self.logger.error('No sky image available for frame %s',
tf.lastname)
raise
def compute_advanced_sky_for_frame(self,
frame,
skyframes,
step=0,
save=True):
self.logger.info('Correcting sky in frame %s', frame.lastname)
self.logger.info('with sky computed from frames')
for i in skyframes:
self.logger.info('%s', i.flat_corrected)
data = []
scales = []
masks = []
# handle the FITS file to close it finally
desc = []
try:
for i in skyframes:
filename = i.flat_corrected
hdulist = fits.open(filename, mode='readonly', memmap=True)
data.append(hdulist['primary'].data[i.valid_region])
desc.append(hdulist)
#scales.append(numpy.median(data[-1]))
if i.objmask_data is not None:
masks.append(i.objmask_data)
self.logger.debug('object mask is shared')
elif i.objmask is not None:
hdulistmask = fits.open(i.objmask,
mode='readonly',
memmap=True)
masks.append(hdulistmask['primary'].data)
desc.append(hdulistmask)
self.logger.debug('object mask is particular')
else:
self.logger.warn('no object mask for %s', filename)
self.logger.debug('computing background with %d frames', len(data))
sky, _, num = nacom.median(data, masks) #, scales=scales)
finally:
# Closing all FITS files
for hdl in desc:
hdl.close()
if numpy.any(num == 0):
# We have pixels without
# sky background information
self.logger.warn(
'pixels without sky information when correcting %s',
frame.flat_corrected)
binmask = num == 0
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
narray.fixpix2(sky, binmask, out=sky, iterations=1)
name = name_skybackgroundmask(frame.label, step)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
name_sky = name_skybackground(frame.label, step)
fits.writeto(name_sky, sky, overwrite=True)
dst = name_skysub_proc(frame.label, step)
prev = frame.lastname
shutil.copyfile(prev, dst)
frame.lastname = dst
with fits.open(frame.lastname, mode='update') as hdulist:
data = hdulist['primary'].data
valid = data[frame.valid_region]
valid -= sky
def compute_regions_from_objs(self, arr, finalshape, box=50, corners=True):
regions = []
catalog, mask = self.create_object_catalog(arr, border=300)
self.save_intermediate_array(mask, 'objmask.fits')
# with the catalog, compute 5 objects
LIMIT_AREA = 5000
NKEEP = 1
idx_small = catalog['npix'] < LIMIT_AREA
objects_small = catalog[idx_small]
idx_flux = objects_small['flux'].argsort()
objects_nth = objects_small[idx_flux][-NKEEP:]
for obj in objects_nth:
print('ref is', obj['x'], obj['y'])
region = nautils.image_box2d(obj['x'], obj['y'], finalshape,
(box, box))
print(region)
regions.append(region)
return regions
def create_object_catalog(self, arr, threshold=3.0, border=0):
if border > 0:
wmap = numpy.ones_like(arr)
wmap[border:-border, border:-border] = 0
else:
wmap = None
bkg = sep.Background(arr)
data_sub = arr - bkg
objects, objmask = sep.extract(data_sub,
threshold,
err=bkg.globalrms *
numpy.ones_like(data_sub),
mask=wmap,
segmentation_map=True)
return objects, objmask
class TestPointSourceRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
self.logger.info('starting processing for object detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
self.logger.debug('finding point sources')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
data = hdulist[0].data
# Copy needed in numpy 1.7
# This seems already bitswapped??
# FIXME: check this works offline/online
# ndata = data.byteswap().newbyteorder()
# data = data.byteswap(inplace=True).newbyteorder()
snr_detect = 5.0
fwhm = 4.0
npixels = 15
box_shape = [64, 64]
self.logger.info('point source detection2')
self.logger.info('using internal mask to remove corners')
# Corners
mask = numpy.zeros_like(data, dtype='int32')
mask[2000:, 0:80] = 1
mask[2028:, 2000:] = 1
mask[:50, 1950:] = 1
mask[:100, :50] = 1
# Remove corner regions
self.logger.info('compute background map, %s', box_shape)
bkg = sep.Background(data)
self.logger.info('reference fwhm is %5.1f pixels', fwhm)
self.logger.info('detect threshold, %3.1f over background', snr_detect)
self.logger.info('convolve with gaussian kernel, FWHM %3.1f pixels',
fwhm)
sigma = fwhm * gaussian_fwhm_to_sigma
#
kernel = Gaussian2DKernel(sigma)
kernel.normalize()
thresh = snr_detect * bkg.globalrms
data_s = data - bkg.back()
objects, segmap = sep.extract(data - bkg.back(),
thresh,
minarea=npixels,
filter_kernel=kernel.array,
segmentation_map=True,
mask=mask)
fits.writeto('segmap.fits', segmap)
self.logger.info('detected %d objects', len(objects))
# Hardcoded values
rs2 = 15.0
fit_rad = 10.0
flux_min = 1000.0
flux_max = 30000.0
self.logger.debug('Flux limit is %6.1f %6.1f', flux_min, flux_max)
# FIXME: this should be a view, not a copy
xall = objects['x']
yall = objects['y']
mm = numpy.array([xall, yall]).T
self.logger.info('computing FWHM')
# Find objects with pairs inside fit_rad
kdtree = KDTree(mm)
nearobjs = (kdtree.query_ball_tree(kdtree, r=fit_rad))
positions = []
for idx, obj in enumerate(objects):
x0 = obj['x']
y0 = obj['y']
sl = image_box2d(x0, y0, data.shape, (fit_rad, fit_rad))
# sl_sky = image_box2d(x0, y0, data.shape, (rs2, rs2))
part_s = data_s[sl]
# Logical coordinates
xx0 = x0 - sl[1].start
yy0 = y0 - sl[0].start
_, fwhm_x, fwhm_y = compute_fwhm_2d_simple(part_s, xx0, yy0)
if min(fwhm_x, fwhm_x) < 3:
continue
if flux_min > obj['peak'] or flux_max < obj['peak']:
continue
# nobjs is the number of object inside fit_rad
nobjs = len(nearobjs[idx])
flag = 0 if nobjs == 1 else 1
positions.append([idx, x0, y0, obj['peak'], fwhm_x, fwhm_y, flag])
self.logger.info('saving photometry')
positions = numpy.array(positions)
positions_alt = positions
self.logger.info('end processing for object detection')
result = self.create_result(
frame=hdulist,
positions=positions_alt,
positions_alt=positions_alt,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class CosmeticsRecipe(EmirRecipe):
"""Detector Cosmetics.
Recipe to find and tag bad pixels in the detector.
"""
obresult = ObservationResultRequirement()
insconf = InstrumentConfigurationRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
lowercut = Parameter(
4.0, 'Values below this sigma level are flagged as dead pixels')
uppercut = Parameter(
4.0, 'Values above this sigma level are flagged as hot pixels')
maxiter = Parameter(30, 'Maximum number of iterations')
ratioframe = Result(prods.ProcessedImage)
maskframe = Result(prods.MasterBadPixelMask)
def run(self, rinput):
# FIXME:
# We need 2 flats
# Of different exposure times
#
# And their calibrations
#
if len(rinput.obresult.frames) < 2:
raise RecipeError('The recipe requires 2 flat frames')
iinfo = []
for frame in rinput.obresult.frames:
with frame.open() as hdulist:
iinfo.append(gather_info(hdulist))
# Loading calibrations
with rinput.master_bias.open() as hdul:
readmode = hdul[0].header.get('READMODE', 'undefined')
if readmode.lower() in ['simple', 'bias']:
self.logger.debug('loading bias')
mbias = hdul[0].data
bias_corrector = proc.BiasCorrector(mbias)
else:
self.logger.debug('ignoring bias')
bias_corrector = numina.util.node.IdNode()
with rinput.master_dark.open() as mdark_hdul:
self.logger.debug('loading dark')
mdark = mdark_hdul[0].data
dark_corrector = proc.DarkCorrector(mdark)
flow = numina.util.flow.SerialFlow([bias_corrector, dark_corrector])
self.logger.info('processing flat #1')
with rinput.obresult.frames[0].open() as hdul:
other = flow(hdul)
f1 = other[0].data.copy() * iinfo[0]['texp'] * 1e-3
self.logger.info('processing flat #2')
with rinput.obresult.frames[1].open() as hdul:
other = flow(hdul)
f2 = other[0].data.copy() * iinfo[1]['texp'] * 1e-3
# Preprocess...
maxiter = rinput.maxiter
lowercut = rinput.lowercut
uppercut = rinput.uppercut
ninvalid = 0
mask = None
if mask:
m = fits.getdata(mask)
ninvalid = numpy.count_nonzero(m)
else:
m = numpy.zeros_like(f1, dtype='int')
for niter in range(1, maxiter + 1):
self.logger.debug('iter %d', niter)
ratio, m, sigma = cosmetics(f1,
f2,
m,
lowercut=lowercut,
uppercut=uppercut)
if self.intermediate_results:
with warnings.catch_warnings():
warnings.simplefilter('ignore')
fits.writeto('numina-cosmetics-i%02d.fits' % niter,
ratio,
overwrite=True)
fits.writeto('numina-mask-i%02d.fits' % niter,
m,
overwrite=True)
fits.writeto('numina-sigma-i%02d.fits' % niter,
m * 0.0 + sigma,
overwrite=True)
self.logger.debug('iter %d, invalid points in input mask: %d',
niter, ninvalid)
self.logger.debug('iter %d, estimated sigma is %f', niter, sigma)
n_ninvalid = numpy.count_nonzero(m)
# Probably there is something wrong here
# too much defective pixels
if ninvalid / m.size >= 0.10:
# This should set a flag in the output
msg = 'defective pixels are greater than 10%'
self.logger.warning(msg)
if n_ninvalid == ninvalid:
self.logger.info('convergence reached after %d iterations',
niter)
break
self.logger.info('new invalid points: %d', n_ninvalid - ninvalid)
ninvalid = n_ninvalid
else:
# This should set a flag in the output
msg = 'convergence not reached after %d iterations' % maxiter
self.logger.warning(msg)
self.logger.info('number of dead pixels %d',
numpy.count_nonzero(m == PIXEL_DEAD))
self.logger.info('number of hot pixels %d',
numpy.count_nonzero(m == PIXEL_HOT))
if self.intermediate_results:
with warnings.catch_warnings():
warnings.simplefilter('ignore')
fits.writeto('numina-cosmetics.fits', ratio, overwrite=True)
fits.writeto('numina-mask.fits', m, overwrite=True)
fits.writeto('numina-sigma.fits',
sigma * numpy.ones_like(m),
overwrite=True)
hdu = fits.PrimaryHDU(ratio)
hdr = hdu.header
hdr['NUMXVER'] = (__version__, 'Numina package version')
hdr['NUMRNAM'] = (self.__class__.__name__, 'Numina recipe name')
hdr['NUMRVER'] = (self.__version__, 'Numina recipe version')
ratiohdl = fits.HDUList([hdu])
maskhdu = fits.PrimaryHDU(m)
hdr = maskhdu.header
hdr['NUMXVER'] = (__version__, 'Numina package version')
hdr['NUMRNAM'] = (self.__class__.__name__, 'Numina recipe name')
hdr['NUMRVER'] = (self.__version__, 'Numina recipe version')
maskhdl = fits.HDUList([maskhdu])
res = self.create_result(ratioframe=ratiohdl, maskframe=maskhdl)
return res
class GainRecipe1(EmirRecipe):
"""Detector Gain Recipe.
Recipe to calibrate the detector gain.
"""
obresult = ObservationResultRequirement()
region = Parameter('channel', 'Region used to compute: '
'(full|quadrant|channel)',
choices=['full', 'quadrant', 'channel'])
gain = Product(MasterGainMap(None, None, None))
ron = Product(MasterRONMap(None, None))
def region(self, reqs):
mm = reqs['region'].tolower()
if mm == 'full':
return ((slice(0, 2048), slice(0, 2048)))
elif mm == 'quadrant':
return QUADRANTS
elif mm == 'channel':
return CHANNELS
else:
raise ValueError
def run(self, rinput):
resets = []
ramps = []
for frame in rinput.obresult.frames:
if frame.itype == 'RESET':
resets.append(frame.label)
_logger.debug('%s is RESET', frame.label)
elif frame.itype == 'RAMP':
ramps.append(frame.label)
_logger.debug('%s is RAMP', frame.label)
else:
raise RecipeError('frame is neither a RAMP nor a RESET')
channels = self.region(rinput)
result_gain = numpy.zeros((len(channels), ))
result_ron = numpy.zeros_like(result_gain)
counts = numpy.zeros((len(ramps), len(channels)))
variance = numpy.zeros_like(counts)
last_reset = resets[-1]
_logger.debug('opening last reset image %s', last_reset)
last_reset_data = fits.getdata(last_reset)
for i, di in enumerate(ramps):
with fits.open(di, mode='readonly') as fd:
restdata = fd[0].data - last_reset_data
for j, channel in enumerate(channels):
c = restdata[channel].mean()
_logger.debug('%f counts in channel', c)
counts[i, j] = c
v = restdata[channel].var(ddof=1)
_logger.debug('%f variance in channel', v)
variance[i, j] = v
for j, _ in enumerate(channels):
res = scipy.stats.linregress(counts[:, j], variance[:, j])
slope, intercept, _r_value, _p_value, _std_err = res
result_gain[j] = 1.0 / slope
result_ron[j] = math.sqrt(intercept)
cube = numpy.zeros((2, 2048, 2048))
for gain, var, channel in zip(result_gain, result_ron, channels):
cube[0][channel] = gain
cube[1][channel] = var
hdu = fits.PrimaryHDU(cube[0])
hduvar = fits.ImageHDU(cube[1])
hdulist = fits.HDUList([hdu, hduvar])
gain = MasterGainMap(mean=result_gain,
var=numpy.array([]),
frame=DataFrame(hdulist))
ron = MasterRONMap(mean=result_ron, var=numpy.array([]))
return self.create_result(gain=gain, ron=ron)
class FullDitheredImagesRecipe(JoinDitheredImagesRecipe):
obresult = ObservationResultRequirement(
query_opts=ResultOf('frame', node='children'))
master_bpm = reqs.MasterBadPixelMaskRequirement()
# extinction = Extinction_Requirement()
# sources = Catalog_Requirement()
# offsets = Offsets_Requirement()
offsets = Requirement(prods.CoordinateList2DType,
'List of pairs of offsets',
optional=True)
iterations = Parameter(4, 'Iterations of the recipe')
sky_images = Parameter(
5, 'Images used to estimate the '
'background before and after current image')
sky_images_sep_time = reqs.SkyImageSepTime_Requirement()
check_photometry_levels = Parameter(
[0.5, 0.8], 'Levels to check the flux of the objects')
check_photometry_actions = Parameter(['warn', 'warn', 'default'],
'Actions to take on images')
frame = Result(prods.ProcessedImage)
sky = Result(prods.ProcessedImage, optional=True)
catalog = Result(prods.SourcesCatalog, optional=True)
def run(self, rinput):
partial_result = self.run_single(rinput)
return partial_result
def run_single(self, rinput):
obresult = rinput.obresult
# just in case images are in result, instead of frames
if not obresult.frames:
frames = obresult.results
else:
frames = obresult.frames
img_info = []
data_hdul = []
for f in frames:
img = f.open()
data_hdul.append(img)
info = {}
info['tstamp'] = img[0].header['tstamp']
info['airmass'] = img[0].header['airmass']
img_info.append(info)
channels = FULL
use_errors = True
# Initial checks
baseimg = data_hdul[0]
has_num_ext = 'NUM' in baseimg
has_bpm_ext = 'BPM' in baseimg
baseshape = baseimg[0].shape
subpixshape = baseshape
base_header = baseimg[0].header
compute_sky = 'NUM-SK' not in base_header
compute_sky_advanced = False
self.logger.debug('base image is: %s',
self.datamodel.get_imgid(baseimg))
self.logger.debug('images have NUM extension: %s', has_num_ext)
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
self.logger.debug('compute sky is needed: %s', compute_sky)
if compute_sky:
self.logger.info('compute sky simple')
sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
self.save_intermediate_img(sky_result, 'sky_init.fits')
sky_result.writeto('sky_init.fits', overwrite=True)
sky_data = sky_result[0].data
self.logger.debug('sky image has shape %s', sky_data.shape)
self.logger.info('sky correction in individual images')
corrector = proc.SkyCorrector(
sky_data,
self.datamodel,
calibid=self.datamodel.get_imgid(sky_result))
# If we do not update keyword SKYADD
# there is no sky subtraction
for m in data_hdul:
m[0].header['SKYADD'] = True
# this is a little hackish
# sky corrected
data_hdul_s = [corrector(m) for m in data_hdul]
base_header = data_hdul_s[0][0].header
else:
sky_result = None
data_hdul_s = data_hdul
self.logger.info('Computing offsets from WCS information')
finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
data_hdul_s, baseshape, subpixshape)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
# Resizing target imgs
data_arr_sr, regions = resize_arrays([m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if has_num_ext:
self.logger.debug('Using NUM extension')
masks = [
numpy.where(m['NUM'].data, 0, 1).astype('int16')
for m in data_hdul
]
elif has_bpm_ext:
self.logger.debug('Using BPM extension')
#
masks = [
numpy.where(m['BPM'].data, 1, 0).astype('int16')
for m in data_hdul
]
else:
self.logger.warning('BPM missing, use zeros instead')
false_mask = numpy.zeros(baseshape, dtype='int16')
masks = [false_mask for _ in data_arr_sr]
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp,
use_errors)
self.save_intermediate_img(hdulist, 'result_initial1.fits')
compute_cross_offsets = True
if compute_cross_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions_c = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions_c = self.compute_regions_from_objs(hdulist[0].data,
finalshape,
box=20)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
data_arr_sr, regions_c, refine=True, tol=1)
#
# Combined offsets
# Offsets in numpy order, swaping
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = offsets_xy_t[:, ::-1]
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape, offsetsp = combine_shape(subpixshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
self.logger.info('Shape of resized array (crosscorr) is %s',
finalshape)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
data_arr_sr, regions = resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate2 img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r,
'interm2_%03d.fits' % idx)
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_initial2.fits')
except Exception as error:
self.logger.warning('Error during cross-correlation, %s',
error)
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
data_arr_0 = [(d[r] + s)
for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
data_arr_r = [d.copy() for d in data_arr_sr]
for inum in range(1, rinput.iterations + 1):
# superflat
sf_data = self.compute_superflat(data_arr_0, objmask, regions,
channels)
fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
# apply superflat
data_arr_rf = data_arr_r
for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
arr_f = arr / sf_data
#arr_f = arr
base[reg] = arr_f
# compute sky advanced
data_arr_sky = []
data_arr_rfs = []
self.logger.info('Step %d, SC: computing advanced sky', inum)
scale = rinput.sky_images_sep_time * 60
tstamps = numpy.array([info['tstamp'] for info in img_info])
for idx, hdu in enumerate(data_hdul):
diff1 = tstamps - tstamps[idx]
idxs1 = (diff1 > 0) & (diff1 < scale)
idxs2 = (diff1 < 0) & (diff1 > -scale)
l1, = numpy.nonzero(idxs1)
l2, = numpy.nonzero(idxs2)
limit1 = l1[-rinput.sky_images:]
limit2 = l2[:rinput.sky_images]
len_l1 = len(limit1)
len_l2 = len(limit2)
self.logger.info('For image %s, using %d-%d images)', idx,
len_l1, len_l2)
if len_l1 + len_l2 == 0:
self.logger.error('No sky image available for frame %d',
idx)
raise ValueError('No sky image')
skydata = []
skymasks = []
skyscales = []
my_region = regions[idx]
my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
for i in numpy.concatenate((limit1, limit2)):
region_s = regions[i]
data_s = data_arr_rf[i][region_s]
mask_s = objmask[region_s]
scale_s = numpy.median(data_s)
skydata.append(data_s)
skymasks.append(mask_s)
skyscales.append(scale_s)
self.logger.debug('computing background with %d frames',
len(skydata))
sky, _, num = median(skydata, skymasks, scales=skyscales)
# rescale
sky *= my_sky_scale
binmask = num == 0
if numpy.any(binmask):
# We have pixels without
# sky background information
self.logger.warn(
'pixels without sky information when correcting %d',
idx)
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
fixpix2(sky, binmask, out=sky, iterations=1)
name = 'sky_%d_%03d.fits' % (inum, idx)
fits.writeto(name, sky, overwrite=True)
name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
data_arr_sky.append(sky)
arr = numpy.copy(data_arr_rf[idx])
arr[my_region] = data_arr_rf[idx][my_region] - sky
data_arr_rfs.append(arr)
# subtract sky advanced
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_rfs):
self.save_intermediate_array(
arr_r, 'interm_%d_%03d.fits' % (inum, idx))
hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)
# For next step
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_0 = [
(d[r] + s)
for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)
]
data_arr_r = [d.copy() for d in data_arr_rfs]
result = self.create_result(frame=hdulist)
self.logger.info('end of dither recipe')
return result
def compute_superflat(self, data_arr_r, objmask, regions, channels):
# superflat
mask = [objmask[r] for r in regions]
scales = [numpy.median(d) for d in data_arr_r]
self.logger.debug('flat scaling %s', scales)
sf_data, _sf_var, sf_num = flatcombine(data_arr_r,
masks=mask,
scales=scales)
for channel in channels:
mask = (sf_num[channel] == 0)
if numpy.any(mask):
fixpix2(sf_data[channel], mask, out=sf_data[channel])
# Normalize, flat has mean = 1
sf_data /= sf_data.mean()
return sf_data
def compute_sky_advanced(self, data_hdul, omasks, base_header, use_errors):
method = combine.mean
self.logger.info('recombine images with segmentation mask')
sky_data = method([m[0].data for m in data_hdul],
masks=omasks,
dtype='float32')
hdu = fits.PrimaryHDU(sky_data[0], header=base_header)
points_no_data = (sky_data[2] == 0).sum()
self.logger.debug('update created sky image result header')
skyid = str(uuid.uuid1())
hdu.header['UUID'] = skyid
hdu.header['history'] = "Combined {} images using '{}'".format(
len(data_hdul), method.__name__)
hdu.header['history'] = 'Combination time {}'.format(
datetime.datetime.utcnow().isoformat())
for img in data_hdul:
hdu.header['history'] = "Image {}".format(
self.datamodel.get_imgid(img))
msg = "missing pixels, total: {}, fraction: {:3.1f}".format(
points_no_data, points_no_data / sky_data[2].size)
hdu.header['history'] = msg
self.logger.debug(msg)
if use_errors:
varhdu = fits.ImageHDU(sky_data[1], name='VARIANCE')
num = fits.ImageHDU(sky_data[2], name='MAP')
sky_result = fits.HDUList([hdu, varhdu, num])
else:
sky_result = fits.HDUList([hdu])
return sky_result
class ABBASpectraFastRectwv(EmirRecipe):
"""Process AB or ABBA images applying a single wavelength calibration
Note that the wavelength calibration has already been determined.
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement()
pattern = Parameter('ABBA',
description='Observation pattern',
choices=['AB', 'ABBA'])
# do not allow 'sum' as combination method in order to avoid
# confusion with number of counts in resulting image
method = Parameter('mean',
description='Combination method',
choices=['mean', 'median', 'sigmaclip'])
method_kwargs = Parameter(dict(),
description='Arguments for combination method')
voffset_pix = Parameter(0.0,
description='Shift (pixels) to move A into B',
optional=True)
reduced_mos_abba = Result(prods.ProcessedMOS)
reduced_mos_abba_combined = Result(prods.ProcessedMOS)
def run(self, rinput):
nimages = len(rinput.obresult.frames)
pattern = rinput.pattern
pattern_length = len(pattern)
# check combination method
if rinput.method != 'sigmaclip':
if rinput.method_kwargs != {}:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# check pattern sequence matches number of images
if nimages % pattern_length != 0:
raise ValueError('Number of images is not a multiple of pattern '
'length: {}, {}'.format(nimages, pattern_length))
nsequences = nimages // pattern_length
rectwv_coeff = rinput.rectwv_coeff
self.logger.info(rectwv_coeff)
self.logger.info('observation pattern: {}'.format(pattern))
self.logger.info('nsequences.........: {}'.format(nsequences))
full_set = pattern * nsequences
self.logger.info('full set of images.: {}'.format(full_set))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# available combination methods
method = getattr(combine, rinput.method)
method_kwargs = rinput.method_kwargs
# basic reduction of A images
list_a = [
rinput.obresult.frames[i] for i, char in enumerate(full_set)
if char == 'A'
]
with contextlib.ExitStack() as stack:
self.logger.info('starting basic reduction of A images')
hduls = [stack.enter_context(fname.open()) for fname in list_a]
reduced_image_a = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
reduced_image_a = flow(reduced_image_a)
hdr = reduced_image_a[0].header
self.set_base_headers(hdr)
# basic reduction of B images
list_b = [
rinput.obresult.frames[i] for i, char in enumerate(full_set)
if char == 'B'
]
with contextlib.ExitStack() as stack:
self.logger.info('starting basic reduction of B images')
hduls = [stack.enter_context(fname.open()) for fname in list_b]
reduced_image_b = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
reduced_image_b = flow(reduced_image_b)
hdr = reduced_image_b[0].header
self.set_base_headers(hdr)
# save intermediate reduced_image_a and reduced_image_b
self.save_intermediate_img(reduced_image_a, 'reduced_image_a.fits')
self.save_intermediate_img(reduced_image_b, 'reduced_image_b.fits')
# computation of A-B
header_a = reduced_image_a[0].header
header_b = reduced_image_b[0].header
data_a = reduced_image_a[0].data.astype('float32')
data_b = reduced_image_b[0].data.astype('float32')
reduced_data = data_a - data_b
# update reduced image header
reduced_image = self.create_reduced_image(
rinput,
reduced_data,
header_a,
header_b,
rinput.pattern,
full_set,
voffset_pix=0,
header_mos_abba=None,
)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# apply rectification and wavelength calibration
self.logger.info('begin rect.+wavecal. reduction of ABBA spectra')
reduced_mos_abba = apply_rectwv_coeff(reduced_image, rectwv_coeff)
header_mos_abba = reduced_mos_abba[0].header
# combine A and B data by shifting B on top of A
voffset_pix = rinput.voffset_pix
if voffset_pix is not None and voffset_pix != 0:
self.logger.info(
'correcting vertical offset (pixesl): {}'.format(voffset_pix))
reduced_mos_abba_data = reduced_mos_abba[0].data.astype('float32')
shifted_a_minus_b_data = shift_image2d(
reduced_mos_abba_data,
yoffset=-voffset_pix,
).astype('float32')
reduced_mos_abba_combined_data = \
reduced_mos_abba_data - shifted_a_minus_b_data
# scale signal to exposure of a single image
reduced_mos_abba_combined_data /= 2.0
else:
reduced_mos_abba_combined_data = None
# update reduced combined image header
reduced_mos_abba_combined = self.create_reduced_image(
rinput,
reduced_mos_abba_combined_data,
header_a,
header_b,
rinput.pattern,
full_set,
voffset_pix,
header_mos_abba=header_mos_abba)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos_abba,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of ABBA spectra')
result = self.create_result(
reduced_mos_abba=reduced_mos_abba,
reduced_mos_abba_combined=reduced_mos_abba_combined)
return result
def create_reduced_image(self, rinput, reduced_data, header_a, header_b,
pattern, full_set, voffset_pix, header_mos_abba):
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
# Copy header of first image
base_header = hduls[0][0].header.copy()
hdu = fits.PrimaryHDU(reduced_data, header=base_header)
self.set_base_headers(hdu.header)
self.logger.debug('update result header')
if header_mos_abba is not None:
self.logger.debug('update result header')
crpix1 = header_mos_abba['crpix1']
crval1 = header_mos_abba['crval1']
cdelt1 = header_mos_abba['cdelt1']
# update wavelength calibration in FITS header
for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']:
if keyword in hdu.header:
hdu.header.remove(keyword)
hdu.header['crpix1'] = (crpix1, 'reference pixel')
hdu.header['crval1'] = (crval1, 'central wavelength at crpix1')
hdu.header['cdelt1'] = \
(cdelt1, 'linear dispersion (Angstrom/pixel)')
hdu.header['cunit1'] = ('Angstrom', 'units along axis1')
hdu.header['ctype1'] = 'WAVELENGTH'
hdu.header['crpix2'] = (0.0, 'reference pixel')
hdu.header['crval2'] = (0.0, 'central value at crpix2')
hdu.header['cdelt2'] = (1.0, 'increment')
hdu.header['ctype2'] = 'PIXEL'
hdu.header['cunit2'] = ('Pixel', 'units along axis2')
for keyword in [
'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2',
'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2'
]:
if keyword in hdu.header:
hdu.header.remove(keyword)
# update additional keywords
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = pattern + ' pattern'
hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2']
hdu.header['history'] = "Processed " + pattern + " pattern"
hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames')
# update history
dm = emirdrp.datamodel.EmirDataModel()
for img, key in zip(hduls, full_set):
imgid = dm.get_imgid(img)
hdu.header['HISTORY'] = "Image '{}' is '{}'".format(imgid, key)
hdu.header['HISTORY'] = "Processed " + pattern + " pattern"
hdu.header['HISTORY'] = '--- Reduction of A images ---'
for line in header_a['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header['HISTORY'] = '--- Reduction of B images ---'
for line in header_b['HISTORY']:
hdu.header['HISTORY'] = line
if voffset_pix is not None and voffset_pix != 0:
hdu.header['HISTORY'] = '--- Combination of AB spectra ---'
hdu.header['HISTORY'] = "voffset_pix between A and B {}".format(
voffset_pix)
hdu.header.add_history('--- rinput.stored() (BEGIN) ---')
for item in rinput.stored():
value = getattr(rinput, item)
cline = '{}: {}'.format(item, value)
hdu.header.add_history(cline)
hdu.header.add_history('--- rinput.stored() (END) ---')
result = fits.HDUList([hdu])
return result
def set_base_headers(self, hdr):
newhdr = super(ABBASpectraFastRectwv, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
def extract_tags_from_obsres(self, obsres, tag_keys):
# this function is necessary to make use of recipe requirements
# that are provided as lists
final_tags = []
for frame in obsres.frames:
ref_img = frame.open()
tag = self.extract_tags_from_ref(ref_img,
tag_keys,
base=obsres.labels)
final_tags.append(tag)
return final_tags
class ABBASpectraRectwv(EmirRecipe):
"""Process images in AB or ABBA mode applying wavelength calibration
Note that in this case the wavelength calibration has already been
determined.
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True)
list_rectwv_coeff = reqs.ListOfRectWaveCoeffRequirement(optional=True)
pattern = Parameter('ABBA',
description='Observation pattern',
choices=['AB', 'ABBA'])
# do not allow 'sum' as combination method in order to avoid
# confusion with number of counts in resulting image
method = Parameter('mean',
description='Combination method',
choices=['mean', 'median', 'sigmaclip'])
method_kwargs = Parameter(dict(),
description='Arguments for combination method',
optional=True)
refine_target_along_slitlet = Parameter(
dict(),
description='Parameters to refine location of target along the slitlet'
)
reduced_mos_abba = Result(prods.ProcessedMOS)
reduced_mos_abba_combined = Result(prods.ProcessedMOS)
def run(self, rinput):
nimages = len(rinput.obresult.frames)
pattern = rinput.pattern
pattern_length = len(pattern)
# check combination method
if rinput.method != 'sigmaclip':
if rinput.method_kwargs != {}:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# check pattern sequence matches number of images
if nimages % pattern_length != 0:
raise ValueError('Number of images is not a multiple of pattern '
'length: {}, {}'.format(nimages, pattern_length))
nsequences = nimages // pattern_length
# check rectification and wavelength calibration information
if rinput.rectwv_coeff is None and rinput.list_rectwv_coeff is None:
raise ValueError('No rectwv_coeff nor list_rectwv_coeff data have '
'been provided')
elif rinput.rectwv_coeff is not None and \
rinput.list_rectwv_coeff is not None:
raise ValueError("rectwv_coeff and list_rectwv_coeff cannot be "
"used simultaneously")
elif rinput.rectwv_coeff is not None:
list_rectwv_coeff = [rinput.rectwv_coeff] * nimages
elif rinput.list_rectwv_coeff is not None:
if len(rinput.list_rectwv_coeff) != nimages:
raise ValueError("Unexpected number of rectwv_coeff files "
"in list_rectwv_coeff")
else:
list_rectwv_coeff = rinput.list_rectwv_coeff
# check filter and grism are the same in all JSON files
for item in ['grism', 'filter']:
list_values = []
for calib in list_rectwv_coeff:
list_values.append(calib.tags[item])
if len(set(list_values)) != 1:
raise ValueError(
'list_rectwv_coeff contains coefficients for '
'different {}s'.format(item))
else:
raise ValueError("Unexpected error!")
# grism and filter names
grism_name = list_rectwv_coeff[0].tags['grism']
filter_name = list_rectwv_coeff[0].tags['filter']
# compute offsets from WCS info in image headers
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
sep_arcsec, spatial_scales = compute_wcs_offsets(hduls)
sep_pixel = np.round(sep_arcsec / spatial_scales, 6)
for i in range(nimages):
self.logger.info(list_rectwv_coeff[i])
self.logger.info(
'observation pattern..................: {}'.format(pattern))
self.logger.info(
'nsequences...........................: {}'.format(nsequences))
self.logger.info(
'offsets (arcsec, from WCS)...........: {}'.format(sep_arcsec))
self.logger.info(
'spatial scales (arcsec/pix, from WCS): {}'.format(spatial_scales))
self.logger.info(
'spatial scales (pixels, from WCS)....: {}'.format(sep_pixel))
full_set = pattern * nsequences
self.logger.info(
'full set of images...................: {}'.format(full_set))
# basic parameters to determine useful pixels in the wavelength
# direction
dict_rtas = rinput.refine_target_along_slitlet
valid_keys = [
'npix_removed_near_ohlines', 'nwidth_medfilt',
'save_individual_images', 'ab_different_target',
'vpix_region_a_target', 'vpix_region_a_sky',
'vpix_region_b_target', 'vpix_region_b_sky',
'list_valid_wvregions_a', 'list_valid_wvregions_b'
]
for dumkey in dict_rtas.keys():
if dumkey not in valid_keys:
raise ValueError('Unexpected key={}'.format(dumkey))
if 'vpix_region_a_target' in dict_rtas.keys():
vpix_region_a_target = dict_rtas['vpix_region_a_target']
else:
vpix_region_a_target = None
if 'vpix_region_a_sky' in dict_rtas.keys():
vpix_region_a_sky = dict_rtas['vpix_region_a_sky']
else:
vpix_region_a_sky = None
if 'vpix_region_b_target' in dict_rtas.keys():
vpix_region_b_target = dict_rtas['vpix_region_b_target']
else:
vpix_region_b_target = None
if 'vpix_region_b_sky' in dict_rtas.keys():
vpix_region_b_sky = dict_rtas['vpix_region_b_sky']
else:
vpix_region_b_sky = None
if 'ab_different_target' in dict_rtas.keys():
ab_different_target = int(dict_rtas['ab_different_target'])
if ab_different_target not in [-1, 0, 1, 9]:
raise ValueError('Invalid ab_different_target={} value'.format(
ab_different_target))
else:
raise ValueError('Missing ab_different_target value')
try:
npix_removed_near_ohlines = \
int(dict_rtas['npix_removed_near_ohlines'])
except KeyError:
npix_removed_near_ohlines = 0
except ValueError:
raise ValueError('wrong value: npix_removed_near_ohlines='
'{}'.format(
dict_rtas['npix_removed_near_ohlines']))
if 'list_valid_wvregions_a' in dict_rtas.keys():
if vpix_region_a_target is None:
raise ValueError('Unexpected list_valid_wvregions_a when '
'vpix_region_a_target is not set')
list_valid_wvregions_a = dict_rtas['list_valid_wvregions_a']
else:
list_valid_wvregions_a = None
if 'list_valid_wvregions_b' in dict_rtas.keys():
if vpix_region_b_target is None:
raise ValueError('Unexpected list_valid_wvregions_b when '
'vpix_region_b_target is not set')
list_valid_wvregions_b = dict_rtas['list_valid_wvregions_b']
else:
list_valid_wvregions_b = None
try:
nwidth_medfilt = int(dict_rtas['nwidth_medfilt'])
except KeyError:
nwidth_medfilt = 0
except ValueError:
raise ValueError('wrong value: nwidth_medfilt={}'.format(
dict_rtas['nwidth_medfilt']))
try:
save_individual_images = int(dict_rtas['save_individual_images'])
except KeyError:
save_individual_images = 0
except ValueError:
raise ValueError('wrong value: save_individual_images={}'.format(
dict_rtas['save_individual_images']))
self.logger.info(
'npix_removed_near_ohlines: {}'.format(npix_removed_near_ohlines))
self.logger.info('nwidth_medfilt: {}'.format(nwidth_medfilt))
self.logger.info(
'vpix_region_a_target: {}'.format(vpix_region_a_target))
self.logger.info(
'vpix_region_b_target: {}'.format(vpix_region_b_target))
self.logger.info(
'list_valid_wvregions_a: {}'.format(list_valid_wvregions_a))
self.logger.info(
'list_valid_wvregions_b: {}'.format(list_valid_wvregions_b))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# basic reduction, rectification and wavelength calibration of
# all the individual images
list_reduced_mos_images = []
self.logger.info('starting reduction of individual images')
for i, char in enumerate(full_set):
frame = rinput.obresult.frames[i]
self.logger.info('image {} ({} of {})'.format(
char, i + 1, nimages))
self.logger.info('image: {}'.format(frame.filename))
with frame.open() as f:
base_header = f[0].header.copy()
data = f[0].data.astype('float32')
grism_name_ = base_header['grism']
if grism_name_ != grism_name:
raise ValueError('Incompatible grism name in '
'rectwv_coeff.json file and FITS image')
filter_name_ = base_header['filter']
if filter_name_ != filter_name:
raise ValueError('Incompatible filter name in '
'rectwv_coeff.json file and FITS image')
hdu = fits.PrimaryHDU(data, header=base_header)
hdu.header['UUID'] = str(uuid.uuid1())
hdul = fits.HDUList([hdu])
# basic reduction
reduced_image = flow(hdul)
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# rectification and wavelength calibration
reduced_mos_image = apply_rectwv_coeff(reduced_image,
list_rectwv_coeff[i])
if save_individual_images != 0:
self.save_intermediate_img(
reduced_mos_image, 'reduced_mos_image_' + char + '_' +
frame.filename[:10] + '.fits')
list_reduced_mos_images.append(reduced_mos_image)
# intermediate PDF file with crosscorrelation plots
if self.intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation_ab.pdf')
else:
pdf = None
# compute offsets between images
self.logger.info('computing offsets between individual images')
first_a = True
first_b = True
xisok_a = None
xisok_b = None
reference_profile_a = None
reference_profile_b = None
refine_a = vpix_region_a_target is not None
refine_b = vpix_region_b_target is not None
list_offsets = []
for i, char in enumerate(full_set):
if char == 'A' and refine_a:
refine_image = True
elif char == 'B' and refine_b:
refine_image = True
else:
refine_image = False
if refine_image:
frame = rinput.obresult.frames[i]
isky = get_isky(i, pattern)
frame_sky = rinput.obresult.frames[isky]
self.logger.info('image {} ({} of {})'.format(
char, i + 1, nimages))
self.logger.info('image: {}'.format(frame.filename))
self.logger.info('(sky): {}'.format(frame_sky.filename))
data = list_reduced_mos_images[i][0].data.copy()
data_sky = list_reduced_mos_images[isky][0].data
data -= data_sky
base_header = list_reduced_mos_images[i][0].header
# get useful pixels in the wavelength direction
if char == 'A' and first_a:
xisok_a = useful_mos_xpixels(
data,
base_header,
vpix_region=vpix_region_a_target,
npix_removed_near_ohlines=npix_removed_near_ohlines,
list_valid_wvregions=list_valid_wvregions_a,
debugplot=0)
elif char == 'B' and first_b:
xisok_b = useful_mos_xpixels(
data,
base_header,
vpix_region=vpix_region_b_target,
npix_removed_near_ohlines=npix_removed_near_ohlines,
list_valid_wvregions=list_valid_wvregions_b,
debugplot=0)
if char == 'A':
nsmin = vpix_region_a_target[0]
nsmax = vpix_region_a_target[1]
xisok = xisok_a
if vpix_region_a_sky is not None:
nsmin_sky = vpix_region_a_sky[0]
nsmax_sky = vpix_region_a_sky[1]
skysubtraction = True
else:
nsmin_sky = None
nsmax_sky = None
skysubtraction = False
elif char == 'B':
nsmin = vpix_region_b_target[0]
nsmax = vpix_region_b_target[1]
xisok = xisok_b
if vpix_region_b_sky is not None:
nsmin_sky = vpix_region_b_sky[0]
nsmax_sky = vpix_region_b_sky[1]
skysubtraction = True
else:
nsmin_sky = None
nsmax_sky = None
skysubtraction = False
else:
raise ValueError('Unexpected char value: {}'.format(char))
# initial slitlet region
slitlet2d = data[(nsmin - 1):nsmax, :].copy()
if skysubtraction:
slitlet2d_sky = data[(nsmin_sky - 1):nsmax_sky, :].copy()
median_sky = np.median(slitlet2d_sky, axis=0)
slitlet2d -= median_sky
# selected wavelength regions after blocking OH lines
slitlet2d_blocked = slitlet2d[:, xisok]
# apply median filter in the X direction
if nwidth_medfilt > 1:
slitlet2d_blocked_smoothed = ndimage.filters.median_filter(
slitlet2d_blocked, size=(1, nwidth_medfilt))
else:
slitlet2d_blocked_smoothed = slitlet2d_blocked
# ---
# convert to 1D series of spatial profiles (note: this
# option does not work very well when the signal is low;
# a simple mean profile does a better job)
# profile = slitlet2d_blocked_smoothed.transpose().ravel()
# ---
profile = np.mean(slitlet2d_blocked_smoothed, axis=1)
if char == 'A' and first_a:
reference_profile_a = profile.copy()
elif char == 'B' and first_b:
reference_profile_b = profile.copy()
# ---
# # To write pickle file
# import pickle
# with open(frame.filename[:10] + '_profile.pkl', 'wb') as f:
# pickle.dump(profile, f)
# # To read pickle file
# with open('test.pkl', 'rb') as f:
# x = pickle.load(f)
# ---
# crosscorrelation to find offset
naround_zero = (nsmax - nsmin) // 3
if char == 'A':
reference_profile = reference_profile_a
elif char == 'B':
reference_profile = reference_profile_b
else:
raise ValueError('Unexpected char value: {}'.format(char))
offset, fpeak = periodic_corr1d(
sp_reference=reference_profile,
sp_offset=profile,
remove_mean=False,
frac_cosbell=0.10,
zero_padding=11,
fminmax=None,
nfit_peak=5,
naround_zero=naround_zero,
sp_label='spatial profile',
plottitle='Image #{} (type {}), {}'.format(
i + 1, char, frame.filename[:10]),
pdf=pdf)
# round to 4 decimal places
if abs(offset) < 1E-4:
offset = 0.0 # avoid -0.0
else:
offset = round(offset, 4)
list_offsets.append(offset)
# end of loop
if char == 'A' and first_a:
first_a = False
elif char == 'B' and first_b:
first_b = False
else:
list_offsets.append(0.0)
self.logger.info('computed offsets: {}'.format(list_offsets))
self.logger.info('correcting vertical offsets between individual '
'images')
list_a = []
list_b = []
for i, (char, offset) in enumerate(zip(full_set, list_offsets)):
frame = rinput.obresult.frames[i]
self.logger.info('image {} ({} of {})'.format(
char, i + 1, nimages))
self.logger.info('image: {}'.format(frame.filename))
reduced_mos_image = list_reduced_mos_images[i]
data = reduced_mos_image[0].data
base_header = reduced_mos_image[0].header
self.logger.info(
'correcting vertical offset (pixesl): {}'.format(offset))
if offset != 0:
reduced_mos_image[0].data = shift_image2d(
data, yoffset=-offset).astype('float32')
base_header['HISTORY'] = 'Applying voffset_pix {}'.format(offset)
if save_individual_images != 0:
self.save_intermediate_img(
reduced_mos_image, 'reduced_mos_image_refined_' + char +
'_' + frame.filename[:10] + '.fits')
# store reduced_mos_image
if char == 'A':
list_a.append(reduced_mos_image)
elif char == 'B':
list_b.append(reduced_mos_image)
else:
raise ValueError('Unexpected char value: {}'.format(char))
# combination method
method = getattr(combine, rinput.method)
method_kwargs = rinput.method_kwargs
# final combination of A images
self.logger.info('combining individual A images')
reduced_mos_image_a = combine_imgs(list_a,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
self.save_intermediate_img(reduced_mos_image_a,
'reduced_mos_image_a.fits')
# final combination of B images
self.logger.info('combining individual B images')
reduced_mos_image_b = combine_imgs(list_b,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
self.save_intermediate_img(reduced_mos_image_b,
'reduced_mos_image_b.fits')
self.logger.info('mixing A and B spectra')
header_a = reduced_mos_image_a[0].header
header_b = reduced_mos_image_b[0].header
data_a = reduced_mos_image_a[0].data.astype('float32')
data_b = reduced_mos_image_b[0].data.astype('float32')
reduced_mos_abba_data = data_a - data_b
# update reduced mos image header
reduced_mos_abba = self.create_mos_abba_image(rinput,
dict_rtas,
reduced_mos_abba_data,
header_a,
header_b,
pattern,
full_set,
list_offsets,
voffset_pix=0)
# combine A and B data by shifting B on top of A
if abs(ab_different_target) == 0:
len_prof_a = len(reference_profile_a)
len_prof_b = len(reference_profile_b)
if len_prof_a == len_prof_b:
reference_profile = reference_profile_a
profile = reference_profile_b
naround_zero = len_prof_a // 3
elif len_prof_a > len_prof_b:
ndiff = len_prof_a - len_prof_b
reference_profile = reference_profile_a
profile = np.concatenate(
(reference_profile_b, np.zeros(ndiff, dtype='float')))
naround_zero = len_prof_a // 3
else:
ndiff = len_prof_b - len_prof_a
reference_profile = np.concatenate(
(reference_profile_a, np.zeros(ndiff, dtype='float')))
profile = reference_profile_b
naround_zero = len_prof_b // 3
offset, fpeak = periodic_corr1d(
sp_reference=reference_profile,
sp_offset=profile,
remove_mean=False,
frac_cosbell=0.10,
zero_padding=11,
fminmax=None,
nfit_peak=5,
naround_zero=naround_zero,
sp_label='spatial profile',
plottitle='Comparison of A and B profiles',
pdf=pdf)
voffset_pix = vpix_region_b_target[0] - vpix_region_a_target[0]
voffset_pix += offset
elif abs(ab_different_target) == 1:
# apply nominal offset (from WCS info) between first A
# and first B
voffset_pix = sep_pixel[1] * ab_different_target
elif ab_different_target == 9:
voffset_pix = None
else:
raise ValueError(
'Invalid ab_different_target={}'.format(ab_different_target))
# close output PDF file
if pdf is not None:
pdf.close()
if voffset_pix is not None:
self.logger.info(
'correcting vertical offset (pixesl): {}'.format(voffset_pix))
shifted_a_minus_b_data = shift_image2d(
reduced_mos_abba_data,
yoffset=-voffset_pix,
).astype('float32')
reduced_mos_abba_combined_data = \
reduced_mos_abba_data - shifted_a_minus_b_data
# scale signal to exposure of a single image
reduced_mos_abba_combined_data /= 2.0
else:
reduced_mos_abba_combined_data = None
# update reduced mos combined image header
reduced_mos_abba_combined = self.create_mos_abba_image(
rinput, dict_rtas, reduced_mos_abba_combined_data, header_a,
header_b, pattern, full_set, list_offsets, voffset_pix)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(list_rectwv_coeff[0])
save_spectral_lines_ds9(list_rectwv_coeff[0])
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos_abba,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of ABBA spectra')
result = self.create_result(
reduced_mos_abba=reduced_mos_abba,
reduced_mos_abba_combined=reduced_mos_abba_combined)
return result
def create_mos_abba_image(self, rinput, dict_rtas, reduced_mos_abba_data,
header_a, header_b, pattern, full_set,
list_offsets, voffset_pix):
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
# Copy header of first image
base_header = hduls[0][0].header.copy()
hdu = fits.PrimaryHDU(reduced_mos_abba_data, header=base_header)
self.set_base_headers(hdu.header)
# check consistency of wavelength calibration paramenters
for param in ['crpix1', 'crval1', 'cdelt1']:
if header_a[param] != header_b[param]:
raise ValueError(
'Headers of A and B images have different '
'values of {}'.format(param))
self.logger.debug('update result header')
crpix1 = header_a['crpix1']
crval1 = header_a['crval1']
cdelt1 = header_a['cdelt1']
# update wavelength calibration in FITS header
for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']:
if keyword in hdu.header:
hdu.header.remove(keyword)
hdu.header['crpix1'] = (crpix1, 'reference pixel')
hdu.header['crval1'] = (crval1, 'central wavelength at crpix1')
hdu.header['cdelt1'] = (cdelt1,
'linear dispersion (Angstrom/pixel)')
hdu.header['cunit1'] = ('Angstrom', 'units along axis1')
hdu.header['ctype1'] = 'WAVELENGTH'
hdu.header['crpix2'] = (0.0, 'reference pixel')
hdu.header['crval2'] = (0.0, 'central value at crpix2')
hdu.header['cdelt2'] = (1.0, 'increment')
hdu.header['ctype2'] = 'PIXEL'
hdu.header['cunit2'] = ('Pixel', 'units along axis2')
for keyword in [
'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2',
'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2'
]:
if keyword in hdu.header:
hdu.header.remove(keyword)
# update additional keywords
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = pattern + ' pattern'
hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2']
hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames')
# update history
hdu.header['HISTORY'] = "Processed " + pattern + " pattern"
hdu.header['HISTORY'] = '--- Reduction of A images ---'
for line in header_a['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header['HISTORY'] = '--- Reduction of B images ---'
for line in header_b['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header['HISTORY'] = '--- Combination of ABBA images ---'
for key in dict_rtas:
hdu.header['HISTORY'] = '{}: {}'.format(key, dict_rtas[key])
dm = emirdrp.datamodel.EmirDataModel()
for img, key, offset in zip(hduls, full_set, list_offsets):
imgid = dm.get_imgid(img)
hdu.header['HISTORY'] = \
"Image '{}' is '{}', with voffset_pix {}".format(
imgid, key, offset)
if voffset_pix is not None and voffset_pix != 0:
hdu.header['HISTORY'] = '--- Combination of AB spectra ---'
hdu.header['HISTORY'] = "voffset_pix between A and B {}".format(
voffset_pix)
hdu.header.add_history('--- rinput.stored() (BEGIN) ---')
for item in rinput.stored():
value = getattr(rinput, item)
cline = '{}: {}'.format(item, value)
hdu.header.add_history(cline)
hdu.header.add_history('--- rinput.stored() (END) ---')
result = fits.HDUList([hdu])
return result
def set_base_headers(self, hdr):
newhdr = super(ABBASpectraRectwv, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
def extract_tags_from_obsres(self, obsres, tag_keys):
# this function is necessary to make use of recipe requirements
# that are provided as lists
final_tags = []
for frame in obsres.frames:
ref_img = frame.open()
tag = self.extract_tags_from_ref(ref_img,
tag_keys,
base=obsres.labels)
final_tags.append(tag)
return final_tags
class BarDetectionRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.NominalPositions,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
average_box_row_size = Parameter(
7, 'Number of rows to average for fine centering (odd)')
average_box_col_size = Parameter(
21, 'Number of columns to extract for fine centering (odd)')
fit_peak_npoints = Parameter(
3, 'Number of points to use for fitting the peak (odd)')
# Recipe Products
frame = Result(prods.ProcessedImage)
# derivative = Result(prods.ProcessedImage)
slits = Result(tarray.ArrayType)
positions3 = Result(tarray.ArrayType)
positions5 = Result(tarray.ArrayType)
positions7 = Result(tarray.ArrayType)
positions9 = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
TSUTC1 = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
def run(self, rinput):
self.logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(hdulist, 'reduced_image.fits')
try:
rotang = hdr['ROTANG']
tsutc1 = hdr['TSUTC1']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
if len(csupos) != 2 * EMIR_NBARS:
raise RecipeError('Number of CSUPOS != 2 * NBARS')
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('start finding bars')
allpos, slits = find_bars(
hdulist,
rinput.bars_nominal_positions,
csupos,
dtur,
average_box_row_size=rinput.average_box_row_size,
average_box_col_size=rinput.average_box_col_size,
fit_peak_npoints=rinput.fit_peak_npoints,
median_filter_size=rinput.median_filter_size,
logger=self.logger)
self.logger.debug('end finding bars')
if self.intermediate_results:
with open('ds9.reg', 'w') as ds9reg:
slits_to_ds9_reg(ds9reg, slits)
result = self.create_result(
frame=hdulist,
slits=slits,
positions9=allpos[9],
positions7=allpos[7],
positions5=allpos[5],
positions3=allpos[3],
DTU=dtub,
ROTANG=rotang,
TSUTC1=tsutc1,
csupos=csupos,
csusens=csusens,
)
return result
class BarDetectionRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.CoordinateList2DType,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
canny_high_threshold = Parameter(0.04,
'High threshold for the canny algorithm')
canny_low_threshold = Parameter(0.01,
'High threshold for the canny algorithm')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
param_median_filter_size = Result(float)
param_canny_high_threshold = Result(float)
param_canny_low_threshold = Result(float)
def run(self, rinput):
logger = logging.getLogger('numina.recipes.emir')
logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
logger.error(error)
raise numina.exceptions.RecipeError(error)
logger.debug('finding bars')
arr = hdulist[0].data
# Median filter
logger.debug('median filtering')
mfilter_size = rinput.median_filter_size
arr_median = median_filter(arr, size=mfilter_size)
# Image is mapped between 0 and 1
# for the full range [0: 2**16]
logger.debug('image scaling to 0-1')
arr_grey = normalize_raw(arr_median)
# Find borders
logger.debug('find borders')
canny_sigma = rinput.canny_sigma
# These threshols corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
edges = canny(arr_grey,
sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Number or rows used
# These other parameters cab be tuned also
total = 5
maxdist = 1.0
bstart = 100
bend = 1900
positions = []
nt = total // 2
xfac = dtur[0] / EMIR_PIXSCALE
yfac = -dtur[1] / EMIR_PIXSCALE
vec = [yfac, xfac]
logger.debug('DTU shift is %s', vec)
# Based on the 'edges image'
# and the table of approx positions of the slits
barstab = rinput.bars_nominal_positions
# Currently, we only use fields 0 and 2
# of the nominal positions file
for coords in barstab:
lbarid = int(coords[0])
rbarid = lbarid + 55
ref_y_coor = coords[2] + vec[1]
prow = coor_to_pix_1d(ref_y_coor) - 1
fits_row = prow + 1 # FITS pixel index
logger.debug('looking for bars with ids %d - %d', lbarid, rbarid)
logger.debug('reference y position is Y %7.2f', ref_y_coor)
# Find the position of each bar
bpos = find_position(edges, prow, bstart, bend, total)
nbars_found = len(bpos)
# If no bar is found, append and empty token
if nbars_found == 0:
logger.debug('bars %d, %d not found at row %d', lbarid, rbarid,
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
elif nbars_found == 2:
# Order values by increasing X
centl, centr = sorted(bpos, key=lambda cen: cen[0])
c1 = centl[0]
c2 = centr[0]
logger.debug('bars found at row %d between %7.2f - %7.2f',
fits_row, c1, c2)
# Compute FWHM of the collapsed profile
cslit = arr_grey[prow - nt:prow + nt + 1, :]
pslit = cslit.mean(axis=0)
# Add 1 to return FITS coordinates
epos, epos_f, error = locate_bar_l(pslit, c1)
thisres1 = lbarid, fits_row, epos + 1, epos_f + 1, error
epos, epos_f, error = locate_bar_r(pslit, c2)
thisres2 = rbarid, fits_row, epos + 1, epos_f + 1, error
elif nbars_found == 1:
logger.warning(
'only 1 edge found at row %d, not yet implemented',
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
else:
logger.warning(
'3 or more edges found at row %d, not yet implemented',
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
positions.append(thisres1)
positions.append(thisres2)
logger.debug('end finding bars')
result = self.create_result(
frame=hdulist,
positions=positions,
DTU=dtub,
ROTANG=rotang,
csupos=csupos,
csusens=csusens,
param_median_filter_size=rinput.median_filter_size,
param_canny_high_threshold=rinput.canny_high_threshold,
param_canny_low_threshold=rinput.canny_low_threshold)
return result
class TestSlitDetectionRecipe(EmirRecipe):
# Recipe Requirements
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm')
canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm')
# Recipe Results
frame = Result(prods.ProcessedImage)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
def run(self, rinput):
self.logger.info('starting slit processing')
self.logger.info('basic image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('finding slits')
# Filter values below 0.0
self.logger.debug('Filter values below 0')
data1 = hdulist[0].data[:]
data1[data1 < 0.0] = 0.0
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
self.logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize_raw(data2)
# Find edges with Canny
self.logger.debug('Find edges, Canny sigma %f', canny_sigma)
# These thresholds corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
self.logger.debug('Find edges, Canny high threshold %f', high_threshold)
self.logger.debug('Find edges, Canny low threshold %f', low_threshold)
edges = canny(img_grey, sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Fill edges
self.logger.debug('Fill holes')
# I do a dilation and erosion to fill
# possible holes in 'edges'
fill = ndimage.binary_dilation(edges)
fill2 = ndimage.binary_fill_holes(fill)
fill_slits = ndimage.binary_erosion(fill2)
self.logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
self.logger.debug('%d objects found', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
self.logger.debug('Find regions and centers')
regions = ndimage.find_objects(label_objects)
centers = ndimage.center_of_mass(data2, labels=label_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=-1.0
)
result = self.create_result(frame=hdulist,
slitstable=table,
DTU=dtub,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa
)
return result
class SpecFlatLowFreq(EmirRecipe):
"""Process continuum exposures of continuum lamp (lamp ON-OFF)
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True)
master_rectwv = reqs.MasterRectWaveRequirement(optional=True)
# note that 'sum' is not allowed as combination method
method = Parameter('sigmaclip',
description='Combination method',
choices=['mean', 'median', 'sigmaclip'])
method_kwargs = Parameter(dict(),
description='Arguments for combination method',
optional=True)
minimum_slitlet_width_mm = Parameter(
float(EMIR_MINIMUM_SLITLET_WIDTH_MM),
description='Minimum width (mm) for a valid slitlet',
optional=True)
maximum_slitlet_width_mm = Parameter(
float(EMIR_MAXIMUM_SLITLET_WIDTH_MM),
description='Maximum width (mm) for a valid slitlet',
optional=True)
global_integer_offset_x_pix = Parameter(
0,
description='Global offset (pixels) in wavelength direction (integer)',
optional=True)
global_integer_offset_y_pix = Parameter(
0,
description='Global offset (pixels) in spatial direction (integer)',
optional=True)
nwindow_x_median = Parameter(
301,
description='Window size to smooth the flatfield in the spectral '
'direction',
optional=True)
nwindow_y_median = Parameter(
11,
description='Window size to smooth the flatfield in the spatial '
'direction',
optional=True)
minimum_fraction = Parameter(
0.01,
description='Minimum useful flatfielding value (fraction of maximum)',
optional=True)
minimum_value_in_output = Parameter(
0.01,
description='Minimum value allowed in output: pixels below this value'
' are set to 1.0',
optional=True)
maximum_value_in_output = Parameter(
10.0,
description='Maximum value allowed in output: pixels above this value'
' are set to 1.0',
optional=True)
debugplot = Parameter(0, description='Debugger parameter', optional=True)
reduced_flatlowfreq = Result(prods.MasterSpectralFlat)
def run(self, rinput):
self.logger.info('starting generation of flatlowfreq')
self.logger.info('rectwv_coeff..........................: {}'.format(
rinput.rectwv_coeff))
self.logger.info('master_rectwv.........................: {}'.format(
rinput.master_rectwv))
self.logger.info('Minimum slitlet width (mm)............: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info('Maximum slitlet width (mm)............: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info('Global offset X direction (pixels)....: {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info('Global offset Y direction (pixels)....: {}'.format(
rinput.global_integer_offset_y_pix))
self.logger.info('nwindow_x_median......................: {}'.format(
rinput.nwindow_x_median))
self.logger.info('nwindow_y_median......................: {}'.format(
rinput.nwindow_y_median))
self.logger.info('Minimum fraction......................: {}'.format(
rinput.minimum_fraction))
self.logger.info('Minimum value in output...............: {}'.format(
rinput.minimum_value_in_output))
self.logger.info('Maximum value in output...............: {}'.format(
rinput.maximum_value_in_output))
# check rectification and wavelength calibration information
if rinput.master_rectwv is None and rinput.rectwv_coeff is None:
raise ValueError('No master_rectwv nor rectwv_coeff data have '
'been provided')
elif rinput.master_rectwv is not None and \
rinput.rectwv_coeff is not None:
self.logger.warning('rectwv_coeff will be used instead of '
'master_rectwv')
if rinput.rectwv_coeff is not None and \
(rinput.global_integer_offset_x_pix != 0 or
rinput.global_integer_offset_y_pix != 0):
raise ValueError('global_integer_offsets cannot be used '
'simultaneously with rectwv_coeff')
# check headers to detect lamp status (on/off)
list_lampincd = []
for fname in rinput.obresult.frames:
with fname.open() as f:
list_lampincd.append(f[0].header['lampincd'])
# check number of images
nimages = len(rinput.obresult.frames)
n_on = list_lampincd.count(1)
n_off = list_lampincd.count(0)
self.logger.info(
'Number of images with lamp ON.........: {}'.format(n_on))
self.logger.info(
'Number of images with lamp OFF........: {}'.format(n_off))
self.logger.info(
'Total number of images................: {}'.format(nimages))
if n_on == 0:
raise ValueError('Insufficient number of images with lamp ON')
if n_on + n_off != nimages:
raise ValueError('Number of images does not match!')
# check combination method
if rinput.method_kwargs == {}:
method_kwargs = None
else:
if rinput.method == 'sigmaclip':
method_kwargs = rinput.method_kwargs
else:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# build object to proceed with bpm, bias, and dark (not flat)
flow = self.init_filters(rinput)
# available combination methods
method = getattr(combine, rinput.method)
# basic reduction of images with lamp ON or OFF
lampmode = {0: 'off', 1: 'on'}
reduced_image_on = None
reduced_image_off = None
for imode in lampmode.keys():
self.logger.info('starting basic reduction of images with'
' lamp {}'.format(lampmode[imode]))
tmplist = [
rinput.obresult.frames[i]
for i, lampincd in enumerate(list_lampincd)
if lampincd == imode
]
if len(tmplist) > 0:
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open()) for fname in tmplist
]
reduced_image = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
if imode == 0:
reduced_image_off = flow(reduced_image)
hdr = reduced_image_off[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_off,
'reduced_image_off.fits')
elif imode == 1:
reduced_image_on = flow(reduced_image)
hdr = reduced_image_on[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_on,
'reduced_image_on.fits')
else:
raise ValueError('Unexpected imode={}'.format(imode))
# computation of ON-OFF
header_on = reduced_image_on[0].header
data_on = reduced_image_on[0].data.astype('float32')
if n_off > 0:
header_off = reduced_image_off[0].header
data_off = reduced_image_off[0].data.astype('float32')
else:
header_off = None
data_off = np.zeros_like(data_on)
reduced_data = data_on - data_off
# update reduced image header
reduced_image = self.create_reduced_image(rinput, reduced_data,
header_on, header_off,
list_lampincd)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# define rectification and wavelength calibration coefficients
if rinput.rectwv_coeff is None:
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image, rinput.master_rectwv)
# set global offsets
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
else:
rectwv_coeff = rinput.rectwv_coeff
# save as JSON in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# apply global offsets (to both, the original and the cleaned version)
image2d = apply_integer_offsets(
image2d=reduced_data,
offx=rectwv_coeff.global_integer_offset_x_pix,
offy=rectwv_coeff.global_integer_offset_y_pix)
# load CSU configuration
csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header)
# determine (pseudo) longslits
dict_longslits = csu_conf.pseudo_longslits()
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in rectwv_coeff.missing_slitlets:
self.logger.info('-> Removing slitlet (not defined): ' + str(idel))
list_valid_islitlets.remove(idel)
# filter out slitlets with widths outside valid range
list_outside_valid_width = []
for islitlet in list_valid_islitlets:
slitwidth = csu_conf.csu_bar_slit_width(islitlet)
if (slitwidth < rinput.minimum_slitlet_width_mm) or \
(slitwidth > rinput.maximum_slitlet_width_mm):
list_outside_valid_width.append(islitlet)
self.logger.info('-> Removing slitlet (width out of range): ' +
str(islitlet))
if len(list_outside_valid_width) > 0:
for idel in list_outside_valid_width:
list_valid_islitlets.remove(idel)
# initialize rectified image
image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1))
# main loop
grism_name = rectwv_coeff.tags['grism']
filter_name = rectwv_coeff.tags['filter']
cout = '0'
debugplot = rinput.debugplot
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=debugplot)
if abs(slt.debugplot) > 10:
print(slt)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d,
subtitle='original image')
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=2,
subtitle='original (cleaned) rectified')
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
if naxis1_slitlet2d != EMIR_NAXIS1:
print('naxis1_slitlet2d: ', naxis1_slitlet2d)
print('EMIR_NAXIS1.....: ', EMIR_NAXIS1)
raise ValueError("Unexpected naxis1_slitlet2d")
# get useful slitlet region (use boundaries)
spectrail = slt.list_spectrails[0]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig
spectrail = slt.list_spectrails[2]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig
# median spectrum
sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :],
axis=0)
# smooth median spectrum along the spectral direction
sp_median = ndimage.median_filter(sp_collapsed,
5,
mode='nearest')
ymax_spmedian = sp_median.max()
y_threshold = ymax_spmedian * rinput.minimum_fraction
lremove = np.where(sp_median < y_threshold)
sp_median[lremove] = 0.0
if abs(slt.debugplot) % 10 != 0:
xaxis1 = np.arange(1, naxis1_slitlet2d + 1)
title = 'Slitlet#' + str(islitlet) + ' (median spectrum)'
ax = ximplotxy(xaxis1,
sp_collapsed,
title=title,
show=False,
**{'label': 'collapsed spectrum'})
ax.plot(xaxis1, sp_median, label='fitted spectrum')
ax.plot([1, naxis1_slitlet2d],
2 * [y_threshold],
label='threshold')
# ax.plot(xknots, yknots, 'o', label='knots')
ax.legend()
ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian)
pause_debugplot(slt.debugplot,
pltshow=True,
tight_layout=True)
# generate rectified slitlet region filled with the
# median spectrum
slitlet2d_rect_spmedian = np.tile(sp_median,
(naxis2_slitlet2d, 1))
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_rectified(
slitlet2d_rect=slitlet2d_rect_spmedian,
subtitle='rectified, filled with median spectrum')
# compute smooth surface
# clipped region
slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy()
slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0
slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0
# unrectified clipped image
slitlet2d_unrect_clipped = slt.rectify(
slitlet2d=slitlet2d_rect_clipped,
resampling=2,
inverse=True,
subtitle='unrectified, filled with median spectrum '
'(clipped)')
# normalize initial slitlet image (avoid division by zero)
slitlet2d_norm_clipped = np.zeros_like(slitlet2d)
for j in range(naxis1_slitlet2d):
for i in range(naxis2_slitlet2d):
den = slitlet2d_unrect_clipped[i, j]
if den == 0:
slitlet2d_norm_clipped[i, j] = 1.0
else:
slitlet2d_norm_clipped[i, j] = \
slitlet2d[i, j] / den
# set to 1.0 one additional pixel at each side (since
# 'den' above is small at the borders and generates wrong
# bright pixels)
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_smooth = ndimage.median_filter(
slitlet2d_norm_clipped,
size=(rinput.nwindow_y_median, rinput.nwindow_x_median),
mode='nearest')
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_clipped,
subtitle='unrectified, pixel-to-pixel (clipped)')
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_smooth,
subtitle='unrectified, pixel-to-pixel (smoothed)')
# ---
# check for (pseudo) longslit with previous and next slitlet
imin = dict_longslits[islitlet].imin()
imax = dict_longslits[islitlet].imax()
if islitlet > 1:
same_slitlet_below = (islitlet - 1) >= imin
else:
same_slitlet_below = False
if islitlet < EMIR_NBARS:
same_slitlet_above = (islitlet + 1) <= imax
else:
same_slitlet_above = False
for j in range(EMIR_NAXIS1):
xchannel = j + 1
y0_lower = slt.list_frontiers[0](xchannel)
y0_upper = slt.list_frontiers[1](xchannel)
n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower,
y0_frontier_upper=y0_upper,
resize=True)
# note that n1 and n2 are scans (ranging from 1 to NAXIS2)
nn1 = n1 - slt.bb_ns1_orig + 1
nn2 = n2 - slt.bb_ns1_orig + 1
image2d_flatfielded[(n1 - 1):n2, j] = \
slitlet2d_norm_smooth[(nn1 - 1):nn2, j]
# force to 1.0 region around frontiers
if not same_slitlet_below:
image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1
if not same_slitlet_above:
image2d_flatfielded[(n2 - 5):n2, j] = 1
cout += '.'
else:
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
self.logger.info(cout)
# restore global offsets
image2d_flatfielded = apply_integer_offsets(
image2d=image2d_flatfielded,
offx=-rectwv_coeff.global_integer_offset_x_pix,
offy=-rectwv_coeff.global_integer_offset_y_pix)
# set pixels below minimum value to 1.0
filtered = np.where(
image2d_flatfielded < rinput.minimum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# set pixels above maximum value to 1.0
filtered = np.where(
image2d_flatfielded > rinput.maximum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# update image header
reduced_flatlowfreq = self.create_reduced_image(
rinput, image2d_flatfielded, header_on, header_off, list_lampincd)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# save results in results directory
self.logger.info('end of flatlowfreq generation')
result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq)
return result
def create_reduced_image(self, rinput, reduced_data, header_on, header_off,
list_lampincd):
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
# Copy header of first image
base_header = hduls[0][0].header.copy()
hdu = fits.PrimaryHDU(reduced_data, header=base_header)
self.set_base_headers(hdu.header)
self.logger.debug('update result header')
# update additional keywords
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = 'flatlowfreq'
hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2']
hdu.header['history'] = "Processed flatlowfreq"
hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames')
# update history
dm = emirdrp.datamodel.EmirDataModel()
for img, lampincd in zip(hduls, list_lampincd):
imgid = dm.get_imgid(img)
hdu.header['HISTORY'] = "Image '{}' has lampincd='{}'".format(
imgid, lampincd)
hdu.header['HISTORY'] = "Processed flatlowfreq"
hdu.header['HISTORY'] = '--- Reduction of images with lamp ON ---'
for line in header_on['HISTORY']:
hdu.header['HISTORY'] = line
if header_off is not None:
hdu.header['HISTORY'] = '--- Reduction of images with lamp OFF ---'
for line in header_off['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header.add_history('--- rinput.stored() (BEGIN) ---')
for item in rinput.stored():
value = getattr(rinput, item)
cline = '{}: {}'.format(item, value)
hdu.header.add_history(cline)
hdu.header.add_history('--- rinput.stored() (END) ---')
result = fits.HDUList([hdu])
return result
def set_base_headers(self, hdr):
newhdr = super(SpecFlatLowFreq, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class MaskSpectraExtractionRecipe(EmirRecipe):
'''
'''
# Recipe Requirements
obresult = ObservationResultRequirement()
master_bpm = MasterBadPixelMaskRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
master_flat = MasterIntensityFlatFieldRequirement()
master_sky = MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
slits_positions = Requirement(ArrayType,
'Positions and widths of the slits')
frame = Product(DataFrameType)
rss = Product(DataFrameType)
regions = Product(ArrayType)
#slitstable = Product(ArrayType)
#DTU = Product(ArrayType)
#ROTANG = Product(float)
#DETPA = Product(float)
#DTUPA = Product(float)
def run(self, rinput):
_logger.info('starting extraction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
# First, prefilter with median
median_filter_size = rinput.median_filter_size
data1 = hdulist[0].data
_logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Normalize input between -1 and +1
data3 = img_norm(data2)
# Tracing parameters
ws = 10
step = 15
hs = 15
tol = 2
doplot = False
npol = 5
_logger.info('Create output images')
rssdata = numpy.zeros(
(rinput.slits_positions.shape[0], data3.shape[1]), dtype='float32')
# FIXME, number of columns depends on polynomial degree
regiontable = numpy.zeros(
(rinput.slits_positions.shape[0], 4 + 2 * (npol + 1)),
dtype='float32')
count = 0
# Loop over slits
for slit_coords in rinput.slits_positions:
col, y1, y2 = convert_to_(*slit_coords)
_logger.info('Processing slit in column %i, row1=%i, row2=%i', col,
y1, y2)
xmin, xmax, ymin, ymax, pfit1, pfit2 = ex_region(data3,
col,
y1,
y2,
step,
hs,
ws,
tol=tol,
doplot=doplot)
_logger.info('Spectrum region is %i, %i, %i, %i', xmin, xmax, ymin,
ymax)
try:
region = data1[ymin:ymax + 1, xmin:xmax + 1]
rssdata[count, xmin:xmax + 1] = region.mean(axis=0)
except ValueError as err:
_logger.error("Error collapsing spectrum: %s", err)
# IN FITS convention
_logger.info('Create regions table')
regiontable[count, :4] = xmin + 1, xmax + 1, ymin + 1, ymax + 1
#regiontable[count, 4:4 + npol + 1] = pfit1
#regiontable[count, 4 + npol + 1:] = pfit2
count += 1
hdurss = fits.PrimaryHDU(rssdata)
result = self.create_result(frame=hdulist,
rss=hdurss,
regions=regiontable)
return result
class TestMaskRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = ObservationResultRequirement()
master_bpm = MasterBadPixelMaskRequirement()
master_bias = MasterBiasRequirement()
master_dark = MasterDarkRequirement()
master_flat = MasterIntensityFlatFieldRequirement()
master_sky = MasterSkyRequirement()
pinhole_nominal_positions = Requirement(CoordinateList2DType,
'Nominal positions of the pinholes'
)
shift_coordinates = Parameter(True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
obj_min_size = Parameter(200, 'Minimum size of the slit')
obj_max_size = Parameter(3000, 'Maximum size of the slit')
slit_size_ratio = Parameter(4.0, 'Minimum ratio between height and width for slits')
# Recipe Products
frame = Product(DataFrameType)
positions = Product(ArrayType)
positions_alt = Product(ArrayType)
slitstable = Product(ArrayType)
DTU = Product(ArrayType)
filter = Product(str)
readmode = Product(str)
ROTANG = Product(float)
DETPA = Product(float)
DTUPA = Product(float)
param_recenter = Product(bool)
param_max_recenter_radius = Product(float)
param_box_half_size = Product(float)
def run(self, rinput):
_logger.info('starting processing for slit detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.debug('finding pinholes')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
_logger.error(error)
raise RecipeError(error)
if rinput.shift_coordinates:
xdtur, ydtur, zdtur = dtur
xfac = xdtur / EMIR_PIXSCALE
yfac = -ydtur / EMIR_PIXSCALE
vec = numpy.array([yfac, xfac])
_logger.info('shift is %s', vec)
ncenters = rinput.pinhole_nominal_positions + vec
else:
_logger.info('using pinhole coordinates as they are')
ncenters = rinput.pinhole_nominal_positions
_logger.info('pinhole characterization')
positions = pinhole_char(
hdulist[0].data,
ncenters,
box=rinput.box_half_size,
recenter_pinhole=rinput.recenter,
maxdist=rinput.max_recenter_radius
)
_logger.info('alternate pinhole characterization')
positions_alt = pinhole_char2(
hdulist[0].data, ncenters,
recenter_pinhole=rinput.recenter,
recenter_half_box=rinput.box_half_size,
recenter_maxdist=rinput.max_recenter_radius
)
_logger.debug('finding slits')
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
obj_min_size = rinput.obj_min_size
obj_max_size = rinput.obj_max_size
data1 = hdulist[0].data
_logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize(data2)
# Find edges with canny
_logger.debug('Find edges with canny, sigma %d', canny_sigma)
edges = canny(img_grey, sigma=canny_sigma)
# Fill edges
_logger.debug('Fill holes')
fill_slits = ndimage.binary_fill_holes(edges)
_logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
_logger.debug('%d objects found', nb_labels)
# Filter on the area of the labeled region
# Perhaps we could ignore this filtering and
# do it later?
_logger.debug('Filter objects by size')
# Sizes of regions
sizes = numpy.bincount(label_objects.ravel())
_logger.debug('Min size is %d', obj_min_size)
_logger.debug('Max size is %d', obj_max_size)
mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size)
# Filter out regions
nids, = numpy.where(mask_sizes)
mm = numpy.in1d(label_objects, nids)
mm.shape = label_objects.shape
fill_slits_clean = numpy.where(mm, 1, 0)
# and relabel
_logger.debug('Label filtered objects')
relabel_objects, nb_labels = ndimage.label(fill_slits_clean)
_logger.debug('%d objects found after filtering', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
_logger.debug('Find regions and centers')
regions = ndimage.find_objects(relabel_objects)
centers = ndimage.center_of_mass(data2, labels=relabel_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=rinput.slit_size_ratio
)
result = self.create_result(frame=hdulist,
positions=positions,
positions_alt=positions_alt,
slitstable=table,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size
)
return result
class MaskImagingRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.CoordinateList2DType,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
average_box_row_size = Parameter(
7, 'Number of rows to average for fine centering (odd)')
average_box_col_size = Parameter(
21, 'Number of columns to extract for fine centering (odd)')
fit_peak_npoints = Parameter(
3, 'Number of points to use for fitting the peak (odd)')
# Recipe Products
frame = Result(prods.ProcessedImage)
# derivative = Result(prods.ProcessedImage)
slits = Result(tarray.ArrayType)
positions3 = Result(tarray.ArrayType)
positions5 = Result(tarray.ArrayType)
positions7 = Result(tarray.ArrayType)
positions9 = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
TSUTC1 = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
def run(self, rinput):
self.logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
tsutc1 = hdr['TSUTC1']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise numina.exceptions.RecipeError(error)
self.logger.debug('finding bars')
# Processed array
arr = hdulist[0].data
# Median filter of processed array (two times)
mfilter_size = rinput.median_filter_size
self.logger.debug('median filtering X, %d columns', mfilter_size)
arr_median = median_filter(arr, size=(1, mfilter_size))
self.logger.debug('median filtering X, %d rows', mfilter_size)
arr_median = median_filter(arr_median, size=(mfilter_size, 1))
# Median filter of processed array (two times) in the other direction
# for Y coordinates
self.logger.debug('median filtering Y, %d rows', mfilter_size)
arr_median_alt = median_filter(arr, size=(mfilter_size, 1))
self.logger.debug('median filtering Y, %d columns', mfilter_size)
arr_median_alt = median_filter(arr_median_alt, size=(1, mfilter_size))
xfac = dtur[0] / EMIR_PIXSCALE
yfac = -dtur[1] / EMIR_PIXSCALE
vec = [yfac, xfac]
self.logger.debug('DTU shift is %s', vec)
# and the table of approx positions of the slits
barstab = rinput.bars_nominal_positions
# Currently, we only use fields 0 and 2
# of the nominal positions file
# Number or rows used
# These other parameters cab be tuned also
bstart = 1
bend = 2047
self.logger.debug('ignoring columns outside %d - %d', bstart, bend - 1)
# extract a region to average
wy = (rinput.average_box_row_size // 2)
wx = (rinput.average_box_col_size // 2)
self.logger.debug('extraction window is %d rows, %d cols', 2 * wy + 1,
2 * wx + 1)
# Fit the peak with these points
wfit = 2 * (rinput.fit_peak_npoints // 2) + 1
self.logger.debug('fit with %d points', wfit)
# Minimum threshold
threshold = 5 * EMIR_RON
# Savitsky and Golay (1964) filter to compute the X derivative
# scipy >= xx has a savgol_filter function
# for compatibility we do it manually
allpos = {}
ypos3_kernel = None
slits = numpy.zeros((EMIR_NBARS, 8), dtype='float')
self.logger.info('start finding bars')
for ks in [3, 5, 7, 9]:
self.logger.debug('kernel size is %d', ks)
# S and G kernel for derivative
kw = ks * (ks * ks - 1) / 12.0
coeffs_are = -numpy.arange((1 - ks) // 2, (ks - 1) // 2 + 1) / kw
if ks == 3:
ypos3_kernel = coeffs_are
self.logger.debug('kernel weights are %s', coeffs_are)
self.logger.debug('derive image in X direction')
arr_deriv = convolve1d(arr_median, coeffs_are, axis=-1)
# Axis 0 is
#
self.logger.debug('derive image in Y direction (with kernel=3)')
arr_deriv_alt = convolve1d(arr_median_alt, ypos3_kernel, axis=0)
positions = []
for coords in barstab:
lbarid = int(coords[0])
rbarid = lbarid + EMIR_NBARS
ref_y_coor = coords[1] + vec[1]
poly_coeffs = coords[2:]
prow = coor_to_pix_1d(ref_y_coor) - 1
fits_row = prow + 1 # FITS pixel index
# A function that returns the center of the bar
# given its X position
def center_of_bar(x):
# Pixel values are 0-based
return polyval(x + 1 - vec[0], poly_coeffs) + vec[1] - 1
self.logger.debug('looking for bars with ids %d - %d', lbarid,
rbarid)
self.logger.debug('reference y position is Y %7.2f',
ref_y_coor)
# if ref_y_coor is outlimits, skip this bar
# ref_y_coor is in FITS format
if (ref_y_coor >= 2047) or (ref_y_coor <= 1):
self.logger.debug(
'reference y position is outlimits, skipping')
positions.append([lbarid, fits_row, fits_row, 1, 0, 3])
positions.append([rbarid, fits_row, fits_row, 1, 0, 3])
continue
# Left bar
self.logger.debug('measure left border (%d)', lbarid)
centery, xpos, fwhm, st = char_bar_peak_l(arr_deriv,
prow,
bstart,
bend,
threshold,
center_of_bar,
wx=wx,
wy=wy,
wfit=wfit)
xpos1 = xpos
positions.append(
[lbarid, centery + 1, fits_row, xpos + 1, fwhm, st])
# Right bar
self.logger.debug('measure rigth border (%d)', rbarid)
centery, xpos, fwhm, st = char_bar_peak_r(arr_deriv,
prow,
bstart,
bend,
threshold,
center_of_bar,
wx=wx,
wy=wy,
wfit=wfit)
positions.append(
[rbarid, centery + 1, fits_row, xpos + 1, fwhm, st])
xpos2 = xpos
#
if st == 0:
self.logger.debug('measure top-bottom borders')
try:
y1, y2, statusy = char_bar_height(arr_deriv_alt,
xpos1,
xpos2,
centery,
threshold,
wh=35,
wfit=wfit)
except Exception as error:
self.logger.warning('Error computing height: %s',
error)
statusy = 44
if statusy in [0, 40]:
# Main border is detected
positions[-1][1] = y2 + 1
positions[-2][1] = y2 + 1
else:
# Update status
positions[-1][-1] = 4
positions[-2][-1] = 4
else:
self.logger.debug('slit is not complete')
y1, y2 = 0, 0
# Update positions
self.logger.debug(
'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d',
*positions[-2])
self.logger.debug(
'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d',
*positions[-1])
if ks == 5:
slits[lbarid -
1] = [xpos1, y2, xpos2, y2, xpos2, y1, xpos1, y1]
# FITS coordinates
slits[lbarid - 1] += 1.0
self.logger.debug('inserting bars %d-%d into "slits"',
lbarid, rbarid)
allpos[ks] = numpy.asarray(
positions, dtype='float') # GCS doesn't like lists of lists
self.logger.debug('end finding bars')
result = self.create_result(
frame=hdulist,
slits=slits,
positions9=allpos[9],
positions7=allpos[7],
positions5=allpos[5],
positions3=allpos[3],
DTU=dtub,
ROTANG=rotang,
TSUTC1=tsutc1,
csupos=csupos,
csusens=csusens,
)
return result
