class SimpleSkyRecipe(EmirRecipe):
"""Recipe to process data taken in intensity flat-field mode.
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
skyframe = Result(prods.MasterSky)
def run(self, rinput):
_logger.info('starting sky reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput,
flow,
method=median,
errors=True)
hdr = hdulist[0].header
self.set_base_headers(hdr)
result = self.create_result(skyframe=hdulist)
return result
class StareSpectraRecipe(EmirRecipe):
"""Process images in Stare spectra mode"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
master_sky = reqs.SpectralSkyRequirement(optional=True)
stare = Result(prods.ProcessedMOS)
def run(self, rinput):
self.logger.info('starting stare spectra reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput,
flow,
method=median)
hdr = hdulist[0].header
self.set_base_headers(hdr)
# Update EXP to 0
hdr['EXP'] = 0
self.logger.info('end stare spectra reduction')
result = self.create_result(stare=hdulist)
return result
class DitherSkyRecipe(EmirRecipe):
"""Recipe to process data taken in dither sky mode.
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
skyframe = Result(prods.MasterSky)
def run(self, rinput):
_logger.debug('instrument %s, mode %s', rinput.obresult.instrument,
rinput.obresult.mode)
_logger.info('starting sky reduction with dither')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_segmentation(rinput,
flow,
method=median,
errors=True)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.info('end sky reduction with dither')
result = self.create_result(skyframe=hdulist)
return result
class TestSkyCorrectRecipe(EmirRecipe):
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = Requirement(prods.MasterIntensityFlat,
'Master Sky calibration')
frame = Result(prods.ProcessedImage)
def run(self, rinput):
self.logger.info('starting simple sky reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput,
flow,
method=median)
hdr = hdulist[0].header
self.set_base_headers(hdr)
# Update SEC to 0
hdr['SEC'] = 0
result = self.create_result(frame=hdulist)
return result
class BiasRecipe(EmirRecipe):
"""
Recipe to process data taken in Bias image Mode.
Bias images only appear in Simple Readout mode.
**Observing modes:**
* Bias Image (3.1)
**Inputs:**
**Outputs:**
* A combined bias frame, with variance extension.
* Statistics of the final image per channel (mean, median, variance)
**Procedure:**
The list of images can be readly processed by
combining them with a median algorithm.
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
biasframe = Result(prods.MasterBias)
def run(self, rinput):
_logger.info('starting bias reduction')
iinfo = gather_info_frames(rinput.obresult.frames)
if iinfo:
mode = iinfo[0]['readmode']
if mode.lower() not in EMIR_BIAS_MODES:
msg = 'readmode %s, is not a bias mode' % mode
_logger.error(msg)
raise RecipeError(msg)
flow = lambda x: x
hdulist = basic_processing_with_combination(rinput,
flow,
method=median,
errors=False)
pdata = hdulist[0].data
# update hdu header with
# reduction keywords
hdr = hdulist[0].header
self.set_base_headers(hdr)
hdr['CCDMEAN'] = pdata.mean()
_logger.info('bias reduction ended')
result = self.create_result(biasframe=DataFrame(hdulist))
return result
class IntensityFlatRecipe(EmirRecipe):
"""Recipe to process data taken in intensity flat-field mode.
Recipe to process intensity flat-fields. The flat-on and
flat-off images are combined (method?) separately and the subtracted
to obtain a thermal subtracted flat-field.
**Observing modes:**
* Intensity Flat-Field
**Inputs:**
* A master dark frame
* Non linearity
* A model of the detector.
**Outputs:**
* TBD
**Procedure:**
* A combined thermal subtracted flat field, normalized to median 1,
with with variance extension and quality flag.
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
flatframe = Result(prods.MasterIntensityFlat)
def run(self, rinput):
_logger.info('starting flat reduction')
errors = True
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput,
flow,
method=median,
errors=errors)
hdr = hdulist[0].header
self.set_base_headers(hdr)
mm = hdulist[0].data.mean()
hdr['CCDMEAN'] = mm
hdulist[0].data /= mm
if errors:
hdulist['variance'].data /= (mm * mm)
result = self.create_result(flatframe=hdulist)
return result
class SkySpecRecipe(EmirRecipe):
"""Recipe to process data taken in spectral sky mode.
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
master_rectwv = reqs.MasterRectWaveRequirement()
skyspec = Result(prods.SkySpectrum)
reduced_image = Result(prods.ProcessedMOS)
def run(self, rinput):
self.logger.info('starting spectral sky reduction')
flow = self.init_filters(rinput)
reduced_image = basic_processing_with_combination(
rinput, flow,
method=median,
errors=True
)
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object with rectification and wavelength calibration
# coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image,
rinput.master_rectwv
)
# save as JSON file in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# generate associated ds9 region files and save them in work directory
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
# apply rectification and wavelength calibration
skyspec = apply_rectwv_coeff(
reduced_image,
rectwv_coeff
)
self.logger.info('end sky spectral reduction')
result = self.create_result(
reduced_image=reduced_image,
skyspec=skyspec
)
return result
class StareImageRecipe2(EmirRecipe):
"""Process images in Stare Image Mode"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
result_image = Result(prods.ProcessedImage)
def run(self, rinput):
self.logger.info('starting stare image reduction (offline)')
frames = rinput.obresult.frames
datamodel = self.datamodel
with extra.manage_frame(frames) as list_of:
c_img = extra.combine_frames(list_of, datamodel, method=combine.mean, errors=False)
flow = self.init_filters(rinput)
# Correct Bias if needed
# Correct Dark if needed
# Correct FF
processed_img = flow(c_img)
hdr = processed_img[0].header
self.set_base_headers(hdr)
self.logger.debug('append BPM')
if rinput.master_bpm is not None:
self.logger.debug('using BPM from inputs')
hdul_bpm = rinput.master_bpm.open()
hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0])
else:
self.logger.debug('using empty BPM')
hdu_bpm = extra.generate_empty_bpm_hdu(processed_img[0])
# Append the BPM to the result
processed_img.append(hdu_bpm)
self.logger.info('end stare image (off) reduction')
result = self.create_result(result_image=processed_img)
return result
def set_base_headers(self, hdr):
"""Set metadata in FITS headers."""
hdr = super(StareImageRecipe2, self).set_base_headers(hdr)
# Set EXP to 0
hdr['EXP'] = 0
return hdr
class DarkRecipe(EmirRecipe):
"""Recipe to process data taken in Dark current image Mode.
Recipe to process dark images. The dark images will be combined
using the median.
They do have to be of the same exposure time t.
**Observing mode:**
* Dark current Image (3.2)
**Inputs:**
**Outputs:**
* A combined dark frame, with variance extension.
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
master_bias = reqs.MasterBiasRequirement()
darkframe = Result(prods.MasterDark)
def run(self, rinput):
_logger.info('starting dark reduction')
flow = self.init_filters(rinput)
iinfo = gather_info_frames(rinput.obresult.frames)
ref_exptime = 0.0
for el in iinfo[1:]:
if abs(el['texp'] - ref_exptime) > 1e-4:
_logger.error('image with wrong exposure time')
raise RecipeError('image with wrong exposure time')
hdulist = basic_processing_with_combination(rinput,
flow,
method=median,
errors=True)
pdata = hdulist[0].data
# update hdu header with
# reduction keywords
hdr = hdulist[0].header
self.set_base_headers(hdr)
hdr['CCDMEAN'] = pdata.mean()
_logger.info('dark reduction ended')
result = self.create_result(darkframe=hdulist)
return result
class IntensityFlatRecipe2(EmirRecipe):
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flatframe = Result(prods.MasterIntensityFlat)
def run(self, rinput):
import emirdrp.core.extra as extra
from numina.array import combine
_logger.info('starting flat reduction')
frames = rinput.obresult.frames
datamodel = self.datamodel
with extra.manage_frame(frames) as list_of:
c_img = extra.combine_frames2(list_of,
datamodel,
method=combine.mean,
errors=False)
self.save_intermediate_img(c_img, 'p0.fits')
flow = self.init_filters(rinput)
processed_img = flow(c_img)
self.save_intermediate_img(processed_img, 'p1.fits')
hdr = processed_img[0].header
self.set_base_headers(hdr)
import scipy.ndimage.filters
_logger.info('median filter')
data_smooth = scipy.ndimage.filters.median_filter(
processed_img[0].data, size=11)
self.save_intermediate_array(data_smooth, 'smooth.fits')
mm = processed_img[0].data.mean()
hdr['CCDMEAN'] = mm
processed_img[0].data /= data_smooth
self.save_intermediate_img(processed_img, 'p2.fits')
result = self.create_result(master_flatframe=processed_img)
return result
class SpectralFlatRecipe(EmirRecipe):
"""Recipe to process data taken in intensity flat-field mode."""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
flatframe = Result(prods.MasterSpectralFlat)
def run(self, rinput):
return self.create_result(flatframe=prods.MasterSpectralFlat())
class StareImageBaseRecipe(EmirRecipe):
"""Process images in Stare Image Mode"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement(optional=True)
frame = Result(prods.ProcessedImage)
def __init__(self, *args, **kwargs):
super(StareImageBaseRecipe, self).__init__(*args, **kwargs)
if False:
self.query_options['master_sky'] = Ignore()
@emirdrp.decorators.loginfo
@timeit
def run(self, rinput):
self.logger.info('starting stare image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(
rinput,
flow,
method=combine.median
)
hdr = hdulist[0].header
self.set_base_headers(hdr)
if rinput.master_bpm:
hdul_bpm = rinput.master_bpm.open()
hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0])
else:
hdu_bpm = extra.generate_empty_bpm_hdu(hdulist[0])
# Append the BPM to the result
hdulist.append(hdu_bpm)
self.logger.info('end stare image reduction')
result = self.create_result(frame=hdulist)
return result
def set_base_headers(self, hdr):
"""Set metadata in FITS headers."""
hdr = super(StareImageBaseRecipe, self).set_base_headers(hdr)
# Update EXP to 0
hdr['EXP'] = 0
return hdr
class MaskCheckRecipe(EmirRecipe):
"""
Acquire a target.
Recipe for the processing of multi-slit/long-slit check images.
**Observing modes:**
* MSM and LSM check
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
def run(self, rinput):
self.logger.info("Start MaskCheckRecipe")
self.logger.info("End MaskCheckRecipe")
return self.create_result()
class BarDetectionRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.CoordinateList2DType,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
canny_high_threshold = Parameter(0.04,
'High threshold for the canny algorithm')
canny_low_threshold = Parameter(0.01,
'High threshold for the canny algorithm')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
param_median_filter_size = Result(float)
param_canny_high_threshold = Result(float)
param_canny_low_threshold = Result(float)
def run(self, rinput):
logger = logging.getLogger('numina.recipes.emir')
logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
logger.error(error)
raise numina.exceptions.RecipeError(error)
logger.debug('finding bars')
arr = hdulist[0].data
# Median filter
logger.debug('median filtering')
mfilter_size = rinput.median_filter_size
arr_median = median_filter(arr, size=mfilter_size)
# Image is mapped between 0 and 1
# for the full range [0: 2**16]
logger.debug('image scaling to 0-1')
arr_grey = normalize_raw(arr_median)
# Find borders
logger.debug('find borders')
canny_sigma = rinput.canny_sigma
# These threshols corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
edges = canny(arr_grey,
sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Number or rows used
# These other parameters cab be tuned also
total = 5
maxdist = 1.0
bstart = 100
bend = 1900
positions = []
nt = total // 2
xfac = dtur[0] / EMIR_PIXSCALE
yfac = -dtur[1] / EMIR_PIXSCALE
vec = [yfac, xfac]
logger.debug('DTU shift is %s', vec)
# Based on the 'edges image'
# and the table of approx positions of the slits
barstab = rinput.bars_nominal_positions
# Currently, we only use fields 0 and 2
# of the nominal positions file
for coords in barstab:
lbarid = int(coords[0])
rbarid = lbarid + 55
ref_y_coor = coords[2] + vec[1]
prow = coor_to_pix_1d(ref_y_coor) - 1
fits_row = prow + 1 # FITS pixel index
logger.debug('looking for bars with ids %d - %d', lbarid, rbarid)
logger.debug('reference y position is Y %7.2f', ref_y_coor)
# Find the position of each bar
bpos = find_position(edges, prow, bstart, bend, total)
nbars_found = len(bpos)
# If no bar is found, append and empty token
if nbars_found == 0:
logger.debug('bars %d, %d not found at row %d', lbarid, rbarid,
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
elif nbars_found == 2:
# Order values by increasing X
centl, centr = sorted(bpos, key=lambda cen: cen[0])
c1 = centl[0]
c2 = centr[0]
logger.debug('bars found at row %d between %7.2f - %7.2f',
fits_row, c1, c2)
# Compute FWHM of the collapsed profile
cslit = arr_grey[prow - nt:prow + nt + 1, :]
pslit = cslit.mean(axis=0)
# Add 1 to return FITS coordinates
epos, epos_f, error = locate_bar_l(pslit, c1)
thisres1 = lbarid, fits_row, epos + 1, epos_f + 1, error
epos, epos_f, error = locate_bar_r(pslit, c2)
thisres2 = rbarid, fits_row, epos + 1, epos_f + 1, error
elif nbars_found == 1:
logger.warning(
'only 1 edge found at row %d, not yet implemented',
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
else:
logger.warning(
'3 or more edges found at row %d, not yet implemented',
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
positions.append(thisres1)
positions.append(thisres2)
logger.debug('end finding bars')
result = self.create_result(
frame=hdulist,
positions=positions,
DTU=dtub,
ROTANG=rotang,
csupos=csupos,
csusens=csusens,
param_median_filter_size=rinput.median_filter_size,
param_canny_high_threshold=rinput.canny_high_threshold,
param_canny_low_threshold=rinput.canny_low_threshold)
return result
class TestSlitDetectionRecipe(EmirRecipe):
# Recipe Requirements
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm')
canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm')
# Recipe Results
frame = Result(prods.ProcessedImage)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
def run(self, rinput):
self.logger.info('starting slit processing')
self.logger.info('basic image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('finding slits')
# Filter values below 0.0
self.logger.debug('Filter values below 0')
data1 = hdulist[0].data[:]
data1[data1 < 0.0] = 0.0
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
self.logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize_raw(data2)
# Find edges with Canny
self.logger.debug('Find edges, Canny sigma %f', canny_sigma)
# These thresholds corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
self.logger.debug('Find edges, Canny high threshold %f', high_threshold)
self.logger.debug('Find edges, Canny low threshold %f', low_threshold)
edges = canny(img_grey, sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Fill edges
self.logger.debug('Fill holes')
# I do a dilation and erosion to fill
# possible holes in 'edges'
fill = ndimage.binary_dilation(edges)
fill2 = ndimage.binary_fill_holes(fill)
fill_slits = ndimage.binary_erosion(fill2)
self.logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
self.logger.debug('%d objects found', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
self.logger.debug('Find regions and centers')
regions = ndimage.find_objects(label_objects)
centers = ndimage.center_of_mass(data2, labels=label_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=-1.0
)
result = self.create_result(frame=hdulist,
slitstable=table,
DTU=dtub,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa
)
return result
class TestPinholeRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
pinhole_nominal_positions = Requirement(
prods.CoordinateList2DType, 'Nominal positions of the pinholes')
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
_logger.info('starting processing for slit detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.debug('finding pinholes')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
_logger.error(error)
raise numina.exceptions.RecipeError(error)
if rinput.shift_coordinates:
xdtur, ydtur, zdtur = dtur
xfac = xdtur / EMIR_PIXSCALE
yfac = -ydtur / EMIR_PIXSCALE
vec = numpy.array([yfac, xfac])
_logger.info('shift is %s', vec)
ncenters = rinput.pinhole_nominal_positions + vec
else:
_logger.info('using pinhole coordinates as they are')
ncenters = rinput.pinhole_nominal_positions
_logger.info('pinhole characterization')
positions = pinhole_char(hdulist[0].data,
ncenters,
box=rinput.box_half_size,
recenter_pinhole=rinput.recenter,
maxdist=rinput.max_recenter_radius)
_logger.info('alternate pinhole characterization')
positions_alt = pinhole_char2(
hdulist[0].data,
ncenters,
recenter_pinhole=rinput.recenter,
recenter_half_box=rinput.box_half_size,
recenter_maxdist=rinput.max_recenter_radius)
result = self.create_result(
frame=hdulist,
positions=positions,
positions_alt=positions_alt,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class ArcCalibrationRecipe(EmirRecipe):
"""Process arc images applying wavelength calibration"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
bound_param = reqs.RefinedBoundaryModelParamRequirement()
lines_catalog = Requirement(LinesCatalog, 'Catalog of lines')
reduced_image = Result(prods.ProcessedImage)
rectwv_coeff = Result(prods.RectWaveCoeff)
reduced_55sp = Result(prods.ProcessedMOS)
reduced_arc = Result(prods.ProcessedMOS)
@emirdrp.decorators.loginfo
def run(self, rinput):
self.logger.info('starting rect.+wavecal. reduction of arc spectra')
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=median)
# update header con additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object (with rectification and wavelength calibration
# coefficients for the particular CSU configuration of the arc image)
# and HDUList object with the FITS image corresponding to 55 median
# spectra of each slitlet
rectwv_coeff, reduced_55sp = rectwv_coeff_from_arc_image(
reduced_image,
rinput.bound_param,
rinput.lines_catalog,
)
# generate associated ds9 region files and save them in work directory
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
# apply rectification and wavelength calibration
reduced_arc = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# save results in result directory
self.logger.info('end rect.+wavecal. reduction of arc spectra')
result = self.create_result(reduced_image=reduced_image,
rectwv_coeff=rectwv_coeff,
reduced_55sp=reduced_55sp,
reduced_arc=reduced_arc)
return result
def set_base_headers(self, hdr):
newhdr = super(ArcCalibrationRecipe, self).set_base_headers(hdr)
# Update SEC to 0
newhdr['SEC'] = 0
return newhdr
class SpecFlatLowFreq(EmirRecipe):
"""Process continuum exposures of continuum lamp (lamp ON-OFF)
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True)
master_rectwv = reqs.MasterRectWaveRequirement(optional=True)
# note that 'sum' is not allowed as combination method
method = Parameter('sigmaclip',
description='Combination method',
choices=['mean', 'median', 'sigmaclip'])
method_kwargs = Parameter(dict(),
description='Arguments for combination method',
optional=True)
minimum_slitlet_width_mm = Parameter(
float(EMIR_MINIMUM_SLITLET_WIDTH_MM),
description='Minimum width (mm) for a valid slitlet',
optional=True)
maximum_slitlet_width_mm = Parameter(
float(EMIR_MAXIMUM_SLITLET_WIDTH_MM),
description='Maximum width (mm) for a valid slitlet',
optional=True)
global_integer_offset_x_pix = Parameter(
0,
description='Global offset (pixels) in wavelength direction (integer)',
optional=True)
global_integer_offset_y_pix = Parameter(
0,
description='Global offset (pixels) in spatial direction (integer)',
optional=True)
nwindow_x_median = Parameter(
301,
description='Window size to smooth the flatfield in the spectral '
'direction',
optional=True)
nwindow_y_median = Parameter(
11,
description='Window size to smooth the flatfield in the spatial '
'direction',
optional=True)
minimum_fraction = Parameter(
0.01,
description='Minimum useful flatfielding value (fraction of maximum)',
optional=True)
minimum_value_in_output = Parameter(
0.01,
description='Minimum value allowed in output: pixels below this value'
' are set to 1.0',
optional=True)
maximum_value_in_output = Parameter(
10.0,
description='Maximum value allowed in output: pixels above this value'
' are set to 1.0',
optional=True)
debugplot = Parameter(0, description='Debugger parameter', optional=True)
reduced_flatlowfreq = Result(prods.MasterSpectralFlat)
def run(self, rinput):
self.logger.info('starting generation of flatlowfreq')
self.logger.info('rectwv_coeff..........................: {}'.format(
rinput.rectwv_coeff))
self.logger.info('master_rectwv.........................: {}'.format(
rinput.master_rectwv))
self.logger.info('Minimum slitlet width (mm)............: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info('Maximum slitlet width (mm)............: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info('Global offset X direction (pixels)....: {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info('Global offset Y direction (pixels)....: {}'.format(
rinput.global_integer_offset_y_pix))
self.logger.info('nwindow_x_median......................: {}'.format(
rinput.nwindow_x_median))
self.logger.info('nwindow_y_median......................: {}'.format(
rinput.nwindow_y_median))
self.logger.info('Minimum fraction......................: {}'.format(
rinput.minimum_fraction))
self.logger.info('Minimum value in output...............: {}'.format(
rinput.minimum_value_in_output))
self.logger.info('Maximum value in output...............: {}'.format(
rinput.maximum_value_in_output))
# check rectification and wavelength calibration information
if rinput.master_rectwv is None and rinput.rectwv_coeff is None:
raise ValueError('No master_rectwv nor rectwv_coeff data have '
'been provided')
elif rinput.master_rectwv is not None and \
rinput.rectwv_coeff is not None:
self.logger.warning('rectwv_coeff will be used instead of '
'master_rectwv')
if rinput.rectwv_coeff is not None and \
(rinput.global_integer_offset_x_pix != 0 or
rinput.global_integer_offset_y_pix != 0):
raise ValueError('global_integer_offsets cannot be used '
'simultaneously with rectwv_coeff')
# check headers to detect lamp status (on/off)
list_lampincd = []
for fname in rinput.obresult.frames:
with fname.open() as f:
list_lampincd.append(f[0].header['lampincd'])
# check number of images
nimages = len(rinput.obresult.frames)
n_on = list_lampincd.count(1)
n_off = list_lampincd.count(0)
self.logger.info(
'Number of images with lamp ON.........: {}'.format(n_on))
self.logger.info(
'Number of images with lamp OFF........: {}'.format(n_off))
self.logger.info(
'Total number of images................: {}'.format(nimages))
if n_on == 0:
raise ValueError('Insufficient number of images with lamp ON')
if n_on + n_off != nimages:
raise ValueError('Number of images does not match!')
# check combination method
if rinput.method_kwargs == {}:
method_kwargs = None
else:
if rinput.method == 'sigmaclip':
method_kwargs = rinput.method_kwargs
else:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# build object to proceed with bpm, bias, and dark (not flat)
flow = self.init_filters(rinput)
# available combination methods
method = getattr(combine, rinput.method)
# basic reduction of images with lamp ON or OFF
lampmode = {0: 'off', 1: 'on'}
reduced_image_on = None
reduced_image_off = None
for imode in lampmode.keys():
self.logger.info('starting basic reduction of images with'
' lamp {}'.format(lampmode[imode]))
tmplist = [
rinput.obresult.frames[i]
for i, lampincd in enumerate(list_lampincd)
if lampincd == imode
]
if len(tmplist) > 0:
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open()) for fname in tmplist
]
reduced_image = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
if imode == 0:
reduced_image_off = flow(reduced_image)
hdr = reduced_image_off[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_off,
'reduced_image_off.fits')
elif imode == 1:
reduced_image_on = flow(reduced_image)
hdr = reduced_image_on[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_on,
'reduced_image_on.fits')
else:
raise ValueError('Unexpected imode={}'.format(imode))
# computation of ON-OFF
header_on = reduced_image_on[0].header
data_on = reduced_image_on[0].data.astype('float32')
if n_off > 0:
header_off = reduced_image_off[0].header
data_off = reduced_image_off[0].data.astype('float32')
else:
header_off = None
data_off = np.zeros_like(data_on)
reduced_data = data_on - data_off
# update reduced image header
reduced_image = self.create_reduced_image(rinput, reduced_data,
header_on, header_off,
list_lampincd)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# define rectification and wavelength calibration coefficients
if rinput.rectwv_coeff is None:
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image, rinput.master_rectwv)
# set global offsets
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
else:
rectwv_coeff = rinput.rectwv_coeff
# save as JSON in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# apply global offsets (to both, the original and the cleaned version)
image2d = apply_integer_offsets(
image2d=reduced_data,
offx=rectwv_coeff.global_integer_offset_x_pix,
offy=rectwv_coeff.global_integer_offset_y_pix)
# load CSU configuration
csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header)
# determine (pseudo) longslits
dict_longslits = csu_conf.pseudo_longslits()
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in rectwv_coeff.missing_slitlets:
self.logger.info('-> Removing slitlet (not defined): ' + str(idel))
list_valid_islitlets.remove(idel)
# filter out slitlets with widths outside valid range
list_outside_valid_width = []
for islitlet in list_valid_islitlets:
slitwidth = csu_conf.csu_bar_slit_width(islitlet)
if (slitwidth < rinput.minimum_slitlet_width_mm) or \
(slitwidth > rinput.maximum_slitlet_width_mm):
list_outside_valid_width.append(islitlet)
self.logger.info('-> Removing slitlet (width out of range): ' +
str(islitlet))
if len(list_outside_valid_width) > 0:
for idel in list_outside_valid_width:
list_valid_islitlets.remove(idel)
# initialize rectified image
image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1))
# main loop
grism_name = rectwv_coeff.tags['grism']
filter_name = rectwv_coeff.tags['filter']
cout = '0'
debugplot = rinput.debugplot
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=debugplot)
if abs(slt.debugplot) > 10:
print(slt)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d,
subtitle='original image')
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=2,
subtitle='original (cleaned) rectified')
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
if naxis1_slitlet2d != EMIR_NAXIS1:
print('naxis1_slitlet2d: ', naxis1_slitlet2d)
print('EMIR_NAXIS1.....: ', EMIR_NAXIS1)
raise ValueError("Unexpected naxis1_slitlet2d")
# get useful slitlet region (use boundaries)
spectrail = slt.list_spectrails[0]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig
spectrail = slt.list_spectrails[2]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig
# median spectrum
sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :],
axis=0)
# smooth median spectrum along the spectral direction
sp_median = ndimage.median_filter(sp_collapsed,
5,
mode='nearest')
ymax_spmedian = sp_median.max()
y_threshold = ymax_spmedian * rinput.minimum_fraction
lremove = np.where(sp_median < y_threshold)
sp_median[lremove] = 0.0
if abs(slt.debugplot) % 10 != 0:
xaxis1 = np.arange(1, naxis1_slitlet2d + 1)
title = 'Slitlet#' + str(islitlet) + ' (median spectrum)'
ax = ximplotxy(xaxis1,
sp_collapsed,
title=title,
show=False,
**{'label': 'collapsed spectrum'})
ax.plot(xaxis1, sp_median, label='fitted spectrum')
ax.plot([1, naxis1_slitlet2d],
2 * [y_threshold],
label='threshold')
# ax.plot(xknots, yknots, 'o', label='knots')
ax.legend()
ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian)
pause_debugplot(slt.debugplot,
pltshow=True,
tight_layout=True)
# generate rectified slitlet region filled with the
# median spectrum
slitlet2d_rect_spmedian = np.tile(sp_median,
(naxis2_slitlet2d, 1))
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_rectified(
slitlet2d_rect=slitlet2d_rect_spmedian,
subtitle='rectified, filled with median spectrum')
# compute smooth surface
# clipped region
slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy()
slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0
slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0
# unrectified clipped image
slitlet2d_unrect_clipped = slt.rectify(
slitlet2d=slitlet2d_rect_clipped,
resampling=2,
inverse=True,
subtitle='unrectified, filled with median spectrum '
'(clipped)')
# normalize initial slitlet image (avoid division by zero)
slitlet2d_norm_clipped = np.zeros_like(slitlet2d)
for j in range(naxis1_slitlet2d):
for i in range(naxis2_slitlet2d):
den = slitlet2d_unrect_clipped[i, j]
if den == 0:
slitlet2d_norm_clipped[i, j] = 1.0
else:
slitlet2d_norm_clipped[i, j] = \
slitlet2d[i, j] / den
# set to 1.0 one additional pixel at each side (since
# 'den' above is small at the borders and generates wrong
# bright pixels)
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_smooth = ndimage.median_filter(
slitlet2d_norm_clipped,
size=(rinput.nwindow_y_median, rinput.nwindow_x_median),
mode='nearest')
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_clipped,
subtitle='unrectified, pixel-to-pixel (clipped)')
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_smooth,
subtitle='unrectified, pixel-to-pixel (smoothed)')
# ---
# check for (pseudo) longslit with previous and next slitlet
imin = dict_longslits[islitlet].imin()
imax = dict_longslits[islitlet].imax()
if islitlet > 1:
same_slitlet_below = (islitlet - 1) >= imin
else:
same_slitlet_below = False
if islitlet < EMIR_NBARS:
same_slitlet_above = (islitlet + 1) <= imax
else:
same_slitlet_above = False
for j in range(EMIR_NAXIS1):
xchannel = j + 1
y0_lower = slt.list_frontiers[0](xchannel)
y0_upper = slt.list_frontiers[1](xchannel)
n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower,
y0_frontier_upper=y0_upper,
resize=True)
# note that n1 and n2 are scans (ranging from 1 to NAXIS2)
nn1 = n1 - slt.bb_ns1_orig + 1
nn2 = n2 - slt.bb_ns1_orig + 1
image2d_flatfielded[(n1 - 1):n2, j] = \
slitlet2d_norm_smooth[(nn1 - 1):nn2, j]
# force to 1.0 region around frontiers
if not same_slitlet_below:
image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1
if not same_slitlet_above:
image2d_flatfielded[(n2 - 5):n2, j] = 1
cout += '.'
else:
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
self.logger.info(cout)
# restore global offsets
image2d_flatfielded = apply_integer_offsets(
image2d=image2d_flatfielded,
offx=-rectwv_coeff.global_integer_offset_x_pix,
offy=-rectwv_coeff.global_integer_offset_y_pix)
# set pixels below minimum value to 1.0
filtered = np.where(
image2d_flatfielded < rinput.minimum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# set pixels above maximum value to 1.0
filtered = np.where(
image2d_flatfielded > rinput.maximum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# update image header
reduced_flatlowfreq = self.create_reduced_image(
rinput, image2d_flatfielded, header_on, header_off, list_lampincd)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# save results in results directory
self.logger.info('end of flatlowfreq generation')
result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq)
return result
def create_reduced_image(self, rinput, reduced_data, header_on, header_off,
list_lampincd):
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
# Copy header of first image
base_header = hduls[0][0].header.copy()
hdu = fits.PrimaryHDU(reduced_data, header=base_header)
self.set_base_headers(hdu.header)
self.logger.debug('update result header')
# update additional keywords
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = 'flatlowfreq'
hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2']
hdu.header['history'] = "Processed flatlowfreq"
hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames')
# update history
dm = emirdrp.datamodel.EmirDataModel()
for img, lampincd in zip(hduls, list_lampincd):
imgid = dm.get_imgid(img)
hdu.header['HISTORY'] = "Image '{}' has lampincd='{}'".format(
imgid, lampincd)
hdu.header['HISTORY'] = "Processed flatlowfreq"
hdu.header['HISTORY'] = '--- Reduction of images with lamp ON ---'
for line in header_on['HISTORY']:
hdu.header['HISTORY'] = line
if header_off is not None:
hdu.header['HISTORY'] = '--- Reduction of images with lamp OFF ---'
for line in header_off['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header.add_history('--- rinput.stored() (BEGIN) ---')
for item in rinput.stored():
value = getattr(rinput, item)
cline = '{}: {}'.format(item, value)
hdu.header.add_history(cline)
hdu.header.add_history('--- rinput.stored() (END) ---')
result = fits.HDUList([hdu])
return result
def set_base_headers(self, hdr):
newhdr = super(SpecFlatLowFreq, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class ABBASpectraFastRectwv(EmirRecipe):
"""Process AB or ABBA images applying a single wavelength calibration
Note that the wavelength calibration has already been determined.
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement()
pattern = Parameter('ABBA',
description='Observation pattern',
choices=['AB', 'ABBA'])
# do not allow 'sum' as combination method in order to avoid
# confusion with number of counts in resulting image
method = Parameter('mean',
description='Combination method',
choices=['mean', 'median', 'sigmaclip'])
method_kwargs = Parameter(dict(),
description='Arguments for combination method')
voffset_pix = Parameter(0.0,
description='Shift (pixels) to move A into B',
optional=True)
reduced_mos_abba = Result(prods.ProcessedMOS)
reduced_mos_abba_combined = Result(prods.ProcessedMOS)
def run(self, rinput):
nimages = len(rinput.obresult.frames)
pattern = rinput.pattern
pattern_length = len(pattern)
# check combination method
if rinput.method != 'sigmaclip':
if rinput.method_kwargs != {}:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# check pattern sequence matches number of images
if nimages % pattern_length != 0:
raise ValueError('Number of images is not a multiple of pattern '
'length: {}, {}'.format(nimages, pattern_length))
nsequences = nimages // pattern_length
rectwv_coeff = rinput.rectwv_coeff
self.logger.info(rectwv_coeff)
self.logger.info('observation pattern: {}'.format(pattern))
self.logger.info('nsequences.........: {}'.format(nsequences))
full_set = pattern * nsequences
self.logger.info('full set of images.: {}'.format(full_set))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# available combination methods
method = getattr(combine, rinput.method)
method_kwargs = rinput.method_kwargs
# basic reduction of A images
list_a = [
rinput.obresult.frames[i] for i, char in enumerate(full_set)
if char == 'A'
]
with contextlib.ExitStack() as stack:
self.logger.info('starting basic reduction of A images')
hduls = [stack.enter_context(fname.open()) for fname in list_a]
reduced_image_a = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
reduced_image_a = flow(reduced_image_a)
hdr = reduced_image_a[0].header
self.set_base_headers(hdr)
# basic reduction of B images
list_b = [
rinput.obresult.frames[i] for i, char in enumerate(full_set)
if char == 'B'
]
with contextlib.ExitStack() as stack:
self.logger.info('starting basic reduction of B images')
hduls = [stack.enter_context(fname.open()) for fname in list_b]
reduced_image_b = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
reduced_image_b = flow(reduced_image_b)
hdr = reduced_image_b[0].header
self.set_base_headers(hdr)
# save intermediate reduced_image_a and reduced_image_b
self.save_intermediate_img(reduced_image_a, 'reduced_image_a.fits')
self.save_intermediate_img(reduced_image_b, 'reduced_image_b.fits')
# computation of A-B
header_a = reduced_image_a[0].header
header_b = reduced_image_b[0].header
data_a = reduced_image_a[0].data.astype('float32')
data_b = reduced_image_b[0].data.astype('float32')
reduced_data = data_a - data_b
# update reduced image header
reduced_image = self.create_reduced_image(
rinput,
reduced_data,
header_a,
header_b,
rinput.pattern,
full_set,
voffset_pix=0,
header_mos_abba=None,
)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# apply rectification and wavelength calibration
self.logger.info('begin rect.+wavecal. reduction of ABBA spectra')
reduced_mos_abba = apply_rectwv_coeff(reduced_image, rectwv_coeff)
header_mos_abba = reduced_mos_abba[0].header
# combine A and B data by shifting B on top of A
voffset_pix = rinput.voffset_pix
if voffset_pix is not None and voffset_pix != 0:
self.logger.info(
'correcting vertical offset (pixesl): {}'.format(voffset_pix))
reduced_mos_abba_data = reduced_mos_abba[0].data.astype('float32')
shifted_a_minus_b_data = shift_image2d(
reduced_mos_abba_data,
yoffset=-voffset_pix,
).astype('float32')
reduced_mos_abba_combined_data = \
reduced_mos_abba_data - shifted_a_minus_b_data
# scale signal to exposure of a single image
reduced_mos_abba_combined_data /= 2.0
else:
reduced_mos_abba_combined_data = None
# update reduced combined image header
reduced_mos_abba_combined = self.create_reduced_image(
rinput,
reduced_mos_abba_combined_data,
header_a,
header_b,
rinput.pattern,
full_set,
voffset_pix,
header_mos_abba=header_mos_abba)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos_abba,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of ABBA spectra')
result = self.create_result(
reduced_mos_abba=reduced_mos_abba,
reduced_mos_abba_combined=reduced_mos_abba_combined)
return result
def create_reduced_image(self, rinput, reduced_data, header_a, header_b,
pattern, full_set, voffset_pix, header_mos_abba):
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
# Copy header of first image
base_header = hduls[0][0].header.copy()
hdu = fits.PrimaryHDU(reduced_data, header=base_header)
self.set_base_headers(hdu.header)
self.logger.debug('update result header')
if header_mos_abba is not None:
self.logger.debug('update result header')
crpix1 = header_mos_abba['crpix1']
crval1 = header_mos_abba['crval1']
cdelt1 = header_mos_abba['cdelt1']
# update wavelength calibration in FITS header
for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']:
if keyword in hdu.header:
hdu.header.remove(keyword)
hdu.header['crpix1'] = (crpix1, 'reference pixel')
hdu.header['crval1'] = (crval1, 'central wavelength at crpix1')
hdu.header['cdelt1'] = \
(cdelt1, 'linear dispersion (Angstrom/pixel)')
hdu.header['cunit1'] = ('Angstrom', 'units along axis1')
hdu.header['ctype1'] = 'WAVELENGTH'
hdu.header['crpix2'] = (0.0, 'reference pixel')
hdu.header['crval2'] = (0.0, 'central value at crpix2')
hdu.header['cdelt2'] = (1.0, 'increment')
hdu.header['ctype2'] = 'PIXEL'
hdu.header['cunit2'] = ('Pixel', 'units along axis2')
for keyword in [
'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2',
'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2'
]:
if keyword in hdu.header:
hdu.header.remove(keyword)
# update additional keywords
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = pattern + ' pattern'
hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2']
hdu.header['history'] = "Processed " + pattern + " pattern"
hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames')
# update history
dm = emirdrp.datamodel.EmirDataModel()
for img, key in zip(hduls, full_set):
imgid = dm.get_imgid(img)
hdu.header['HISTORY'] = "Image '{}' is '{}'".format(imgid, key)
hdu.header['HISTORY'] = "Processed " + pattern + " pattern"
hdu.header['HISTORY'] = '--- Reduction of A images ---'
for line in header_a['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header['HISTORY'] = '--- Reduction of B images ---'
for line in header_b['HISTORY']:
hdu.header['HISTORY'] = line
if voffset_pix is not None and voffset_pix != 0:
hdu.header['HISTORY'] = '--- Combination of AB spectra ---'
hdu.header['HISTORY'] = "voffset_pix between A and B {}".format(
voffset_pix)
hdu.header.add_history('--- rinput.stored() (BEGIN) ---')
for item in rinput.stored():
value = getattr(rinput, item)
cline = '{}: {}'.format(item, value)
hdu.header.add_history(cline)
hdu.header.add_history('--- rinput.stored() (END) ---')
result = fits.HDUList([hdu])
return result
def set_base_headers(self, hdr):
newhdr = super(ABBASpectraFastRectwv, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
def extract_tags_from_obsres(self, obsres, tag_keys):
# this function is necessary to make use of recipe requirements
# that are provided as lists
final_tags = []
for frame in obsres.frames:
ref_img = frame.open()
tag = self.extract_tags_from_ref(ref_img,
tag_keys,
base=obsres.labels)
final_tags.append(tag)
return final_tags
class StareSpectraRectwv(EmirRecipe):
"""Process images in Stare spectra mode applying wavelength calibration
Note that in this case the wavelength calibration has already been
determined.
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement()
reduced_mos = Result(prods.ProcessedMOS)
def run(self, rinput):
self.logger.info('applying existing rect.+wavecal. calibration of '
'stare spectra')
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=median)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rinput.rectwv_coeff)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rinput.rectwv_coeff)
save_spectral_lines_ds9(rinput.rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of stare spectra')
result = self.create_result(reduced_mos=reduced_mos)
return result
def set_base_headers(self, hdr):
newhdr = super(StareSpectraRectwv, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class GenerateRectwvCoeff(EmirRecipe):
"""Process images in Stare spectra.
This recipe generates a rectified and wavelength calibrated
image after applying a model (master_rectwv). This calibration
can be refined (using refine_wavecalib_mode != 0).
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
master_rectwv = reqs.MasterRectWaveRequirement()
refine_wavecalib_mode = Parameter(
0,
description='Apply wavelength calibration refinement',
choices=[0, 1, 2, 11, 12])
minimum_slitlet_width_mm = Parameter(
float(EMIR_MINIMUM_SLITLET_WIDTH_MM),
description='Minimum width (mm) for a valid slitlet',
)
maximum_slitlet_width_mm = Parameter(
float(EMIR_MAXIMUM_SLITLET_WIDTH_MM),
description='Maximum width (mm) for a valid slitlet',
)
global_integer_offset_x_pix = Parameter(
0,
description='Global offset (pixels) in wavelength direction (integer)',
)
global_integer_offset_y_pix = Parameter(
0,
description='Global offset (pixels) in spatial direction (integer)',
)
reduced_mos = Result(prods.ProcessedMOS)
rectwv_coeff = Result(RectWaveCoeff)
def run(self, rinput):
self.logger.info('starting rect.+wavecal. reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
self.logger.info('Wavelength calibration refinement mode: {}'.format(
rinput.refine_wavecalib_mode))
self.logger.info('Minimum slitlet width (mm)............: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info('Maximum slitlet width (mm)............: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info('Global offset X direction (pixels)....: {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info('Global offset Y direction (pixels)....: {}'.format(
rinput.global_integer_offset_y_pix))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=median)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image,
rinput.master_rectwv)
# set global offsets
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
# apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# wavelength calibration refinement
# 0 -> no refinement
# 1 -> apply global offset to all the slitlets (using ARC lines)
# 2 -> apply individual offset to each slitlet (using ARC lines)
# 11 -> apply global offset to all the slitlets (using OH lines)
# 12 -> apply individual offset to each slitlet (using OH lines)
if rinput.refine_wavecalib_mode != 0:
self.logger.info(
'Refining wavelength calibration (mode={})'.format(
rinput.refine_wavecalib_mode))
# refine RectWaveCoeff object
rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff(
reduced_mos,
rectwv_coeff,
rinput.refine_wavecalib_mode,
rinput.minimum_slitlet_width_mm,
rinput.maximum_slitlet_width_mm,
save_intermediate_results=self.intermediate_results)
self.save_intermediate_img(expected_catalog_lines,
'expected_catalog_lines.fits')
# re-apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of stare spectra')
result = self.create_result(reduced_mos=reduced_mos,
rectwv_coeff=rectwv_coeff)
return result
def set_base_headers(self, hdr):
newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class ABBASpectraRectwv(EmirRecipe):
"""Process images in AB or ABBA mode applying wavelength calibration
Note that in this case the wavelength calibration has already been
determined.
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
rectwv_coeff = reqs.RectWaveCoeffRequirement(optional=True)
list_rectwv_coeff = reqs.ListOfRectWaveCoeffRequirement(optional=True)
pattern = Parameter('ABBA',
description='Observation pattern',
choices=['AB', 'ABBA'])
# do not allow 'sum' as combination method in order to avoid
# confusion with number of counts in resulting image
method = Parameter('mean',
description='Combination method',
choices=['mean', 'median', 'sigmaclip'])
method_kwargs = Parameter(dict(),
description='Arguments for combination method',
optional=True)
refine_target_along_slitlet = Parameter(
dict(),
description='Parameters to refine location of target along the slitlet'
)
reduced_mos_abba = Result(prods.ProcessedMOS)
reduced_mos_abba_combined = Result(prods.ProcessedMOS)
def run(self, rinput):
nimages = len(rinput.obresult.frames)
pattern = rinput.pattern
pattern_length = len(pattern)
# check combination method
if rinput.method != 'sigmaclip':
if rinput.method_kwargs != {}:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# check pattern sequence matches number of images
if nimages % pattern_length != 0:
raise ValueError('Number of images is not a multiple of pattern '
'length: {}, {}'.format(nimages, pattern_length))
nsequences = nimages // pattern_length
# check rectification and wavelength calibration information
if rinput.rectwv_coeff is None and rinput.list_rectwv_coeff is None:
raise ValueError('No rectwv_coeff nor list_rectwv_coeff data have '
'been provided')
elif rinput.rectwv_coeff is not None and \
rinput.list_rectwv_coeff is not None:
raise ValueError("rectwv_coeff and list_rectwv_coeff cannot be "
"used simultaneously")
elif rinput.rectwv_coeff is not None:
list_rectwv_coeff = [rinput.rectwv_coeff] * nimages
elif rinput.list_rectwv_coeff is not None:
if len(rinput.list_rectwv_coeff) != nimages:
raise ValueError("Unexpected number of rectwv_coeff files "
"in list_rectwv_coeff")
else:
list_rectwv_coeff = rinput.list_rectwv_coeff
# check filter and grism are the same in all JSON files
for item in ['grism', 'filter']:
list_values = []
for calib in list_rectwv_coeff:
list_values.append(calib.tags[item])
if len(set(list_values)) != 1:
raise ValueError(
'list_rectwv_coeff contains coefficients for '
'different {}s'.format(item))
else:
raise ValueError("Unexpected error!")
# grism and filter names
grism_name = list_rectwv_coeff[0].tags['grism']
filter_name = list_rectwv_coeff[0].tags['filter']
# compute offsets from WCS info in image headers
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
sep_arcsec, spatial_scales = compute_wcs_offsets(hduls)
sep_pixel = np.round(sep_arcsec / spatial_scales, 6)
for i in range(nimages):
self.logger.info(list_rectwv_coeff[i])
self.logger.info(
'observation pattern..................: {}'.format(pattern))
self.logger.info(
'nsequences...........................: {}'.format(nsequences))
self.logger.info(
'offsets (arcsec, from WCS)...........: {}'.format(sep_arcsec))
self.logger.info(
'spatial scales (arcsec/pix, from WCS): {}'.format(spatial_scales))
self.logger.info(
'spatial scales (pixels, from WCS)....: {}'.format(sep_pixel))
full_set = pattern * nsequences
self.logger.info(
'full set of images...................: {}'.format(full_set))
# basic parameters to determine useful pixels in the wavelength
# direction
dict_rtas = rinput.refine_target_along_slitlet
valid_keys = [
'npix_removed_near_ohlines', 'nwidth_medfilt',
'save_individual_images', 'ab_different_target',
'vpix_region_a_target', 'vpix_region_a_sky',
'vpix_region_b_target', 'vpix_region_b_sky',
'list_valid_wvregions_a', 'list_valid_wvregions_b'
]
for dumkey in dict_rtas.keys():
if dumkey not in valid_keys:
raise ValueError('Unexpected key={}'.format(dumkey))
if 'vpix_region_a_target' in dict_rtas.keys():
vpix_region_a_target = dict_rtas['vpix_region_a_target']
else:
vpix_region_a_target = None
if 'vpix_region_a_sky' in dict_rtas.keys():
vpix_region_a_sky = dict_rtas['vpix_region_a_sky']
else:
vpix_region_a_sky = None
if 'vpix_region_b_target' in dict_rtas.keys():
vpix_region_b_target = dict_rtas['vpix_region_b_target']
else:
vpix_region_b_target = None
if 'vpix_region_b_sky' in dict_rtas.keys():
vpix_region_b_sky = dict_rtas['vpix_region_b_sky']
else:
vpix_region_b_sky = None
if 'ab_different_target' in dict_rtas.keys():
ab_different_target = int(dict_rtas['ab_different_target'])
if ab_different_target not in [-1, 0, 1, 9]:
raise ValueError('Invalid ab_different_target={} value'.format(
ab_different_target))
else:
raise ValueError('Missing ab_different_target value')
try:
npix_removed_near_ohlines = \
int(dict_rtas['npix_removed_near_ohlines'])
except KeyError:
npix_removed_near_ohlines = 0
except ValueError:
raise ValueError('wrong value: npix_removed_near_ohlines='
'{}'.format(
dict_rtas['npix_removed_near_ohlines']))
if 'list_valid_wvregions_a' in dict_rtas.keys():
if vpix_region_a_target is None:
raise ValueError('Unexpected list_valid_wvregions_a when '
'vpix_region_a_target is not set')
list_valid_wvregions_a = dict_rtas['list_valid_wvregions_a']
else:
list_valid_wvregions_a = None
if 'list_valid_wvregions_b' in dict_rtas.keys():
if vpix_region_b_target is None:
raise ValueError('Unexpected list_valid_wvregions_b when '
'vpix_region_b_target is not set')
list_valid_wvregions_b = dict_rtas['list_valid_wvregions_b']
else:
list_valid_wvregions_b = None
try:
nwidth_medfilt = int(dict_rtas['nwidth_medfilt'])
except KeyError:
nwidth_medfilt = 0
except ValueError:
raise ValueError('wrong value: nwidth_medfilt={}'.format(
dict_rtas['nwidth_medfilt']))
try:
save_individual_images = int(dict_rtas['save_individual_images'])
except KeyError:
save_individual_images = 0
except ValueError:
raise ValueError('wrong value: save_individual_images={}'.format(
dict_rtas['save_individual_images']))
self.logger.info(
'npix_removed_near_ohlines: {}'.format(npix_removed_near_ohlines))
self.logger.info('nwidth_medfilt: {}'.format(nwidth_medfilt))
self.logger.info(
'vpix_region_a_target: {}'.format(vpix_region_a_target))
self.logger.info(
'vpix_region_b_target: {}'.format(vpix_region_b_target))
self.logger.info(
'list_valid_wvregions_a: {}'.format(list_valid_wvregions_a))
self.logger.info(
'list_valid_wvregions_b: {}'.format(list_valid_wvregions_b))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# basic reduction, rectification and wavelength calibration of
# all the individual images
list_reduced_mos_images = []
self.logger.info('starting reduction of individual images')
for i, char in enumerate(full_set):
frame = rinput.obresult.frames[i]
self.logger.info('image {} ({} of {})'.format(
char, i + 1, nimages))
self.logger.info('image: {}'.format(frame.filename))
with frame.open() as f:
base_header = f[0].header.copy()
data = f[0].data.astype('float32')
grism_name_ = base_header['grism']
if grism_name_ != grism_name:
raise ValueError('Incompatible grism name in '
'rectwv_coeff.json file and FITS image')
filter_name_ = base_header['filter']
if filter_name_ != filter_name:
raise ValueError('Incompatible filter name in '
'rectwv_coeff.json file and FITS image')
hdu = fits.PrimaryHDU(data, header=base_header)
hdu.header['UUID'] = str(uuid.uuid1())
hdul = fits.HDUList([hdu])
# basic reduction
reduced_image = flow(hdul)
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# rectification and wavelength calibration
reduced_mos_image = apply_rectwv_coeff(reduced_image,
list_rectwv_coeff[i])
if save_individual_images != 0:
self.save_intermediate_img(
reduced_mos_image, 'reduced_mos_image_' + char + '_' +
frame.filename[:10] + '.fits')
list_reduced_mos_images.append(reduced_mos_image)
# intermediate PDF file with crosscorrelation plots
if self.intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation_ab.pdf')
else:
pdf = None
# compute offsets between images
self.logger.info('computing offsets between individual images')
first_a = True
first_b = True
xisok_a = None
xisok_b = None
reference_profile_a = None
reference_profile_b = None
refine_a = vpix_region_a_target is not None
refine_b = vpix_region_b_target is not None
list_offsets = []
for i, char in enumerate(full_set):
if char == 'A' and refine_a:
refine_image = True
elif char == 'B' and refine_b:
refine_image = True
else:
refine_image = False
if refine_image:
frame = rinput.obresult.frames[i]
isky = get_isky(i, pattern)
frame_sky = rinput.obresult.frames[isky]
self.logger.info('image {} ({} of {})'.format(
char, i + 1, nimages))
self.logger.info('image: {}'.format(frame.filename))
self.logger.info('(sky): {}'.format(frame_sky.filename))
data = list_reduced_mos_images[i][0].data.copy()
data_sky = list_reduced_mos_images[isky][0].data
data -= data_sky
base_header = list_reduced_mos_images[i][0].header
# get useful pixels in the wavelength direction
if char == 'A' and first_a:
xisok_a = useful_mos_xpixels(
data,
base_header,
vpix_region=vpix_region_a_target,
npix_removed_near_ohlines=npix_removed_near_ohlines,
list_valid_wvregions=list_valid_wvregions_a,
debugplot=0)
elif char == 'B' and first_b:
xisok_b = useful_mos_xpixels(
data,
base_header,
vpix_region=vpix_region_b_target,
npix_removed_near_ohlines=npix_removed_near_ohlines,
list_valid_wvregions=list_valid_wvregions_b,
debugplot=0)
if char == 'A':
nsmin = vpix_region_a_target[0]
nsmax = vpix_region_a_target[1]
xisok = xisok_a
if vpix_region_a_sky is not None:
nsmin_sky = vpix_region_a_sky[0]
nsmax_sky = vpix_region_a_sky[1]
skysubtraction = True
else:
nsmin_sky = None
nsmax_sky = None
skysubtraction = False
elif char == 'B':
nsmin = vpix_region_b_target[0]
nsmax = vpix_region_b_target[1]
xisok = xisok_b
if vpix_region_b_sky is not None:
nsmin_sky = vpix_region_b_sky[0]
nsmax_sky = vpix_region_b_sky[1]
skysubtraction = True
else:
nsmin_sky = None
nsmax_sky = None
skysubtraction = False
else:
raise ValueError('Unexpected char value: {}'.format(char))
# initial slitlet region
slitlet2d = data[(nsmin - 1):nsmax, :].copy()
if skysubtraction:
slitlet2d_sky = data[(nsmin_sky - 1):nsmax_sky, :].copy()
median_sky = np.median(slitlet2d_sky, axis=0)
slitlet2d -= median_sky
# selected wavelength regions after blocking OH lines
slitlet2d_blocked = slitlet2d[:, xisok]
# apply median filter in the X direction
if nwidth_medfilt > 1:
slitlet2d_blocked_smoothed = ndimage.filters.median_filter(
slitlet2d_blocked, size=(1, nwidth_medfilt))
else:
slitlet2d_blocked_smoothed = slitlet2d_blocked
# ---
# convert to 1D series of spatial profiles (note: this
# option does not work very well when the signal is low;
# a simple mean profile does a better job)
# profile = slitlet2d_blocked_smoothed.transpose().ravel()
# ---
profile = np.mean(slitlet2d_blocked_smoothed, axis=1)
if char == 'A' and first_a:
reference_profile_a = profile.copy()
elif char == 'B' and first_b:
reference_profile_b = profile.copy()
# ---
# # To write pickle file
# import pickle
# with open(frame.filename[:10] + '_profile.pkl', 'wb') as f:
# pickle.dump(profile, f)
# # To read pickle file
# with open('test.pkl', 'rb') as f:
# x = pickle.load(f)
# ---
# crosscorrelation to find offset
naround_zero = (nsmax - nsmin) // 3
if char == 'A':
reference_profile = reference_profile_a
elif char == 'B':
reference_profile = reference_profile_b
else:
raise ValueError('Unexpected char value: {}'.format(char))
offset, fpeak = periodic_corr1d(
sp_reference=reference_profile,
sp_offset=profile,
remove_mean=False,
frac_cosbell=0.10,
zero_padding=11,
fminmax=None,
nfit_peak=5,
naround_zero=naround_zero,
sp_label='spatial profile',
plottitle='Image #{} (type {}), {}'.format(
i + 1, char, frame.filename[:10]),
pdf=pdf)
# round to 4 decimal places
if abs(offset) < 1E-4:
offset = 0.0 # avoid -0.0
else:
offset = round(offset, 4)
list_offsets.append(offset)
# end of loop
if char == 'A' and first_a:
first_a = False
elif char == 'B' and first_b:
first_b = False
else:
list_offsets.append(0.0)
self.logger.info('computed offsets: {}'.format(list_offsets))
self.logger.info('correcting vertical offsets between individual '
'images')
list_a = []
list_b = []
for i, (char, offset) in enumerate(zip(full_set, list_offsets)):
frame = rinput.obresult.frames[i]
self.logger.info('image {} ({} of {})'.format(
char, i + 1, nimages))
self.logger.info('image: {}'.format(frame.filename))
reduced_mos_image = list_reduced_mos_images[i]
data = reduced_mos_image[0].data
base_header = reduced_mos_image[0].header
self.logger.info(
'correcting vertical offset (pixesl): {}'.format(offset))
if offset != 0:
reduced_mos_image[0].data = shift_image2d(
data, yoffset=-offset).astype('float32')
base_header['HISTORY'] = 'Applying voffset_pix {}'.format(offset)
if save_individual_images != 0:
self.save_intermediate_img(
reduced_mos_image, 'reduced_mos_image_refined_' + char +
'_' + frame.filename[:10] + '.fits')
# store reduced_mos_image
if char == 'A':
list_a.append(reduced_mos_image)
elif char == 'B':
list_b.append(reduced_mos_image)
else:
raise ValueError('Unexpected char value: {}'.format(char))
# combination method
method = getattr(combine, rinput.method)
method_kwargs = rinput.method_kwargs
# final combination of A images
self.logger.info('combining individual A images')
reduced_mos_image_a = combine_imgs(list_a,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
self.save_intermediate_img(reduced_mos_image_a,
'reduced_mos_image_a.fits')
# final combination of B images
self.logger.info('combining individual B images')
reduced_mos_image_b = combine_imgs(list_b,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
self.save_intermediate_img(reduced_mos_image_b,
'reduced_mos_image_b.fits')
self.logger.info('mixing A and B spectra')
header_a = reduced_mos_image_a[0].header
header_b = reduced_mos_image_b[0].header
data_a = reduced_mos_image_a[0].data.astype('float32')
data_b = reduced_mos_image_b[0].data.astype('float32')
reduced_mos_abba_data = data_a - data_b
# update reduced mos image header
reduced_mos_abba = self.create_mos_abba_image(rinput,
dict_rtas,
reduced_mos_abba_data,
header_a,
header_b,
pattern,
full_set,
list_offsets,
voffset_pix=0)
# combine A and B data by shifting B on top of A
if abs(ab_different_target) == 0:
len_prof_a = len(reference_profile_a)
len_prof_b = len(reference_profile_b)
if len_prof_a == len_prof_b:
reference_profile = reference_profile_a
profile = reference_profile_b
naround_zero = len_prof_a // 3
elif len_prof_a > len_prof_b:
ndiff = len_prof_a - len_prof_b
reference_profile = reference_profile_a
profile = np.concatenate(
(reference_profile_b, np.zeros(ndiff, dtype='float')))
naround_zero = len_prof_a // 3
else:
ndiff = len_prof_b - len_prof_a
reference_profile = np.concatenate(
(reference_profile_a, np.zeros(ndiff, dtype='float')))
profile = reference_profile_b
naround_zero = len_prof_b // 3
offset, fpeak = periodic_corr1d(
sp_reference=reference_profile,
sp_offset=profile,
remove_mean=False,
frac_cosbell=0.10,
zero_padding=11,
fminmax=None,
nfit_peak=5,
naround_zero=naround_zero,
sp_label='spatial profile',
plottitle='Comparison of A and B profiles',
pdf=pdf)
voffset_pix = vpix_region_b_target[0] - vpix_region_a_target[0]
voffset_pix += offset
elif abs(ab_different_target) == 1:
# apply nominal offset (from WCS info) between first A
# and first B
voffset_pix = sep_pixel[1] * ab_different_target
elif ab_different_target == 9:
voffset_pix = None
else:
raise ValueError(
'Invalid ab_different_target={}'.format(ab_different_target))
# close output PDF file
if pdf is not None:
pdf.close()
if voffset_pix is not None:
self.logger.info(
'correcting vertical offset (pixesl): {}'.format(voffset_pix))
shifted_a_minus_b_data = shift_image2d(
reduced_mos_abba_data,
yoffset=-voffset_pix,
).astype('float32')
reduced_mos_abba_combined_data = \
reduced_mos_abba_data - shifted_a_minus_b_data
# scale signal to exposure of a single image
reduced_mos_abba_combined_data /= 2.0
else:
reduced_mos_abba_combined_data = None
# update reduced mos combined image header
reduced_mos_abba_combined = self.create_mos_abba_image(
rinput, dict_rtas, reduced_mos_abba_combined_data, header_a,
header_b, pattern, full_set, list_offsets, voffset_pix)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(list_rectwv_coeff[0])
save_spectral_lines_ds9(list_rectwv_coeff[0])
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos_abba,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of ABBA spectra')
result = self.create_result(
reduced_mos_abba=reduced_mos_abba,
reduced_mos_abba_combined=reduced_mos_abba_combined)
return result
def create_mos_abba_image(self, rinput, dict_rtas, reduced_mos_abba_data,
header_a, header_b, pattern, full_set,
list_offsets, voffset_pix):
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open())
for fname in rinput.obresult.frames
]
# Copy header of first image
base_header = hduls[0][0].header.copy()
hdu = fits.PrimaryHDU(reduced_mos_abba_data, header=base_header)
self.set_base_headers(hdu.header)
# check consistency of wavelength calibration paramenters
for param in ['crpix1', 'crval1', 'cdelt1']:
if header_a[param] != header_b[param]:
raise ValueError(
'Headers of A and B images have different '
'values of {}'.format(param))
self.logger.debug('update result header')
crpix1 = header_a['crpix1']
crval1 = header_a['crval1']
cdelt1 = header_a['cdelt1']
# update wavelength calibration in FITS header
for keyword in ['crval1', 'crpix1', 'crval2', 'crpix2']:
if keyword in hdu.header:
hdu.header.remove(keyword)
hdu.header['crpix1'] = (crpix1, 'reference pixel')
hdu.header['crval1'] = (crval1, 'central wavelength at crpix1')
hdu.header['cdelt1'] = (cdelt1,
'linear dispersion (Angstrom/pixel)')
hdu.header['cunit1'] = ('Angstrom', 'units along axis1')
hdu.header['ctype1'] = 'WAVELENGTH'
hdu.header['crpix2'] = (0.0, 'reference pixel')
hdu.header['crval2'] = (0.0, 'central value at crpix2')
hdu.header['cdelt2'] = (1.0, 'increment')
hdu.header['ctype2'] = 'PIXEL'
hdu.header['cunit2'] = ('Pixel', 'units along axis2')
for keyword in [
'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'PCD1_1', 'PCD1_2',
'PCD2_1', 'PCD2_2', 'PCRPIX1', 'PCRPIX2'
]:
if keyword in hdu.header:
hdu.header.remove(keyword)
# update additional keywords
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = pattern + ' pattern'
hdu.header['TSUTC2'] = hduls[-1][0].header['TSUTC2']
hdu.header['NUM-NCOM'] = (len(hduls), 'Number of combined frames')
# update history
hdu.header['HISTORY'] = "Processed " + pattern + " pattern"
hdu.header['HISTORY'] = '--- Reduction of A images ---'
for line in header_a['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header['HISTORY'] = '--- Reduction of B images ---'
for line in header_b['HISTORY']:
hdu.header['HISTORY'] = line
hdu.header['HISTORY'] = '--- Combination of ABBA images ---'
for key in dict_rtas:
hdu.header['HISTORY'] = '{}: {}'.format(key, dict_rtas[key])
dm = emirdrp.datamodel.EmirDataModel()
for img, key, offset in zip(hduls, full_set, list_offsets):
imgid = dm.get_imgid(img)
hdu.header['HISTORY'] = \
"Image '{}' is '{}', with voffset_pix {}".format(
imgid, key, offset)
if voffset_pix is not None and voffset_pix != 0:
hdu.header['HISTORY'] = '--- Combination of AB spectra ---'
hdu.header['HISTORY'] = "voffset_pix between A and B {}".format(
voffset_pix)
hdu.header.add_history('--- rinput.stored() (BEGIN) ---')
for item in rinput.stored():
value = getattr(rinput, item)
cline = '{}: {}'.format(item, value)
hdu.header.add_history(cline)
hdu.header.add_history('--- rinput.stored() (END) ---')
result = fits.HDUList([hdu])
return result
def set_base_headers(self, hdr):
newhdr = super(ABBASpectraRectwv, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
def extract_tags_from_obsres(self, obsres, tag_keys):
# this function is necessary to make use of recipe requirements
# that are provided as lists
final_tags = []
for frame in obsres.frames:
ref_img = frame.open()
tag = self.extract_tags_from_ref(ref_img,
tag_keys,
base=obsres.labels)
final_tags.append(tag)
return final_tags
class StareSpectraWaveRecipe(EmirRecipe):
"""Process images in Stare spectra at the GTC.
This recipe is intended to be used at GTC. The rectification
and wavelength calibration can computed from a model if this
model (master_rectwv) is provided as input.
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
master_rectwv = reqs.MasterRectWaveRequirement(optional=True)
master_sky = reqs.SpectralSkyRequirement(optional=True)
reduced_image = Result(prods.ProcessedImage)
stare = Result(prods.ProcessedMOS)
def run(self, rinput):
self.logger.info('starting reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=median)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# rectification and wavelength calibration (if a model has
# been provided)
if rinput.master_rectwv:
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image, rinput.master_rectwv)
# apply rectification and wavelength calibration
stare_image = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# save as JSON file in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(stare_image,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# image_wl_calibrated = True
else:
stare_image = reduced_image
self.logger.info('No wavelength calibration provided')
grism_value = hdr.get('GRISM', 'unknown')
self.logger.debug('GRISM is %s', grism_value)
if grism_value.lower() == 'open':
self.logger.debug('GRISM is %s, so this seems OK', grism_value)
# image_wl_calibrated = False
if rinput.master_sky:
# Sky subtraction after rectification
msky = rinput.master_sky.open()
# Check if images have the same size.
# if so, go ahead
if msky[0].data.shape != stare_image[0].data.shape:
self.logger.warning(
"sky and current image don't have the same shape")
else:
sky_corrector = proc.SkyCorrector(
msky[0].data,
datamodel=self.datamodel,
calibid=self.datamodel.get_imgid(msky))
stare_image = sky_corrector(stare_image)
else:
self.logger.info('No sky image provided')
# save results in results directory
self.logger.info('end reduction of stare spectra')
result = self.create_result(reduced_image=reduced_image,
stare=stare_image)
return result
def set_base_headers(self, hdr):
newhdr = super(StareSpectraWaveRecipe, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class TestMaskRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
pinhole_nominal_positions = Requirement(
prods.CoordinateList2DType, 'Nominal positions of the pinholes')
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
obj_min_size = Parameter(200, 'Minimum size of the slit')
obj_max_size = Parameter(3000, 'Maximum size of the slit')
slit_size_ratio = Parameter(
4.0, 'Minimum ratio between height and width for slits')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
_logger.info('starting processing for slit detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.debug('finding pinholes')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
_logger.error(error)
raise numina.exceptions.RecipeError(error)
if rinput.shift_coordinates:
xdtur, ydtur, zdtur = dtur
xfac = xdtur / EMIR_PIXSCALE
yfac = -ydtur / EMIR_PIXSCALE
vec = numpy.array([yfac, xfac])
_logger.info('shift is %s', vec)
ncenters = rinput.pinhole_nominal_positions + vec
else:
_logger.info('using pinhole coordinates as they are')
ncenters = rinput.pinhole_nominal_positions
_logger.info('pinhole characterization')
positions = pinhole_char(hdulist[0].data,
ncenters,
box=rinput.box_half_size,
recenter_pinhole=rinput.recenter,
maxdist=rinput.max_recenter_radius)
_logger.info('alternate pinhole characterization')
positions_alt = pinhole_char2(
hdulist[0].data,
ncenters,
recenter_pinhole=rinput.recenter,
recenter_half_box=rinput.box_half_size,
recenter_maxdist=rinput.max_recenter_radius)
_logger.debug('finding slits')
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
obj_min_size = rinput.obj_min_size
obj_max_size = rinput.obj_max_size
data1 = hdulist[0].data
_logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize(data2)
# Find edges with canny
_logger.debug('Find edges with canny, sigma %d', canny_sigma)
edges = canny(img_grey, sigma=canny_sigma)
# Fill edges
_logger.debug('Fill holes')
fill_slits = ndimage.binary_fill_holes(edges)
_logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
_logger.debug('%d objects found', nb_labels)
# Filter on the area of the labeled region
# Perhaps we could ignore this filtering and
# do it later?
_logger.debug('Filter objects by size')
# Sizes of regions
sizes = numpy.bincount(label_objects.ravel())
_logger.debug('Min size is %d', obj_min_size)
_logger.debug('Max size is %d', obj_max_size)
mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size)
# Filter out regions
nids, = numpy.where(mask_sizes)
mm = numpy.in1d(label_objects, nids)
mm.shape = label_objects.shape
fill_slits_clean = numpy.where(mm, 1, 0)
# and relabel
_logger.debug('Label filtered objects')
relabel_objects, nb_labels = ndimage.label(fill_slits_clean)
_logger.debug('%d objects found after filtering', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
_logger.debug('Find regions and centers')
regions = ndimage.find_objects(relabel_objects)
centers = ndimage.center_of_mass(data2,
labels=relabel_objects,
index=ids)
table = char_slit(data2,
regions,
slit_size_ratio=rinput.slit_size_ratio)
result = self.create_result(
frame=hdulist,
positions=positions,
positions_alt=positions_alt,
slitstable=table,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class BarDetectionRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.NominalPositions,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
average_box_row_size = Parameter(
7, 'Number of rows to average for fine centering (odd)')
average_box_col_size = Parameter(
21, 'Number of columns to extract for fine centering (odd)')
fit_peak_npoints = Parameter(
3, 'Number of points to use for fitting the peak (odd)')
# Recipe Products
frame = Result(prods.ProcessedImage)
# derivative = Result(prods.ProcessedImage)
slits = Result(tarray.ArrayType)
positions3 = Result(tarray.ArrayType)
positions5 = Result(tarray.ArrayType)
positions7 = Result(tarray.ArrayType)
positions9 = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
TSUTC1 = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
def run(self, rinput):
self.logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(hdulist, 'reduced_image.fits')
try:
rotang = hdr['ROTANG']
tsutc1 = hdr['TSUTC1']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
if len(csupos) != 2 * EMIR_NBARS:
raise RecipeError('Number of CSUPOS != 2 * NBARS')
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('start finding bars')
allpos, slits = find_bars(
hdulist,
rinput.bars_nominal_positions,
csupos,
dtur,
average_box_row_size=rinput.average_box_row_size,
average_box_col_size=rinput.average_box_col_size,
fit_peak_npoints=rinput.fit_peak_npoints,
median_filter_size=rinput.median_filter_size,
logger=self.logger)
self.logger.debug('end finding bars')
if self.intermediate_results:
with open('ds9.reg', 'w') as ds9reg:
slits_to_ds9_reg(ds9reg, slits)
result = self.create_result(
frame=hdulist,
slits=slits,
positions9=allpos[9],
positions7=allpos[7],
positions5=allpos[5],
positions3=allpos[3],
DTU=dtub,
ROTANG=rotang,
TSUTC1=tsutc1,
csupos=csupos,
csusens=csusens,
)
return result
class GenerateRectwvCoeff(EmirRecipe):
"""Process images in Stare spectra.
This recipe generates a rectified and wavelength calibrated
image after applying a model (master_rectwv). This calibration
can be refined (using refine_wavecalib_mode != 0).
"""
logger = logging.getLogger(__name__)
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterSpectralFlatFieldRequirement()
master_rectwv = reqs.MasterRectWaveRequirement()
refine_wavecalib_mode = Parameter(
0,
description='Apply wavelength calibration refinement',
optional=True,
choices=[0, 1, 2, 11, 12])
minimum_slitlet_width_mm = Parameter(
float(EMIR_MINIMUM_SLITLET_WIDTH_MM),
description='Minimum width (mm) for a valid slitlet',
optional=True)
maximum_slitlet_width_mm = Parameter(
float(EMIR_MAXIMUM_SLITLET_WIDTH_MM),
description='Maximum width (mm) for a valid slitlet',
optional=True)
global_integer_offsets_mode = Parameter(
'fixed',
description='Global integer offsets computation',
choices=['auto', 'fixed'])
global_integer_offset_x_pix = Parameter(
0,
description='Global integer offset (pixels) in wavelength direction',
optional=True)
global_integer_offset_y_pix = Parameter(
0,
description='Global integer offset (pixels) in spatial direction',
optional=True)
reduced_mos = Result(prods.ProcessedMOS)
rectwv_coeff = Result(RectWaveCoeff)
def run(self, rinput):
self.logger.info('starting rect.+wavecal. reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
self.logger.info(
'Wavelength calibration refinement mode....: {}'.format(
rinput.refine_wavecalib_mode))
self.logger.info(
'Minimum slitlet width (mm)................: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info(
'Maximum slitlet width (mm)................: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info(
'Global integer offsets mode...............: {}'.format(
rinput.global_integer_offsets_mode))
self.logger.info(
'Global integer offset X direction (pixels): {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info(
'Global integer offset Y direction (pixels): {}'.format(
rinput.global_integer_offset_y_pix))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=sigmaclip)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image,
rinput.master_rectwv)
# wavelength calibration refinement
# 0 -> no refinement
# 1 -> apply global offset to all the slitlets (using ARC lines)
# 2 -> apply individual offset to each slitlet (using ARC lines)
# 11 -> apply global offset to all the slitlets (using OH lines)
# 12 -> apply individual offset to each slitlet (using OH lines)
if rinput.refine_wavecalib_mode != 0:
main_header = reduced_image[0].header
# determine useful slitlets
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=rinput.minimum_slitlet_width_mm,
maxwidth=rinput.maximum_slitlet_width_mm)
# remove missing slitlets
if len(rectwv_coeff.missing_slitlets) > 0:
for iremove in rectwv_coeff.missing_slitlets:
if iremove in list_useful_slitlets:
list_useful_slitlets.remove(iremove)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
self.logger.info(
'list of useful slitlets: {}'.format(list_useful_slitlets))
self.logger.info('list of unusable slitlets: {}'.format(
list_not_useful_slitlets))
# retrieve arc/OH lines
catlines_all_wave, catlines_all_flux = retrieve_catlines(
rinput.refine_wavecalib_mode, main_header['grism'])
# global integer offsets
if rinput.global_integer_offsets_mode == 'auto':
if (rinput.global_integer_offset_x_pix != 0) or \
(rinput.global_integer_offset_y_pix != 0):
raise ValueError('Global integer offsets must be zero when'
' mode=auto')
# ToDo: include additional airglow emission lines
self.logger.info('computing synthetic image')
# generate synthetic image
synthetic_raw_data = synthetic_lines_rawdata(
catlines_all_wave, catlines_all_flux, list_useful_slitlets,
rectwv_coeff)
synthetic_raw_header = main_header.copy()
synthetic_raw_header['DATE-OBS'] = \
datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
chistory = 'Synthetic image'
synthetic_raw_header.add_history(chistory)
hdu = fits.PrimaryHDU(synthetic_raw_data.astype('float32'),
header=synthetic_raw_header)
synthetic_raw_image = fits.HDUList([hdu])
if self.intermediate_results:
self.save_intermediate_img(synthetic_raw_image,
'synthetic_raw_image.fits')
# cross-correlation to determine global integer offsets
# (rescaling data arrays to [0, 1] before using skimage
# function)
data1_rs, coef1_rs = rescale_array_to_z1z2(
reduced_image[0].data, (0, 1))
data2_rs, coef2_rs = rescale_array_to_z1z2(
synthetic_raw_data, (0, 1))
shifts, error, diffphase = register_translation(
data1_rs, data2_rs, 100)
self.logger.info(
'global_float_offset_x_pix..: {}'.format(-shifts[1]))
self.logger.info(
'global_float_offset_y_pix..: {}'.format(-shifts[0]))
rectwv_coeff.global_integer_offset_x_pix = \
-int(round(shifts[1]))
rectwv_coeff.global_integer_offset_y_pix = \
-int(round(shifts[0]))
self.logger.info('global_integer_offset_x_pix: {}'.format(
rectwv_coeff.global_integer_offset_x_pix))
self.logger.info('global_integer_offset_y_pix: {}'.format(
rectwv_coeff.global_integer_offset_y_pix))
if self.intermediate_results:
data_product = np.fft.fft2(data1_rs) * \
np.fft.fft2(data2_rs).conj()
cc_image = np.fft.fftshift(np.fft.ifft2(data_product))
power = np.log10(cc_image.real)
hdu_power = fits.PrimaryHDU(power)
hdul_power = fits.HDUList([hdu_power])
hdul_power.writeto('power.fits', overwrite=True)
else:
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
# apply initial rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
self.logger.info(
'Refining wavelength calibration (mode={})'.format(
rinput.refine_wavecalib_mode))
# refine RectWaveCoeff object
rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff(
reduced_mos,
rectwv_coeff,
catlines_all_wave,
catlines_all_flux,
rinput.refine_wavecalib_mode,
list_useful_slitlets,
save_intermediate_results=self.intermediate_results)
self.save_intermediate_img(expected_catalog_lines,
'expected_catalog_lines.fits')
# apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of stare spectra')
result = self.create_result(reduced_mos=reduced_mos,
rectwv_coeff=rectwv_coeff)
return result
def set_base_headers(self, hdr):
newhdr = super(GenerateRectwvCoeff, self).set_base_headers(hdr)
# Update EXP to 0
newhdr['EXP'] = 0
return newhdr
class MaskImagingRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.CoordinateList2DType,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
average_box_row_size = Parameter(
7, 'Number of rows to average for fine centering (odd)')
average_box_col_size = Parameter(
21, 'Number of columns to extract for fine centering (odd)')
fit_peak_npoints = Parameter(
3, 'Number of points to use for fitting the peak (odd)')
# Recipe Products
frame = Result(prods.ProcessedImage)
# derivative = Result(prods.ProcessedImage)
slits = Result(tarray.ArrayType)
positions3 = Result(tarray.ArrayType)
positions5 = Result(tarray.ArrayType)
positions7 = Result(tarray.ArrayType)
positions9 = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
TSUTC1 = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
def run(self, rinput):
self.logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
tsutc1 = hdr['TSUTC1']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise numina.exceptions.RecipeError(error)
self.logger.debug('finding bars')
# Processed array
arr = hdulist[0].data
# Median filter of processed array (two times)
mfilter_size = rinput.median_filter_size
self.logger.debug('median filtering X, %d columns', mfilter_size)
arr_median = median_filter(arr, size=(1, mfilter_size))
self.logger.debug('median filtering X, %d rows', mfilter_size)
arr_median = median_filter(arr_median, size=(mfilter_size, 1))
# Median filter of processed array (two times) in the other direction
# for Y coordinates
self.logger.debug('median filtering Y, %d rows', mfilter_size)
arr_median_alt = median_filter(arr, size=(mfilter_size, 1))
self.logger.debug('median filtering Y, %d columns', mfilter_size)
arr_median_alt = median_filter(arr_median_alt, size=(1, mfilter_size))
xfac = dtur[0] / EMIR_PIXSCALE
yfac = -dtur[1] / EMIR_PIXSCALE
vec = [yfac, xfac]
self.logger.debug('DTU shift is %s', vec)
# and the table of approx positions of the slits
barstab = rinput.bars_nominal_positions
# Currently, we only use fields 0 and 2
# of the nominal positions file
# Number or rows used
# These other parameters cab be tuned also
bstart = 1
bend = 2047
self.logger.debug('ignoring columns outside %d - %d', bstart, bend - 1)
# extract a region to average
wy = (rinput.average_box_row_size // 2)
wx = (rinput.average_box_col_size // 2)
self.logger.debug('extraction window is %d rows, %d cols', 2 * wy + 1,
2 * wx + 1)
# Fit the peak with these points
wfit = 2 * (rinput.fit_peak_npoints // 2) + 1
self.logger.debug('fit with %d points', wfit)
# Minimum threshold
threshold = 5 * EMIR_RON
# Savitsky and Golay (1964) filter to compute the X derivative
# scipy >= xx has a savgol_filter function
# for compatibility we do it manually
allpos = {}
ypos3_kernel = None
slits = numpy.zeros((EMIR_NBARS, 8), dtype='float')
self.logger.info('start finding bars')
for ks in [3, 5, 7, 9]:
self.logger.debug('kernel size is %d', ks)
# S and G kernel for derivative
kw = ks * (ks * ks - 1) / 12.0
coeffs_are = -numpy.arange((1 - ks) // 2, (ks - 1) // 2 + 1) / kw
if ks == 3:
ypos3_kernel = coeffs_are
self.logger.debug('kernel weights are %s', coeffs_are)
self.logger.debug('derive image in X direction')
arr_deriv = convolve1d(arr_median, coeffs_are, axis=-1)
# Axis 0 is
#
self.logger.debug('derive image in Y direction (with kernel=3)')
arr_deriv_alt = convolve1d(arr_median_alt, ypos3_kernel, axis=0)
positions = []
for coords in barstab:
lbarid = int(coords[0])
rbarid = lbarid + EMIR_NBARS
ref_y_coor = coords[1] + vec[1]
poly_coeffs = coords[2:]
prow = coor_to_pix_1d(ref_y_coor) - 1
fits_row = prow + 1 # FITS pixel index
# A function that returns the center of the bar
# given its X position
def center_of_bar(x):
# Pixel values are 0-based
return polyval(x + 1 - vec[0], poly_coeffs) + vec[1] - 1
self.logger.debug('looking for bars with ids %d - %d', lbarid,
rbarid)
self.logger.debug('reference y position is Y %7.2f',
ref_y_coor)
# if ref_y_coor is outlimits, skip this bar
# ref_y_coor is in FITS format
if (ref_y_coor >= 2047) or (ref_y_coor <= 1):
self.logger.debug(
'reference y position is outlimits, skipping')
positions.append([lbarid, fits_row, fits_row, 1, 0, 3])
positions.append([rbarid, fits_row, fits_row, 1, 0, 3])
continue
# Left bar
self.logger.debug('measure left border (%d)', lbarid)
centery, xpos, fwhm, st = char_bar_peak_l(arr_deriv,
prow,
bstart,
bend,
threshold,
center_of_bar,
wx=wx,
wy=wy,
wfit=wfit)
xpos1 = xpos
positions.append(
[lbarid, centery + 1, fits_row, xpos + 1, fwhm, st])
# Right bar
self.logger.debug('measure rigth border (%d)', rbarid)
centery, xpos, fwhm, st = char_bar_peak_r(arr_deriv,
prow,
bstart,
bend,
threshold,
center_of_bar,
wx=wx,
wy=wy,
wfit=wfit)
positions.append(
[rbarid, centery + 1, fits_row, xpos + 1, fwhm, st])
xpos2 = xpos
#
if st == 0:
self.logger.debug('measure top-bottom borders')
try:
y1, y2, statusy = char_bar_height(arr_deriv_alt,
xpos1,
xpos2,
centery,
threshold,
wh=35,
wfit=wfit)
except Exception as error:
self.logger.warning('Error computing height: %s',
error)
statusy = 44
if statusy in [0, 40]:
# Main border is detected
positions[-1][1] = y2 + 1
positions[-2][1] = y2 + 1
else:
# Update status
positions[-1][-1] = 4
positions[-2][-1] = 4
else:
self.logger.debug('slit is not complete')
y1, y2 = 0, 0
# Update positions
self.logger.debug(
'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d',
*positions[-2])
self.logger.debug(
'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d',
*positions[-1])
if ks == 5:
slits[lbarid -
1] = [xpos1, y2, xpos2, y2, xpos2, y1, xpos1, y1]
# FITS coordinates
slits[lbarid - 1] += 1.0
self.logger.debug('inserting bars %d-%d into "slits"',
lbarid, rbarid)
allpos[ks] = numpy.asarray(
positions, dtype='float') # GCS doesn't like lists of lists
self.logger.debug('end finding bars')
result = self.create_result(
frame=hdulist,
slits=slits,
positions9=allpos[9],
positions7=allpos[7],
positions5=allpos[5],
positions3=allpos[3],
DTU=dtub,
ROTANG=rotang,
TSUTC1=tsutc1,
csupos=csupos,
csusens=csusens,
)
return result
class TestSlitMaskDetectionRecipe(EmirRecipe):
# Recipe Requirements
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the Canny algorithm')
canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm')
canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm')
obj_min_size = Parameter(200, 'Minimum size of the slit')
obj_max_size = Parameter(3000, 'Maximum size of the slit')
slit_size_ratio = Parameter(4.0, 'Minimum ratio between height and width for slits')
# Recipe Results
frame = Result(prods.DataFrameType)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
def run(self, rinput):
self.logger.info('starting slit processing')
self.logger.info('basic image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('finding slits')
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
obj_min_size = rinput.obj_min_size
obj_max_size = rinput.obj_max_size
data1 = hdulist[0].data
self.logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize_raw(data2)
# Find edges with Canny
self.logger.debug('Find edges with Canny, sigma %f', canny_sigma)
# These thresholds corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
self.logger.debug('Find edges, Canny high threshold %f', high_threshold)
self.logger.debug('Find edges, Canny low threshold %f', low_threshold)
edges = canny(img_grey, sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Fill edges
self.logger.debug('Fill holes')
fill_slits = ndimage.binary_fill_holes(edges)
self.logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
self.logger.debug('%d objects found', nb_labels)
# Filter on the area of the labeled region
# Perhaps we could ignore this filtering and
# do it later?
self.logger.debug('Filter objects by size')
# Sizes of regions
sizes = numpy.bincount(label_objects.ravel())
self.logger.debug('Min size is %d', obj_min_size)
self.logger.debug('Max size is %d', obj_max_size)
mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size)
# Filter out regions
nids, = numpy.where(mask_sizes)
mm = numpy.in1d(label_objects, nids)
mm.shape = label_objects.shape
fill_slits_clean = numpy.where(mm, 1, 0)
#plt.imshow(fill_slits_clean)
# and relabel
self.logger.debug('Label filtered objects')
relabel_objects, nb_labels = ndimage.label(fill_slits_clean)
self.logger.debug('%d objects found after filtering', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
self.logger.debug('Find regions and centers')
regions = ndimage.find_objects(relabel_objects)
centers = ndimage.center_of_mass(data2, labels=relabel_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=rinput.slit_size_ratio
)
result = self.create_result(frame=hdulist, slitstable=table,
DTU=dtub,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa
)
return result
class TestPointSourceRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
self.logger.info('starting processing for object detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
self.logger.debug('finding point sources')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
data = hdulist[0].data
# Copy needed in numpy 1.7
# This seems already bitswapped??
# FIXME: check this works offline/online
# ndata = data.byteswap().newbyteorder()
# data = data.byteswap(inplace=True).newbyteorder()
snr_detect = 5.0
fwhm = 4.0
npixels = 15
box_shape = [64, 64]
self.logger.info('point source detection2')
self.logger.info('using internal mask to remove corners')
# Corners
mask = numpy.zeros_like(data, dtype='int32')
mask[2000:, 0:80] = 1
mask[2028:, 2000:] = 1
mask[:50, 1950:] = 1
mask[:100, :50] = 1
# Remove corner regions
self.logger.info('compute background map, %s', box_shape)
bkg = sep.Background(data)
self.logger.info('reference fwhm is %5.1f pixels', fwhm)
self.logger.info('detect threshold, %3.1f over background', snr_detect)
self.logger.info('convolve with gaussian kernel, FWHM %3.1f pixels',
fwhm)
sigma = fwhm * gaussian_fwhm_to_sigma
#
kernel = Gaussian2DKernel(sigma)
kernel.normalize()
thresh = snr_detect * bkg.globalrms
data_s = data - bkg.back()
objects, segmap = sep.extract(data - bkg.back(),
thresh,
minarea=npixels,
filter_kernel=kernel.array,
segmentation_map=True,
mask=mask)
fits.writeto('segmap.fits', segmap)
self.logger.info('detected %d objects', len(objects))
# Hardcoded values
rs2 = 15.0
fit_rad = 10.0
flux_min = 1000.0
flux_max = 30000.0
self.logger.debug('Flux limit is %6.1f %6.1f', flux_min, flux_max)
# FIXME: this should be a view, not a copy
xall = objects['x']
yall = objects['y']
mm = numpy.array([xall, yall]).T
self.logger.info('computing FWHM')
# Find objects with pairs inside fit_rad
kdtree = KDTree(mm)
nearobjs = (kdtree.query_ball_tree(kdtree, r=fit_rad))
positions = []
for idx, obj in enumerate(objects):
x0 = obj['x']
y0 = obj['y']
sl = image_box2d(x0, y0, data.shape, (fit_rad, fit_rad))
# sl_sky = image_box2d(x0, y0, data.shape, (rs2, rs2))
part_s = data_s[sl]
# Logical coordinates
xx0 = x0 - sl[1].start
yy0 = y0 - sl[0].start
_, fwhm_x, fwhm_y = compute_fwhm_2d_simple(part_s, xx0, yy0)
if min(fwhm_x, fwhm_x) < 3:
continue
if flux_min > obj['peak'] or flux_max < obj['peak']:
continue
# nobjs is the number of object inside fit_rad
nobjs = len(nearobjs[idx])
flag = 0 if nobjs == 1 else 1
positions.append([idx, x0, y0, obj['peak'], fwhm_x, fwhm_y, flag])
self.logger.info('saving photometry')
positions = numpy.array(positions)
positions_alt = positions
self.logger.info('end processing for object detection')
result = self.create_result(
frame=hdulist,
positions=positions_alt,
positions_alt=positions_alt,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class MultiTwilightFlatRecipe(EmirRecipe):
"""Create a list of twilight flats"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
twflatframes = Result(dt.ListOfType(prods.MasterIntensityFlat))
def run(self, rinput):
results = []
self.logger.info('starting multiflat flat reduction')
# Uncomment this line
# to revert to non-ramp
# flow = self.init_filters(rinput)
saturation = 45000.0
iinfo = gather_info_frames(rinput.obresult.frames)
image_groups = {}
self.logger.info('group images by filter')
for idx, info in enumerate(iinfo):
filt = info['filter']
if filt not in image_groups:
self.logger.debug('new filter %s', filt)
image_groups[filt] = []
img = rinput.obresult.frames[idx]
self.logger.debug('image %s in group %s', img, filt)
image_groups[filt].append(img)
for filt, frames in image_groups.items():
self.logger.info('processing filter %s', filt)
# Uncomment this line and comment the following
# to revert to non-ramp
# res = self.run_per_filter(frames, flow)
try:
res = self.run_per_filter_ramp(frames, saturation=saturation)
results.append(res)
except ValueError:
self.logger.info('filter %s cannot be processed', filt)
self.logger.info('end multiflat flat reduction')
result = self.create_result(twflatframes=results)
return result
def run_per_filter(self, frames, flow):
errors = True
self.logger.debug('using errors: %s', errors)
hdulist = basic_processing_with_combination_frames(frames,
flow,
method=median,
errors=errors)
hdr = hdulist[0].header
self.set_base_headers(hdr)
mm = hdulist[0].data.mean()
self.logger.info('mean value of flat is: %f', mm)
hdr['CCDMEAN'] = mm
self.logger.debug('normalize image')
hdulist[0].data /= mm
if errors:
self.logger.debug('normalize VAR extension')
hdulist['variance'].data /= (mm * mm)
return hdulist
def run_per_filter_ramp(self, frames, saturation, errors=False):
imgs = [frame.open() for frame in frames]
return self.run_img_per_filter_ramp(imgs, saturation, errors)
def run_img_per_filter_ramp(self, imgs, saturation, errors=False):
nimages = len(imgs)
if nimages == 0:
raise ValueError('len(images) == 0')
median_frames = numpy.empty((nimages, ))
exptime_frames = []
utc_frames = []
# generate 3D cube
bshape = self.datamodel.shape
flat_frames = numpy.empty((bshape[0], bshape[1], nimages))
for idx, image in enumerate(imgs):
flat_frames[:, :, idx] = image['primary'].data
exptime_frames.append(image[0].header['EXPTIME'])
median_frames[idx] = numpy.median(image['primary'].data)
utc_frames.append(image[0].header['UTC'])
self.logger.debug('image %d exptime %f median %f UTC %s', idx,
exptime_frames[idx], median_frames[idx],
utc_frames[-1])
# filter saturated images
good_images = median_frames < saturation
ngood_images = good_images.sum()
slope_scaled_var = None
slope_scaled_num = None
if ngood_images == 0:
self.logger.warning('We have only %d good images', ngood_images)
raise ValueError('No images under saturation')
elif ngood_images < 2:
self.logger.warning('We have only %d good images', ngood_images)
# Reference image
ref_image = imgs[0]
slope_scaled = numpy.ones(bshape) * exptime_frames[0]
if errors:
slope_scaled_var = numpy.zeros_like(slope_scaled)
slope_scaled_num = numpy.zeros_like(
slope_scaled, dtype='int16') + ngood_images
else:
nsaturated = nimages - good_images.sum()
if nsaturated > 0:
self.logger.debug(
'we have %d images with median value over saturation (%f)',
nsaturated, saturation)
m = flat_frames[:, :, good_images]
# Reshape array to obtain a 2D array
m_r = m.reshape((bshape[0] * bshape[1], ngood_images))
self.logger.debug('fitting slopes with Theil-Sen')
# self.logger.debug('fitting slopes with mean-squares')
# ll = nppol.polyfit(median_frames[good_images], m_r.T, deg=1)
ll = self.filter_nsigma(median_frames[good_images], m_r.T)
slope = ll[1].reshape(bshape)
base = ll[0].reshape(bshape)
# First good frame
index_of_first_good = numpy.nonzero(good_images)[0][0]
slope_scaled = slope * exptime_frames[index_of_first_good]
if errors:
slope_scaled_var = numpy.zeros_like(slope_scaled)
slope_scaled_num = numpy.zeros_like(
slope_scaled, dtype='int16') + ngood_images
cdata = []
for idx, img in enumerate(imgs):
if good_images[idx]:
cdata.append(img)
result = self.compose_result(cdata, slope_scaled, errors,
slope_scaled_var, slope_scaled_num)
return result
def filter_nsigma(self, median_val, image_val, nsigma=10.0, nloop=1):
# Initial estimation
ll = fit_theil_sen(median_val, image_val)
ni = 0
self.logger.debug('initial estimation')
while ni < nloop:
# Prediction
self.logger.debug('loop %d', ni + 1)
base, slope = ll
image_val_pred = base + median_val[:, numpy.newaxis] * slope
image_diff = image_val - image_val_pred
# Compute MAD
mad = compute_mad(image_diff)
sigma_robust = nsigma * 1.4826 * mad
self.logger.debug('compute robust std deviation')
self.logger.debug('min %7.1f max %7.1f mean %7.1f',
sigma_robust.min(), sigma_robust.max(),
sigma_robust.mean())
# Check values over sigma
mask_over = numpy.abs(image_diff) >= sigma_robust[:, numpy.newaxis]
self.logger.debug('values over sigma: %d', mask_over.sum())
# Insert expected values in image
# instead of masking
image_val[mask_over] = image_val_pred[mask_over]
#
self.logger.debug('Theil-Sen fit')
ll = fit_theil_sen(median_val, image_val)
ni += 1
return ll
def compose_result(self,
imgs,
slope_scaled,
errors=False,
slope_scaled_var=None,
slope_scaled_num=None):
self.logger.debug('update result header')
cnum = len(imgs)
method_name = 'Theil-Sen'
base_header = imgs[0][0].header
cdata = imgs
hdu = fits.PrimaryHDU(data=slope_scaled, header=base_header)
self.set_base_headers(hdu.header)
hdu.header['history'] = "Combined %d images using '%s'" % (cnum,
method_name)
hdu.header['history'] = 'Combination time {}'.format(
datetime.datetime.utcnow().isoformat())
for img in cdata:
hdu.header['history'] = "Image {}".format(
self.datamodel.get_imgid(img))
prevnum = base_header.get('NUM-NCOM', 1)
hdu.header['NUM-NCOM'] = prevnum * cnum
hdu.header['UUID'] = str(uuid.uuid1())
# Headers of last image
hdu.header['TSUTC2'] = cdata[-1][0].header['TSUTC2']
# TODO: use BPM to compute mean
mm = hdu.data.mean()
self.logger.info('mean value of flat is: %f', mm)
hdu.header['CCDMEAN'] = mm
self.logger.debug('normalize image')
hdu.data /= mm
if errors:
varhdu = fits.ImageHDU(slope_scaled_var, name='VARIANCE')
num = fits.ImageHDU(slope_scaled_num, name='MAP')
self.logger.debug('normalize VAR extension')
varhdu.data /= (mm * mm)
result = fits.HDUList([hdu, varhdu, num])
else:
result = fits.HDUList([hdu])
return result
